[med-svn] [Git][med-team/cwltool][upstream] New upstream version 3.1.20221201130942
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Wed Dec 21 11:53:04 GMT 2022
Michael R. Crusoe pushed to branch upstream at Debian Med / cwltool
Commits:
dfc36fe7 by Michael R. Crusoe at 2022-12-21T12:28:35+01:00
New upstream version 3.1.20221201130942
- - - - -
25 changed files:
- PKG-INFO
- cwltool.egg-info/PKG-INFO
- cwltool/builder.py
- cwltool/command_line_tool.py
- cwltool/context.py
- cwltool/cwlrdf.py
- cwltool/docker.py
- cwltool/job.py
- cwltool/load_tool.py
- cwltool/main.py
- cwltool/pack.py
- cwltool/process.py
- cwltool/procgenerator.py
- cwltool/update.py
- cwltool/validate_js.py
- cwltool/workflow.py
- mypy-requirements.txt
- mypy.ini
- setup.cfg
- tests/test_anon_types.py
- tests/test_examples.py
- tests/test_mpi.py
- tests/test_path_checks.py
- tests/test_streaming.py
- tests/test_tmpdir.py
Changes:
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20221109155812
+Version: 3.1.20221201130942
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
=====================================
cwltool.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20221109155812
+Version: 3.1.20221201130942
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
=====================================
cwltool/builder.py
=====================================
@@ -17,7 +17,6 @@ from typing import (
from cwl_utils import expression
from cwl_utils.file_formats import check_format
from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap
from schema_salad.avro.schema import Names, Schema, make_avsc_object
from schema_salad.exceptions import ValidationException
from schema_salad.sourceline import SourceLine
@@ -25,6 +24,8 @@ from schema_salad.utils import convert_to_dict, json_dumps
from schema_salad.validate import validate
from typing_extensions import TYPE_CHECKING, Type # pylint: disable=unused-import
+from ruamel.yaml.comments import CommentedMap
+
from .errors import WorkflowException
from .loghandler import _logger
from .mutation import MutationManager
=====================================
cwltool/command_line_tool.py
=====================================
@@ -30,7 +30,6 @@ from typing import (
)
import shellescape
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.avro.schema import Schema
from schema_salad.exceptions import ValidationException
from schema_salad.ref_resolver import file_uri, uri_file_path
@@ -39,6 +38,8 @@ from schema_salad.utils import json_dumps
from schema_salad.validate import validate_ex
from typing_extensions import TYPE_CHECKING, Type
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from .builder import (
INPUT_OBJ_VOCAB,
Builder,
@@ -46,7 +47,7 @@ from .builder import (
substitute,
)
from .context import LoadingContext, RuntimeContext, getdefault
-from .docker import DockerCommandLineJob
+from .docker import DockerCommandLineJob, PodmanCommandLineJob
from .errors import UnsupportedRequirement, WorkflowException
from .flatten import flatten
from .job import CommandLineJob, JobBase
@@ -460,6 +461,8 @@ class CommandLineTool(Process):
raise UnsupportedRequirement(
"Both Docker and MPI have been hinted - don't know what to do"
)
+ if runtimeContext.podman:
+ return PodmanCommandLineJob
return DockerCommandLineJob
if dockerRequired:
raise UnsupportedRequirement(
=====================================
cwltool/context.py
=====================================
@@ -17,13 +17,14 @@ from typing import (
Union,
)
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
-from ruamel.yaml.comments import CommentedMap
from schema_salad.avro.schema import Names
from schema_salad.ref_resolver import Loader
from schema_salad.utils import FetcherCallableType
from typing_extensions import TYPE_CHECKING
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from ruamel.yaml.comments import CommentedMap
+
from .builder import Builder
from .mpi import MpiConfig
from .mutation import MutationManager
=====================================
cwltool/cwlrdf.py
=====================================
@@ -4,10 +4,11 @@ from typing import Any, Dict, Iterator, Optional, TextIO, Union, cast
from rdflib import Graph
from rdflib.query import ResultRow
-from ruamel.yaml.comments import CommentedMap
from schema_salad.jsonld_context import makerdf
from schema_salad.utils import ContextType
+from ruamel.yaml.comments import CommentedMap
+
from .cwlviewer import CWLViewer
from .process import Process
=====================================
cwltool/docker.py
=====================================
@@ -90,9 +90,10 @@ class DockerCommandLineJob(ContainerCommandLineJob):
) -> None:
"""Initialize a command line builder using the Docker software container engine."""
super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+ self.docker_exec = "docker"
- @staticmethod
def get_image(
+ self,
docker_requirement: Dict[str, str],
pull_image: bool,
force_pull: bool,
@@ -117,7 +118,7 @@ class DockerCommandLineJob(ContainerCommandLineJob):
for line in (
subprocess.check_output( # nosec
- ["docker", "images", "--no-trunc", "--all"]
+ [self.docker_exec, "images", "--no-trunc", "--all"]
)
.decode("utf-8")
.splitlines()
@@ -151,7 +152,7 @@ class DockerCommandLineJob(ContainerCommandLineJob):
if (force_pull or not found) and pull_image:
cmd = [] # type: List[str]
if "dockerPull" in docker_requirement:
- cmd = ["docker", "pull", str(docker_requirement["dockerPull"])]
+ cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])]
_logger.info(str(cmd))
subprocess.check_call(cmd, stdout=sys.stderr) # nosec
found = True
@@ -160,7 +161,7 @@ class DockerCommandLineJob(ContainerCommandLineJob):
with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
dfile.write(docker_requirement["dockerFile"])
cmd = [
- "docker",
+ self.docker_exec,
"build",
"--tag=%s" % str(docker_requirement["dockerImageId"]),
dockerfile_dir,
@@ -169,7 +170,7 @@ class DockerCommandLineJob(ContainerCommandLineJob):
subprocess.check_call(cmd, stdout=sys.stderr) # nosec
found = True
elif "dockerLoad" in docker_requirement:
- cmd = ["docker", "load"]
+ cmd = [self.docker_exec, "load"]
_logger.info(str(cmd))
if os.path.exists(docker_requirement["dockerLoad"]):
_logger.info(
@@ -203,7 +204,7 @@ class DockerCommandLineJob(ContainerCommandLineJob):
found = True
elif "dockerImport" in docker_requirement:
cmd = [
- "docker",
+ self.docker_exec,
"import",
str(docker_requirement["dockerImport"]),
str(docker_requirement["dockerImageId"]),
@@ -225,8 +226,8 @@ class DockerCommandLineJob(ContainerCommandLineJob):
force_pull: bool,
tmp_outdir_prefix: str,
) -> Optional[str]:
- if not shutil.which("docker"):
- raise WorkflowException("docker executable is not available")
+ if not shutil.which(self.docker_exec):
+ raise WorkflowException(f"{self.docker_exec} executable is not available")
if self.get_image(
cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix
@@ -341,10 +342,10 @@ class DockerCommandLineJob(ContainerCommandLineJob):
runtime = [user_space_docker_cmd, "--quiet", "run", "--nobanner"]
else:
runtime = [user_space_docker_cmd, "run"]
- elif runtimeContext.podman:
- runtime = ["podman", "run", "-i", "--userns=keep-id"]
else:
- runtime = ["docker", "run", "-i"]
+ runtime = [self.docker_exec, "run", "-i"]
+ if runtimeContext.podman:
+ runtime.append("--userns=keep-id")
self.append_volume(
runtime, os.path.realpath(self.outdir), self.builder.outdir, writable=True
)
@@ -460,3 +461,20 @@ class DockerCommandLineJob(ContainerCommandLineJob):
)
return runtime, cidfile_path
+
+
+class PodmanCommandLineJob(DockerCommandLineJob):
+ """Runs a CommandLineJob in a software container using the podman engine."""
+
+ def __init__(
+ self,
+ builder: Builder,
+ joborder: CWLObjectType,
+ make_path_mapper: Callable[..., PathMapper],
+ requirements: List[CWLObjectType],
+ hints: List[CWLObjectType],
+ name: str,
+ ) -> None:
+ """Initialize a command line builder using the Podman software container engine."""
+ super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+ self.docker_exec = "podman"
=====================================
cwltool/job.py
=====================================
@@ -874,6 +874,7 @@ class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
cidfile,
runtimeContext.tmpdir_prefix,
not bool(runtimeContext.cidfile_dir),
+ "podman" if runtimeContext.podman else "docker",
)
elif runtimeContext.user_space_docker_cmd:
monitor_function = functools.partial(self.process_monitor)
@@ -884,6 +885,7 @@ class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
cidfile: str,
tmpdir_prefix: str,
cleanup_cidfile: bool,
+ docker_exe: str,
process, # type: subprocess.Popen[str]
) -> None:
"""Record memory usage of the running Docker container."""
@@ -901,7 +903,7 @@ class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
os.remove(cidfile)
except OSError as exc:
_logger.warning(
- "Ignored error cleaning up Docker cidfile: %s", exc
+ "Ignored error cleaning up %s cidfile: %s", docker_exe, exc
)
return
try:
@@ -915,15 +917,19 @@ class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
stats_file_name = stats_file.name
try:
with open(stats_file_name, mode="w") as stats_file_handle:
+ cmds = [docker_exe, "stats"]
+ if "podman" not in docker_exe:
+ cmds.append("--no-trunc")
+ cmds.extend(["--format", "{{.MemPerc}}", cid])
stats_proc = subprocess.Popen( # nosec
- ["docker", "stats", "--no-trunc", "--format", "{{.MemPerc}}", cid],
+ cmds,
stdout=stats_file_handle,
stderr=subprocess.DEVNULL,
)
process.wait()
stats_proc.kill()
except OSError as exc:
- _logger.warning("Ignored error with docker stats: %s", exc)
+ _logger.warning("Ignored error with %s stats: %s", docker_exe, exc)
return
max_mem_percent = 0 # type: float
mem_percent = 0 # type: float
@@ -938,8 +944,10 @@ class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
)
if mem_percent > max_mem_percent:
max_mem_percent = mem_percent
- except ValueError:
- break
+ except ValueError as exc:
+ _logger.debug(
+ "%s stats parsing error in line %s: %s", docker_exe, line, exc
+ )
_logger.info(
"[job %s] Max memory used: %iMiB",
self.name,
=====================================
cwltool/load_tool.py
=====================================
@@ -21,7 +21,6 @@ from typing import (
)
from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.exceptions import ValidationException
from schema_salad.ref_resolver import Loader, file_uri
from schema_salad.schema import validate_doc
@@ -34,6 +33,8 @@ from schema_salad.utils import (
json_dumps,
)
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from . import CWL_CONTENT_TYPES, process, update
from .context import LoadingContext
from .errors import GraphTargetMissingException
=====================================
cwltool/main.py
=====================================
@@ -36,14 +36,15 @@ from typing import (
import argcomplete
import coloredlogs
import pkg_resources # part of setuptools
-import ruamel.yaml
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
from schema_salad.exceptions import ValidationException
from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
from schema_salad.sourceline import cmap, strip_dup_lineno
from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts
+import ruamel.yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+from ruamel.yaml.main import YAML
+
from . import CWL_CONTENT_TYPES, workflow
from .argparser import arg_parser, generate_parser, get_default_args
from .context import LoadingContext, RuntimeContext, getdefault
@@ -105,6 +106,8 @@ from .utils import (
)
from .workflow import Workflow
+docker_exe: str
+
def _terminate_processes() -> None:
"""Kill all spawned processes.
@@ -117,6 +120,7 @@ def _terminate_processes() -> None:
continuing to execute while it kills the processes that they've
spawned. This may occasionally lead to unexpected behaviour.
"""
+ global docker_exe
# It's possible that another thread will spawn a new task while
# we're executing, so it's not safe to use a for loop here.
while processes_to_kill:
@@ -130,7 +134,7 @@ def _terminate_processes() -> None:
try:
with open(cidfile[0]) as inp_stream:
p = subprocess.Popen( # nosec
- ["docker", "kill", inp_stream.read()], shell=False # nosec
+ [docker_exe, "kill", inp_stream.read()], shell=False # nosec
)
try:
p.wait(timeout=10)
@@ -1009,6 +1013,7 @@ def main(
stderr_handler = _logger.handlers[-1]
workflowobj = None
prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None
+ global docker_exe
try:
if args is None:
if argsl is None:
@@ -1030,6 +1035,10 @@ def main(
else:
runtimeContext = runtimeContext.copy()
+ if runtimeContext.podman:
+ docker_exe = "podman"
+ else:
+ docker_exe = "docker"
# If caller parsed its own arguments, it may not include every
# cwltool option, so fill in defaults to avoid crashing when
# dereferencing them in args.
=====================================
cwltool/pack.py
=====================================
@@ -14,10 +14,11 @@ from typing import (
cast,
)
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.ref_resolver import Loader, SubLoader
from schema_salad.utils import ResolveType
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from .context import LoadingContext
from .load_tool import fetch_document, resolve_and_validate_document
from .process import shortname, uniquename
=====================================
cwltool/process.py
=====================================
@@ -35,7 +35,6 @@ from cwl_utils import expression
from mypy_extensions import mypyc_attr
from pkg_resources import resource_stream
from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.avro.schema import (
Names,
Schema,
@@ -50,6 +49,8 @@ from schema_salad.utils import convert_to_dict
from schema_salad.validate import avro_type_name, validate_ex
from typing_extensions import TYPE_CHECKING
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from .builder import INPUT_OBJ_VOCAB, Builder
from .context import LoadingContext, RuntimeContext, getdefault
from .errors import UnsupportedRequirement, WorkflowException
=====================================
cwltool/procgenerator.py
=====================================
@@ -1,10 +1,11 @@
import copy
from typing import Dict, Optional, Tuple, cast
-from ruamel.yaml.comments import CommentedMap
from schema_salad.exceptions import ValidationException
from schema_salad.sourceline import indent
+from ruamel.yaml.comments import CommentedMap
+
from .context import LoadingContext, RuntimeContext
from .errors import WorkflowException
from .load_tool import load_tool
=====================================
cwltool/update.py
=====================================
@@ -11,11 +11,12 @@ from typing import (
cast,
)
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.exceptions import ValidationException
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import SourceLine
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from .loghandler import _logger
from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field
=====================================
cwltool/validate_js.py
=====================================
@@ -19,7 +19,6 @@ from cwl_utils.errors import SubstitutionError
from cwl_utils.expression import scanner as scan_expression
from cwl_utils.sandboxjs import code_fragment_to_js, exec_js_process
from pkg_resources import resource_stream
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.avro.schema import (
ArraySchema,
EnumSchema,
@@ -31,6 +30,8 @@ from schema_salad.sourceline import SourceLine
from schema_salad.utils import json_dumps
from schema_salad.validate import validate_ex
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
from .errors import WorkflowException
from .loghandler import _logger
=====================================
cwltool/workflow.py
=====================================
@@ -17,10 +17,11 @@ from typing import (
)
from uuid import UUID
-from ruamel.yaml.comments import CommentedMap
from schema_salad.exceptions import ValidationException
from schema_salad.sourceline import SourceLine, indent
+from ruamel.yaml.comments import CommentedMap
+
from . import command_line_tool, context, procgenerator
from .checker import circular_dependency_checker, loop_checker, static_checker
from .context import LoadingContext, RuntimeContext, getdefault
=====================================
mypy-requirements.txt
=====================================
@@ -1,4 +1,4 @@
-mypy==0.990
+mypy==0.991
ruamel.yaml>=0.16.0,<0.17.22
schema-salad>=8.2.20211104054942,<9
cwl-utils >=0.19
=====================================
mypy.ini
=====================================
@@ -6,15 +6,6 @@ show_error_codes = true
pretty = true
warn_unreachable = True
-[mypy-schema_salad.tests.*]
-ignore_errors = True
-
-[mypy-ruamel.*]
-ignore_errors = True
-
-[mypy-past.builtins.*]
-ignore_missing_imports = True
-
[mypy-galaxy.tool_util.*]
ignore_missing_imports = True
=====================================
setup.cfg
=====================================
@@ -18,6 +18,6 @@ line_length = 88
ignore-words-list = ORE,ore,RO,ro
[egg_info]
-tag_build = .20221109155812
+tag_build = .20221201130942
tag_date = 0
=====================================
tests/test_anon_types.py
=====================================
@@ -1,11 +1,11 @@
from typing import cast
import pytest
-from ruamel.yaml.comments import CommentedMap
from schema_salad.sourceline import cmap
from cwltool.command_line_tool import CommandLineTool
from cwltool.context import LoadingContext
+from ruamel.yaml.comments import CommentedMap
snippet = cast(
CommentedMap,
=====================================
tests/test_examples.py
=====================================
@@ -15,7 +15,6 @@ import pydot
import pytest
from cwl_utils.errors import JavascriptException
from cwl_utils.sandboxjs import param_re
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.exceptions import ValidationException
import cwltool.checker
@@ -29,6 +28,7 @@ from cwltool.errors import WorkflowException
from cwltool.main import main
from cwltool.process import CWL_IANA
from cwltool.utils import CWLObjectType, dedup
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
from .util import get_data, get_main_output, needs_docker, working_directory
=====================================
tests/test_mpi.py
=====================================
@@ -8,7 +8,6 @@ from typing import Any, Generator, List, MutableMapping, Optional, Tuple
import pkg_resources
import pytest
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.avro.schema import Names
from schema_salad.utils import yaml_no_ts
@@ -19,6 +18,7 @@ from cwltool.command_line_tool import CommandLineTool
from cwltool.context import LoadingContext, RuntimeContext
from cwltool.main import main
from cwltool.mpi import MpiConfig, MPIRequirementName
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
from .util import get_data, working_directory
=====================================
tests/test_path_checks.py
=====================================
@@ -4,7 +4,6 @@ from pathlib import Path
from typing import IO, Any, List, cast
import pytest
-from ruamel.yaml.comments import CommentedMap
from schema_salad.sourceline import cmap
from cwltool.command_line_tool import CommandLineTool
@@ -13,6 +12,7 @@ from cwltool.main import main
from cwltool.stdfsaccess import StdFsAccess
from cwltool.update import INTERNAL_VERSION
from cwltool.utils import CWLObjectType
+from ruamel.yaml.comments import CommentedMap
from .util import needs_docker
=====================================
tests/test_streaming.py
=====================================
@@ -4,7 +4,6 @@ from pathlib import Path
from typing import cast
import pytest
-from ruamel.yaml.comments import CommentedMap
from schema_salad.sourceline import cmap
from cwltool.command_line_tool import CommandLineTool
@@ -13,6 +12,7 @@ from cwltool.errors import WorkflowException
from cwltool.job import JobBase
from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION
from cwltool.utils import CWLObjectType
+from ruamel.yaml.comments import CommentedMap
from .util import get_data
=====================================
tests/test_tmpdir.py
=====================================
@@ -5,7 +5,6 @@ from pathlib import Path
from typing import List, cast
import pytest
-from ruamel.yaml.comments import CommentedMap
from schema_salad.avro import schema
from schema_salad.sourceline import cmap
@@ -19,6 +18,7 @@ from cwltool.pathmapper import MapperEnt, PathMapper
from cwltool.stdfsaccess import StdFsAccess
from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION
from cwltool.utils import create_tmp_dir
+from ruamel.yaml.comments import CommentedMap
from .util import get_data, needs_docker
@@ -116,8 +116,39 @@ def test_dockerfile_tmpdir_prefix(
monkeypatch.setattr(
target=subprocess, name="check_call", value=lambda *args, **kwargs: True
)
- tmp_outdir_prefix = tmp_path / "1"
- assert DockerCommandLineJob.get_image(
+ (tmp_path / "out").mkdir()
+ tmp_outdir_prefix = tmp_path / "out" / "1"
+ (tmp_path / "3").mkdir()
+ tmpdir_prefix = str(tmp_path / "3" / "ttmp")
+ runtime_context = RuntimeContext(
+ {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None}
+ )
+ builder = Builder(
+ {},
+ [],
+ [],
+ {},
+ schema.Names(),
+ [],
+ [],
+ {},
+ None,
+ None,
+ StdFsAccess,
+ StdFsAccess(""),
+ None,
+ 0.1,
+ False,
+ False,
+ False,
+ "",
+ runtime_context.get_outdir(),
+ runtime_context.get_tmpdir(),
+ runtime_context.get_stagedir(),
+ INTERNAL_VERSION,
+ "docker",
+ )
+ assert DockerCommandLineJob(builder, {}, PathMapper, [], [], "").get_image(
{
"class": "DockerRequirement",
"dockerFile": "FROM debian:stable-slim",
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/dfc36fe70a062516833d6d54444ca29ff9944bea
--
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/dfc36fe70a062516833d6d54444ca29ff9944bea
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20221221/a06e4c8b/attachment-0001.htm>
More information about the debian-med-commit
mailing list