[med-svn] [Git][med-team/cwltool][upstream] New upstream version 3.2.20260413085819
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Thu Apr 16 14:14:47 BST 2026
Michael R. Crusoe pushed to branch upstream at Debian Med / cwltool
Commits:
40a00967 by Michael R. Crusoe at 2026-04-16T14:08:25+02:00
New upstream version 3.2.20260413085819
- - - - -
23 changed files:
- PKG-INFO
- cwltool.egg-info/PKG-INFO
- cwltool/builder.py
- cwltool/checker.py
- cwltool/command_line_tool.py
- cwltool/cwlprov/provenance_profile.py
- cwltool/cwlprov/ro.py
- cwltool/job.py
- cwltool/load_tool.py
- cwltool/main.py
- cwltool/pack.py
- cwltool/pathmapper.py
- cwltool/process.py
- cwltool/update.py
- cwltool/utils.py
- cwltool/workflow_job.py
- mypy-requirements.txt
- pyproject.toml
- tests/test_examples.py
- tests/test_http_input.py
- tests/test_js_sandbox.py
- tests/test_parallel.py
- tests/test_secrets.py
Changes:
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: cwltool
-Version: 3.2.20260411152607
+Version: 3.2.20260413085819
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Download-URL: https://github.com/common-workflow-language/cwltool
=====================================
cwltool.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: cwltool
-Version: 3.2.20260411152607
+Version: 3.2.20260413085819
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Download-URL: https://github.com/common-workflow-language/cwltool
=====================================
cwltool/builder.py
=====================================
@@ -94,7 +94,7 @@ class Builder(HasReqsHints):
def __init__(
self,
job: CWLObjectType,
- files: list[CWLFileType | CWLDirectoryType],
+ files: MutableSequence[CWLFileType | CWLDirectoryType],
bindings: list[CWLObjectType],
schemaDefs: MutableMapping[str, CWLObjectType],
names: Names,
@@ -458,8 +458,8 @@ class Builder(HasReqsHints):
if not found:
def addsf(
- files: MutableSequence[CWLObjectType],
- newsf: CWLObjectType,
+ files: MutableSequence[CWLFileType | CWLDirectoryType],
+ newsf: CWLFileType | CWLDirectoryType,
) -> None:
for f in files:
if f["location"] == newsf["location"]:
@@ -469,23 +469,19 @@ class Builder(HasReqsHints):
if isinstance(sfname, MutableMapping):
addsf(
- cast(
- MutableSequence[CWLObjectType],
- fdatum["secondaryFiles"],
- ),
- sfname,
+ fdatum["secondaryFiles"],
+ cast(CWLFileType | CWLDirectoryType, sfname),
)
elif discover_secondaryFiles and self.fs_access.exists(sf_location):
addsf(
- cast(
- MutableSequence[CWLObjectType],
- fdatum["secondaryFiles"],
+ fdatum["secondaryFiles"],
+ CWLFileType(
+ **{
+ "location": sf_location,
+ "basename": sfname,
+ "class": "File",
+ }
),
- {
- "location": sf_location,
- "basename": sfname,
- "class": "File",
- },
)
elif sf_required:
raise SourceLine(
=====================================
cwltool/checker.py
=====================================
@@ -3,7 +3,7 @@
from collections.abc import Iterator, MutableMapping, MutableSequence, Sized
from typing import Any, Literal, NamedTuple, Optional, Union, cast
-from cwl_utils.types import CWLObjectType, CWLOutputType
+from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType
from schema_salad.exceptions import ValidationException
from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno
from schema_salad.utils import json_dumps
@@ -11,7 +11,7 @@ from schema_salad.utils import json_dumps
from .errors import WorkflowException
from .loghandler import _logger
from .process import shortname
-from .utils import SinkType, aslist
+from .utils import aslist
def _get_type(tp: Any) -> Any:
@@ -22,8 +22,8 @@ def _get_type(tp: Any) -> Any:
def check_types(
- srctype: SinkType,
- sinktype: SinkType,
+ srctype: SinkType | None,
+ sinktype: SinkType | None,
linkMerge: str | None,
pickValue: str | None,
valueFrom: str | None,
@@ -83,18 +83,20 @@ def check_types(
raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}")
-def merge_flatten_type(src: SinkType) -> CWLOutputType:
+def merge_flatten_type(src: SinkType | None) -> CWLOutputType | None:
"""Return the merge flattened type of the source type."""
match src:
case MutableSequence():
- return [merge_flatten_type(cast(SinkType, t)) for t in src]
+ return [merge_flatten_type(t) for t in src]
case {"type": "array"}:
return src
case _:
return {"items": src, "type": "array"}
-def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool:
+def can_assign_src_to_sink(
+ src: SinkType | None, sink: SinkType | None, strict: bool = False
+) -> bool:
"""
Check for identical type specifications, ignoring extra keys like inputBinding.
@@ -112,8 +114,8 @@ def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool =
return False
if src["type"] == "array" and sink["type"] == "array":
return can_assign_src_to_sink(
- cast(SinkType, src["items"]),
- cast(SinkType, sink["items"]),
+ cast(MutableSequence[CWLOutputType | None], src["items"]),
+ cast(MutableSequence[CWLOutputType | None], sink["items"]),
strict,
)
if src["type"] == "record" and sink["type"] == "record":
@@ -128,15 +130,15 @@ def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool =
if strict:
return False
return True
- return can_assign_src_to_sink(cast(SinkType, src["type"]), sink["type"], strict)
+ return can_assign_src_to_sink(src["type"], sink["type"], strict)
if isinstance(src, MutableSequence):
if strict:
for this_src in src:
- if not can_assign_src_to_sink(cast(SinkType, this_src), sink):
+ if not can_assign_src_to_sink(this_src, sink):
return False
return True
for this_src in src:
- if this_src != "null" and can_assign_src_to_sink(cast(SinkType, this_src), sink):
+ if this_src != "null" and can_assign_src_to_sink(this_src, sink):
return True
return False
if isinstance(sink, MutableSequence):
=====================================
cwltool/command_line_tool.py
=====================================
@@ -12,19 +12,21 @@ import shutil
import threading
import urllib
import urllib.parse
-from collections.abc import (
- Generator,
- Iterable,
- Mapping,
- MutableMapping,
- MutableSequence,
-)
+from collections.abc import Generator, Mapping, MutableMapping, MutableSequence
from enum import Enum
from functools import cmp_to_key, partial
from re import Pattern
from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast
-from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ CWLOutputType,
+ is_directory,
+ is_file,
+ is_file_or_directory,
+)
from mypy_extensions import mypyc_attr
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.avro.schema import RecordSchema
@@ -177,12 +179,7 @@ class ExpressionJob:
try:
normalizeFilesDirs(self.builder.job)
ev = self.builder.do_eval(self.script)
- normalizeFilesDirs(
- cast(
- None | MutableSequence[CWLFileType | CWLDirectoryType] | CWLObjectType,
- ev,
- )
- )
+ normalizeFilesDirs(ev)
if self.output_callback:
self.output_callback(cast(Optional[CWLObjectType], ev), "success")
except WorkflowException as err:
@@ -232,7 +229,9 @@ def remove_path(f: CWLObjectType) -> None:
del f["path"]
-def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectType | None:
+def revmap_file(
+ builder: Builder, outdir: str, f: CWLFileType | CWLDirectoryType
+) -> CWLFileType | CWLDirectoryType:
"""
Remap a file from internal path to external path.
@@ -251,18 +250,18 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectTyp
# quoted any further.
if "location" in f and "path" not in f:
- location = cast(str, f["location"])
+ location = f["location"]
if location.startswith("file://"):
f["path"] = uri_file_path(location)
else:
- f["location"] = builder.fs_access.join(outdir, cast(str, f["location"]))
+ f["location"] = builder.fs_access.join(outdir, f["location"])
return f
- if "dirname" in f:
+ if is_file(f) and "dirname" in f:
del f["dirname"]
if "path" in f:
- path = builder.fs_access.join(builder.outdir, cast(str, f["path"]))
+ path = builder.fs_access.join(builder.outdir, f["path"])
uripath = file_uri(path)
del f["path"]
@@ -372,9 +371,9 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None:
location = cast(str, ob["location"])
if location.startswith("_:"):
pass
- if ob["class"] == "File" and not fs_access.isfile(location):
+ if is_file(ob) and not fs_access.isfile(location):
raise ValidationException("Does not exist or is not a File: '%s'" % location)
- if ob["class"] == "Directory" and not fs_access.isdir(location):
+ if is_directory(ob) and not fs_access.isdir(location):
raise ValidationException("Does not exist or is not a Directory: '%s'" % location)
@@ -600,10 +599,7 @@ class CommandLineTool(Process):
filelist = True
for e1 in entry3:
- if not isinstance(e1, MutableMapping) or e1.get("class") not in (
- "File",
- "Directory",
- ):
+ if not is_file_or_directory(e1):
filelist = False
break
@@ -627,19 +623,11 @@ class CommandLineTool(Process):
writable = bool(t.get("writable", False))
et_writable = writable
- if isinstance(entry3, Mapping) and entry3.get("class") in (
- "File",
- "Directory",
- ):
- entry_fd = cast(CWLFileType | CWLDirectoryType, entry3)
- if (
- writable
- and entry_fd["class"] == "File"
- and "secondaryFiles" in entry_fd
- ):
- for sf in entry_fd["secondaryFiles"]:
+ if is_file_or_directory(entry3):
+ if writable and is_file(entry3) and "secondaryFiles" in entry3:
+ for sf in entry3["secondaryFiles"]:
sf["writable"] = writable # type: ignore[typeddict-unknown-key]
- et_entry: str | CWLFileType | CWLDirectoryType = entry_fd
+ et_entry: str | CWLFileType | CWLDirectoryType = entry3
else:
if isinstance(entry3, str):
et_entry = entry3
@@ -728,7 +716,7 @@ class CommandLineTool(Process):
)
)
- if t2["entry"].get("class") not in ("File", "Directory"):
+ if not is_file_or_directory(t2["entry"]):
raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError(
"Entry at index %s of listing is not a File or Directory object, was %s"
% (i, t2)
@@ -745,7 +733,7 @@ class CommandLineTool(Process):
ls[i] = t2["entry"]
for i, t3 in enumerate(ls):
- if t3.get("class") not in ("File", "Directory"):
+ if not is_file_or_directory(t3):
# Check that every item is a File or Directory object now
raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError(
f"Entry at index {i} of listing is not a Dirent, File or "
@@ -794,7 +782,7 @@ class CommandLineTool(Process):
dirname = os.path.join(builder.outdir, os.path.dirname(basename))
entry4["dirname"] = dirname # type: ignore[typeddict-unknown-key]
entry4["basename"] = os.path.basename(basename)
- if entry4["class"] == "File" and "secondaryFiles" in entry4:
+ if is_file(entry4) and "secondaryFiles" in entry4:
for sec_file in cast(
MutableSequence[CWLObjectType], entry4["secondaryFiles"]
):
@@ -806,7 +794,7 @@ class CommandLineTool(Process):
entry4,
)
- if entry4["class"] == "Directory" and "listing" in entry4:
+ if is_directory(entry4) and "listing" in entry4:
def remove_dirname(d: CWLObjectType) -> None:
if "dirname" in d:
@@ -881,9 +869,9 @@ class CommandLineTool(Process):
def calc_checksum(location: str) -> str | None:
for e in cachebuilder.files:
if (
- e["class"] == "File"
- and "location" in e
+ "location" in e
and e["location"] == location
+ and is_file(e)
and "checksum" in e
and e["checksum"] != "sha1$hash"
):
@@ -1046,7 +1034,7 @@ class CommandLineTool(Process):
)
j.stdin = stdin_eval
if j.stdin:
- reffiles.append({"class": "File", "path": j.stdin})
+ reffiles.append(CWLFileType(**{"class": "File", "path": j.stdin}))
if self.tool.get("stderr"):
with SourceLine(self.tool, "stderr", ValidationException, debug):
@@ -1326,7 +1314,7 @@ class CommandLineTool(Process):
fs_access: StdFsAccess,
compute_checksum: bool = True,
) -> CWLOutputType | None:
- r: list[CWLOutputType] = []
+ r: MutableSequence[CWLFileType | CWLDirectoryType] = []
empty_and_optional = False
debug = _logger.isEnabledFor(logging.DEBUG)
result: CWLOutputType | None = None
@@ -1374,9 +1362,9 @@ class CommandLineTool(Process):
key=cmp_to_key(locale.strcoll),
)
r.extend(
- cast(
- Iterable[CWLOutputType],
- [
+ [
+ cast(
+ CWLFileType | CWLDirectoryType,
{
"location": g,
"path": fs_access.join(
@@ -1387,16 +1375,16 @@ class CommandLineTool(Process):
"nameroot": os.path.splitext(decoded_basename)[0],
"nameext": os.path.splitext(decoded_basename)[1],
"class": "File" if fs_access.isfile(g) else "Directory",
- }
- for g, decoded_basename in zip(
+ },
+ )
+ for g, decoded_basename in zip(
+ sorted_glob_result,
+ map(
+ lambda x: os.path.basename(urllib.parse.unquote(x)),
sorted_glob_result,
- map(
- lambda x: os.path.basename(urllib.parse.unquote(x)),
- sorted_glob_result,
- ),
- )
- ],
- )
+ ),
+ )
+ ],
)
except OSError as e:
_logger.warning(str(e), exc_info=builder.debug)
@@ -1404,28 +1392,28 @@ class CommandLineTool(Process):
_logger.error("Unexpected error from fs_access", exc_info=True)
raise
- for files in cast(list[dict[str, Optional[CWLOutputType]]], r):
+ for files in r:
rfile = files.copy()
revmap(rfile)
- if files["class"] == "Directory":
+ if is_directory(files):
ll = binding.get("loadListing") or builder.loadListing
if ll and ll != "no_listing":
get_listing(fs_access, files, (ll == "deep_listing"))
else:
if binding.get("loadContents"):
- with fs_access.open(cast(str, rfile["location"]), "rb") as f:
+ with fs_access.open(rfile["location"], "rb") as f:
files["contents"] = str(
content_limit_respected_read_bytes(f), "utf-8"
)
if compute_checksum:
- with fs_access.open(cast(str, rfile["location"]), "rb") as f:
+ with fs_access.open(rfile["location"], "rb") as f:
checksum = hashlib.sha1() # nosec
contents = f.read(1024 * 1024)
while contents != b"":
checksum.update(contents)
contents = f.read(1024 * 1024)
files["checksum"] = "sha1$%s" % checksum.hexdigest()
- files["size"] = fs_access.size(cast(str, rfile["location"]))
+ files["size"] = fs_access.size(rfile["location"])
optional = False
single = False
=====================================
cwltool/cwlprov/provenance_profile.py
=====================================
@@ -6,9 +6,15 @@ import uuid
from collections.abc import MutableMapping, MutableSequence, Sequence
from io import BytesIO
from pathlib import PurePath, PurePosixPath
-from typing import TYPE_CHECKING, Any, cast
-
-from cwl_utils.types import CWLObjectType
+from typing import TYPE_CHECKING, Any, TypedDict, cast
+
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ is_directory,
+ is_file,
+)
from prov.identifier import Identifier, QualifiedName
from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity
from schema_salad.sourceline import SourceLine
@@ -41,6 +47,16 @@ from .writablebagfile import create_job, write_bag_file # change this later
if TYPE_CHECKING:
from .ro import ResearchObject
+CWLArtifact = TypedDict("CWLArtifact", {"@id": str})
+
+
+class _CWLDirectoryArtifact(CWLArtifact, CWLDirectoryType):
+ pass
+
+
+class _CWLFileArtifact(CWLArtifact, CWLFileType):
+ pass
+
def copy_job_order(job: Process | JobsType, job_order_object: CWLObjectType) -> CWLObjectType:
"""Create copy of job object for provenance."""
@@ -244,15 +260,15 @@ class ProvenanceProfile:
self.generate_output_prov(outputs, process_run_id, process_name)
self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when)
- def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]:
+ def declare_file(self, value: _CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, str]:
"""Construct a FileEntity for the given CWL File object."""
- if value["class"] != "File":
+ if not is_file(value):
raise ValueError("Must have class:File: %s" % value)
# Need to determine file hash aka RO filename
entity: ProvEntity | None = None
checksum = None
if "checksum" in value:
- csum = cast(str, value["checksum"])
+ csum = value["checksum"]
method, checksum = csum.split("$", 1)
if method == SHA1 and self.research_object.has_data_file(checksum):
entity = self.document.entity("data:" + checksum)
@@ -270,7 +286,7 @@ class ProvenanceProfile:
if not entity and "contents" in value:
# Anonymous file, add content as string
- entity, checksum = self.declare_string(cast(str, value["contents"]))
+ entity, checksum = self.declare_string(value["contents"])
# By here one of them should have worked!
if not entity or not checksum:
@@ -280,7 +296,7 @@ class ProvenanceProfile:
# secondaryFiles. Note that multiple uses of a file might thus record
# different names for the same entity, so we'll
# make/track a specialized entity by UUID
- file_id = cast(str, value.setdefault("@id", uuid.uuid4().urn))
+ file_id = value.setdefault("@id", uuid.uuid4().urn)
# A specialized entity that has just these names
file_entity = self.document.entity(
file_id,
@@ -288,20 +304,20 @@ class ProvenanceProfile:
)
if "basename" in value:
- file_entity.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])})
+ file_entity.add_attributes({CWLPROV["basename"]: value["basename"]})
if "nameroot" in value:
- file_entity.add_attributes({CWLPROV["nameroot"]: cast(str, value["nameroot"])})
+ file_entity.add_attributes({CWLPROV["nameroot"]: value["nameroot"]})
if "nameext" in value:
- file_entity.add_attributes({CWLPROV["nameext"]: cast(str, value["nameext"])})
+ file_entity.add_attributes({CWLPROV["nameext"]: value["nameext"]})
self.document.specializationOf(file_entity, entity)
# Check for secondaries
- for sec in cast(MutableSequence[CWLObjectType], value.get("secondaryFiles", [])):
+ for sec in value.get("secondaryFiles", []):
# TODO: Record these in a specializationOf entity with UUID?
- if sec["class"] == "File":
- sec_entity, _, _ = self.declare_file(sec)
- elif sec["class"] == "Directory":
- sec_entity = self.declare_directory(sec)
+ if is_file(sec):
+ sec_entity, _, _ = self.declare_file(cast(_CWLFileArtifact, sec))
+ elif is_directory(sec):
+ sec_entity = self.declare_directory(cast(_CWLDirectoryArtifact, sec))
else:
raise ValueError(f"Got unexpected secondaryFiles value: {sec}")
# We don't know how/when/where the secondary file was generated,
@@ -316,14 +332,14 @@ class ProvenanceProfile:
return file_entity, entity, checksum
- def declare_directory(self, value: CWLObjectType) -> ProvEntity:
+ def declare_directory(self, value: _CWLDirectoryArtifact) -> ProvEntity:
"""Register any nested files/directories."""
# FIXME: Calculate a hash-like identifier for directory
# so we get same value if it's the same filenames/hashes
# in a different location.
# For now, mint a new UUID to identify this directory, but
# attempt to keep it inside the value dictionary
- dir_id = cast(str, value.setdefault("@id", uuid.uuid4().urn))
+ dir_id = value.setdefault("@id", uuid.uuid4().urn)
# New annotation file to keep the ORE Folder listing
ore_doc_fn = dir_id.replace("urn:uuid:", "directory-") + ".ttl"
@@ -340,7 +356,7 @@ class ProvenanceProfile:
)
if "basename" in value:
- coll.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])})
+ coll.add_attributes({CWLPROV["basename"]: value["basename"]})
# ORE description of ro:Folder, saved separately
coll_b = dir_bundle.entity(
@@ -476,11 +492,11 @@ class ProvenanceProfile:
# Base case - we found a File we need to update
case {"class": "File"}:
- entity = self.declare_file(value)[0]
+ entity = self.declare_file(cast(_CWLFileArtifact, value))[0]
value["@id"] = entity.identifier.uri
return entity
case {"class": "Directory"}:
- entity = self.declare_directory(value)
+ entity = self.declare_directory(cast(_CWLDirectoryArtifact, value))
value["@id"] = entity.identifier.uri
return entity
case {**rest}:
=====================================
cwltool/cwlprov/ro.py
=====================================
@@ -13,7 +13,15 @@ from socket import getfqdn
from typing import IO, TYPE_CHECKING, Any, Optional, cast
import prov.model as provM
-from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ CWLOutputType,
+ is_directory,
+ is_file,
+ is_file_or_directory,
+)
from prov.model import ProvDocument
from ..loghandler import _logger
@@ -489,7 +497,7 @@ class ResearchObject:
return authored_by
return None
- def generate_snapshot(self, prov_dep: CWLFileType) -> None:
+ def generate_snapshot(self, prov_dep: CWLFileType | CWLDirectoryType) -> None:
"""Copy all of the CWL files to the snapshot/ directory."""
self.self_check()
for key, value in prov_dep.items():
@@ -516,7 +524,7 @@ class ResearchObject:
pass # FIXME: avoids duplicate snapshotting; need better solution
elif key in ("secondaryFiles", "listing"):
for files in cast(MutableSequence[CWLFileType | CWLDirectoryType], value):
- if files["class"] == "File":
+ if is_file_or_directory(files):
self.generate_snapshot(files)
else:
pass
@@ -629,17 +637,17 @@ class ResearchObject:
def _relativise_files(
self,
- structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType],
+ structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType] | None,
) -> None:
"""Save any file objects into the RO and update the local paths."""
# Base case - we found a File we need to update
_logger.debug("[provenance] Relativising: %s", structure)
if isinstance(structure, MutableMapping):
- if structure.get("class") == "File":
+ if is_file(structure):
relative_path: str | PurePosixPath | None = None
if "checksum" in structure:
- raw_checksum = cast(str, structure["checksum"])
+ raw_checksum = structure["checksum"]
alg, checksum = raw_checksum.split("$")
if alg != SHA1:
raise TypeError(
@@ -653,7 +661,7 @@ class ResearchObject:
# Register in RO; but why was this not picked
# up by used_artefacts?
_logger.info("[provenance] Adding to RO %s", structure["location"])
- with self.fsaccess.open(cast(str, structure["location"]), "rb") as fp:
+ with self.fsaccess.open(structure["location"], "rb") as fp:
relative_path = self.add_data_file(fp)
checksum = PurePosixPath(relative_path).name
structure["checksum"] = f"{SHA1}${checksum}"
@@ -662,16 +670,14 @@ class ResearchObject:
if "path" in structure:
del structure["path"]
- if structure.get("class") == "Directory":
+ if is_directory(structure):
# TODO: Generate anonymous Directory with a "listing"
# pointing to the hashed files
del structure["location"]
for val in structure.values():
try:
- self._relativise_files(
- cast(CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], val)
- )
+ self._relativise_files(cast(CWLOutputType, val))
except OSError:
pass
return
@@ -679,6 +685,4 @@ class ResearchObject:
if isinstance(structure, MutableSequence):
for obj in structure:
# Recurse and rewrite any nested File objects
- self._relativise_files(
- cast(CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], obj)
- )
+ self._relativise_files(obj)
=====================================
cwltool/job.py
=====================================
@@ -138,11 +138,13 @@ class JobBase(HasReqsHints, metaclass=ABCMeta):
self.tmpdir = ""
self.environment: MutableMapping[str, str] = {}
- self.generatefiles: CWLDirectoryType = {
- "class": "Directory",
- "listing": [],
- "basename": "",
- }
+ self.generatefiles = CWLDirectoryType(
+ **{
+ "class": "Directory",
+ "listing": [],
+ "basename": "",
+ }
+ )
self.stagedir: str | None = None
self.inplace_update = False
self.prov_obj: ProvenanceProfile | None = None
=====================================
cwltool/load_tool.py
=====================================
@@ -13,6 +13,7 @@ from typing import Any, Union, cast
from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
from cwl_utils.types import CWLObjectType
+from mypy_extensions import i32, i64
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad.exceptions import ValidationException
from schema_salad.fetcher import Fetcher
@@ -255,10 +256,12 @@ def _fast_parser_convert_stdstreams_to_files(
cwl_v1_2_utils.convert_stdstreams_to_files(processobj)
case cwl_v1_2.Workflow(steps=steps):
for st in steps:
- _fast_parser_convert_stdstreams_to_files(st.run)
+ if not isinstance(st.run, str):
+ _fast_parser_convert_stdstreams_to_files(st.run)
case MutableSequence():
for p in processobj:
- _fast_parser_convert_stdstreams_to_files(p)
+ if not isinstance(p, str):
+ _fast_parser_convert_stdstreams_to_files(p)
def _fast_parser_expand_hint_class(
@@ -282,7 +285,8 @@ def _fast_parser_handle_hints(
case cwl_v1_2.Workflow(steps=steps):
for st in steps:
_fast_parser_expand_hint_class(st.hints, loadingOptions)
- _fast_parser_handle_hints(st.run, loadingOptions)
+ if not isinstance(st.run, str):
+ _fast_parser_handle_hints(st.run, loadingOptions)
case MutableSequence():
for p in processobj:
_fast_parser_handle_hints(p, loadingOptions)
@@ -315,7 +319,7 @@ def fast_parser(
_fast_parser_convert_stdstreams_to_files(objects)
_fast_parser_handle_hints(objects, loadopt)
- processobj: MutableMapping[str, Any] | MutableSequence[Any] | float | str | None
+ processobj: MutableMapping[str, Any] | MutableSequence[Any] | i32 | i64 | float | str | None
processobj = cwl_v1_2.save(objects, relative_uris=False)
=====================================
cwltool/main.py
=====================================
@@ -23,7 +23,13 @@ import argcomplete
import coloredlogs
import requests
import ruamel.yaml
-from cwl_utils.types import CWLFileType, CWLObjectType, CWLOutputType
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ CWLOutputType,
+ is_file,
+)
from rich_argparse import RichHelpFormatter
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from ruamel.yaml.main import YAML
@@ -297,9 +303,7 @@ def realize_input_schema(
if isinstance(entry["type"], Mapping):
entry["type"] = cast(
CWLOutputType,
- realize_input_schema(
- cast(MutableSequence[str | CWLObjectType], [entry["type"]]), schema_defs
- ),
+ realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs),
)
if entry["type"] == "array":
items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]]
@@ -569,13 +573,11 @@ def prov_deps(
) -> CWLFileType:
deps = find_deps(obj, document_loader, uri, basedir=basedir)
- def remove_non_cwl(deps: CWLFileType) -> None:
- if "secondaryFiles" in deps:
+ def remove_non_cwl(deps: CWLFileType | CWLDirectoryType) -> None:
+ if is_file(deps) and "secondaryFiles" in deps:
sec_files = deps["secondaryFiles"]
for index, entry in enumerate(sec_files):
- if not (
- entry["class"] == "File" and "format" in entry and entry["format"] == CWL_IANA
- ):
+ if not (is_file(entry) and "format" in entry and entry["format"] == CWL_IANA):
del sec_files[index]
else:
remove_non_cwl(entry)
@@ -592,7 +594,13 @@ def find_deps(
nestdirs: bool = True,
) -> CWLFileType:
"""Find the dependencies of the CWL document."""
- deps = CWLFileType(**{"class": "File", "location": uri, "format": CWL_IANA})
+ deps = CWLFileType(
+ **{
+ "class": "File",
+ "location": uri,
+ "format": CWL_IANA,
+ }
+ )
def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None:
return document_loader.fetch(document_loader.fetcher.urljoin(base, uri))
=====================================
cwltool/pack.py
=====================================
@@ -79,7 +79,7 @@ def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> No
def import_embed(
- d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType,
+ d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType | None,
seen: set[str],
) -> None:
if isinstance(d, MutableSequence):
=====================================
cwltool/pathmapper.py
=====================================
@@ -6,7 +6,7 @@ import uuid
from collections.abc import ItemsView, Iterable, Iterator, KeysView, MutableSequence
from typing import NamedTuple, Optional, cast
-from cwl_utils.types import CWLDirectoryType, CWLFileType
+from cwl_utils.types import CWLDirectoryType, CWLFileType, is_directory
from mypy_extensions import mypyc_attr
from schema_salad.exceptions import ValidationException
from schema_salad.ref_resolver import uri_file_path
@@ -89,7 +89,7 @@ class PathMapper:
def visitlisting(
self,
- listing: list[CWLFileType | CWLDirectoryType],
+ listing: MutableSequence[CWLFileType | CWLDirectoryType],
stagedir: str,
basedir: str,
copy: bool = False,
@@ -119,7 +119,7 @@ class PathMapper:
stagedir,
obj["basename"],
)
- if obj["class"] == "Directory":
+ if is_directory(obj):
location = obj["location"]
if location.startswith("file://"):
resolved = uri_file_path(location)
@@ -131,13 +131,13 @@ class PathMapper:
if location.startswith("file://"):
staged = False
self.visitlisting(
- cast(list[CWLFileType | CWLDirectoryType], obj.get("listing", [])),
+ obj.get("listing", []),
tgt,
basedir,
copy=copy,
staged=staged,
)
- elif obj["class"] == "File":
+ else:
path = obj["location"]
ab = abspath(path, basedir)
if "contents" in obj and path.startswith("_:"):
@@ -173,14 +173,16 @@ class PathMapper:
deref, tgt, "WritableFile" if copy else "File", staged
)
self.visitlisting(
- cast(list[CWLFileType | CWLDirectoryType], obj.get("secondaryFiles", [])),
+ obj.get("secondaryFiles", []),
stagedir,
basedir,
copy=copy,
staged=staged,
)
- def setup(self, referenced_files: list[CWLFileType | CWLDirectoryType], basedir: str) -> None:
+ def setup(
+ self, referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str
+ ) -> None:
"""
For each file, set the target to its own directory.
=====================================
cwltool/process.py
=====================================
@@ -20,6 +20,7 @@ from collections.abc import (
Mapping,
MutableMapping,
MutableSequence,
+ Sequence,
Sized,
)
from importlib.resources import files
@@ -27,7 +28,15 @@ from os import scandir
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from cwl_utils import expression
-from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ CWLOutputType,
+ is_directory,
+ is_file,
+ is_file_or_directory,
+)
from mypy_extensions import mypyc_attr
from rdflib import Graph
from ruamel.yaml.comments import CommentedMap, CommentedSeq
@@ -308,27 +317,14 @@ def relocateOutputs(
return outputObj
def _collectDirEntries(
- obj: (
- CWLObjectType
- | CWLFileType
- | CWLDirectoryType
- | MutableSequence[CWLFileType | CWLDirectoryType | CWLObjectType]
- ),
+ obj: CWLObjectType | MutableSequence[CWLObjectType] | CWLOutputType | None,
) -> Iterator[CWLFileType | CWLDirectoryType]:
if isinstance(obj, Mapping):
- if "class" in obj and obj["class"] in ("File", "Directory"):
- yield cast(CWLFileType | CWLDirectoryType, obj)
+ if is_file_or_directory(obj):
+ yield obj
else:
for sub_obj in obj.values():
- yield from _collectDirEntries(
- cast(
- CWLObjectType
- | CWLFileType
- | CWLDirectoryType
- | MutableSequence[CWLFileType | CWLDirectoryType | CWLObjectType],
- sub_obj,
- )
- )
+ yield from _collectDirEntries(sub_obj)
elif isinstance(obj, MutableSequence):
for sub_obj in obj:
yield from _collectDirEntries(sub_obj)
@@ -823,7 +819,7 @@ hints:
except (ValidationException, WorkflowException) as err:
raise WorkflowException("Invalid job input record:\n" + str(err)) from err
- files: list[CWLFileType | CWLDirectoryType] = []
+ files: MutableSequence[CWLFileType | CWLDirectoryType] = []
bindings = CommentedSeq()
outdir = ""
tmpdir = ""
@@ -1140,7 +1136,7 @@ def uniquename(stem: str, names: set[str] | None = None) -> str:
def nestdir(base: str, deps: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType:
- """Insert Directory objects from the target location up to the base."""
+ """Add intermediate directory objects to preserve the relative layout."""
dirname = os.path.dirname(base) + "/"
subid = deps["location"]
if subid.startswith(dirname):
@@ -1175,12 +1171,12 @@ def mergedirs(
"Conflicting basename in listing or secondaryFiles, '%s' used by both '%s' and '%s'"
% (basename, e["location"], ents[basename]["location"])
)
- elif e["class"] == "Directory" and "listing" in e:
+ elif is_directory(e) and "listing" in e:
# name already in entries
# merge it into the existing listing
cast(CWLDirectoryType, ents[basename]).setdefault("listing", []).extend(e["listing"])
for e in ents.values():
- if e["class"] == "Directory" and "listing" in e:
+ if is_directory(e) and "listing" in e:
e["listing"] = mergedirs(e["listing"])
r.extend(ents.values())
return r
@@ -1192,9 +1188,10 @@ CWL_IANA = "https://www.iana.org/assignments/media-types/application/cwl"
def scandeps(
base: str,
doc: (
- CWLObjectType
- | MutableSequence[CWLObjectType]
- | MutableSequence[CWLFileType | CWLDirectoryType]
+ CWLFileType
+ | CWLDirectoryType
+ | CWLObjectType
+ | Sequence[CWLFileType | CWLDirectoryType | CWLObjectType]
),
reffields: set[str],
urlfields: set[str],
@@ -1227,63 +1224,68 @@ def scandeps(
if cast(str, doc["id"]).startswith("file://"):
df, _ = urllib.parse.urldefrag(cast(str, doc["id"]))
if base != df:
- r.append({"class": "File", "location": df, "format": CWL_IANA})
+ r.append(CWLFileType(**{"class": "File", "location": df, "format": CWL_IANA}))
base = df
- if doc.get("class") in ("File", "Directory") and "location" in urlfields:
- doc_fd = cast(CWLFileType | CWLDirectoryType, doc)
- u = doc_fd.get("location", doc_fd.get("path"))
+ if is_file_or_directory(doc) and "location" in urlfields:
+ u = doc.get("location", doc.get("path"))
if u and not u.startswith("_:"):
- deps = cast(
- CWLFileType | CWLDirectoryType,
- {
- "class": doc_fd["class"],
- "location": urljoin(base, u),
- },
- )
- if "basename" in doc_fd:
- deps["basename"] = doc_fd["basename"]
- if doc_fd["class"] == "Directory" and "listing" in doc_fd:
- cast(CWLDirectoryType, deps)["listing"] = doc_fd["listing"]
- if doc_fd["class"] == "File" and "secondaryFiles" in doc_fd:
- cast(CWLFileType, deps)["secondaryFiles"] = scandeps(
- base,
- doc_fd["secondaryFiles"],
- reffields,
- urlfields,
- loadref,
- urljoin=urljoin,
- nestdirs=nestdirs,
+ deps: CWLFileType | CWLDirectoryType
+ if is_file(doc):
+ deps = CWLFileType(
+ **{
+ "class": "File",
+ }
+ )
+ if "secondaryFiles" in doc:
+ deps["secondaryFiles"] = scandeps(
+ base,
+ doc["secondaryFiles"],
+ reffields,
+ urlfields,
+ loadref,
+ urljoin=urljoin,
+ nestdirs=nestdirs,
+ )
+ else:
+ deps = CWLDirectoryType(
+ **{
+ "class": "Directory",
+ }
)
+ if "listing" in doc:
+ deps["listing"] = doc["listing"]
+ deps["location"] = urljoin(base, u)
+ if "basename" in doc:
+ deps["basename"] = doc["basename"]
if nestdirs:
deps = nestdir(base, deps)
r.append(deps)
else:
- match doc_fd:
- case {"class": "Directory", "listing": listing}:
- r.extend(
- scandeps(
- base,
- cast(MutableSequence[CWLObjectType], listing),
- reffields,
- urlfields,
- loadref,
- urljoin=urljoin,
- nestdirs=nestdirs,
- )
+ if is_directory(doc) and "listing" in doc:
+ r.extend(
+ scandeps(
+ base,
+ doc["listing"],
+ reffields,
+ urlfields,
+ loadref,
+ urljoin=urljoin,
+ nestdirs=nestdirs,
)
- case {"class": "File", "secondaryFiles": sec_files}:
- r.extend(
- scandeps(
- base,
- cast(MutableSequence[CWLObjectType], sec_files),
- reffields,
- urlfields,
- loadref,
- urljoin=urljoin,
- nestdirs=nestdirs,
- )
+ )
+ elif is_file(doc) and "secondaryFiles" in doc:
+ r.extend(
+ scandeps(
+ base,
+ doc["secondaryFiles"],
+ reffields,
+ urlfields,
+ loadref,
+ urljoin=urljoin,
+ nestdirs=nestdirs,
)
+ )
for k, v in doc.items():
if k in reffields:
@@ -1310,11 +1312,13 @@ def scandeps(
Union[MutableSequence[CWLObjectType], CWLObjectType],
loadref(base, u2),
)
- deps2: CWLFileType = {
- "class": "File",
- "location": subid,
- "format": CWL_IANA,
- }
+ deps2 = CWLFileType(
+ **{
+ "class": "File",
+ "location": subid,
+ "format": CWL_IANA,
+ }
+ )
sf = scandeps(
subid,
sub,
@@ -1332,11 +1336,11 @@ def scandeps(
r.append(deps2)
elif k in urlfields and k != "location":
for u3 in aslist(v):
- deps = {"class": "File", "location": urljoin(base, u3)}
+ deps = CWLFileType(**{"class": "File", "location": urljoin(base, u3)})
if nestdirs:
deps = nestdir(base, deps)
r.append(deps)
- elif doc.get("class") in ("File", "Directory") and k in (
+ elif is_file_or_directory(doc) and k in (
"listing",
"secondaryFiles",
):
@@ -1359,12 +1363,7 @@ def scandeps(
r.extend(
scandeps(
base,
- cast(
- CWLObjectType
- | MutableSequence[CWLObjectType]
- | MutableSequence[CWLFileType | CWLDirectoryType],
- d,
- ),
+ d,
reffields,
urlfields,
loadref,
=====================================
cwltool/update.py
=====================================
@@ -125,7 +125,7 @@ def v1_0to1_1(
rewrite_requirements(s)
def update_secondaryFiles(
- t: CWLOutputType, top: bool = False
+ t: CWLOutputType | None, top: bool = False
) -> MutableSequence[MutableMapping[str, str]] | MutableMapping[str, str]:
if isinstance(t, CommentedSeq):
new_seq = copy.deepcopy(t)
=====================================
cwltool/utils.py
=====================================
@@ -2,6 +2,15 @@
import collections
+from cwl_utils.types import (
+ CWLDirectoryType,
+ CWLFileType,
+ CWLObjectType,
+ CWLOutputType,
+ is_directory,
+ is_file,
+)
+
try:
import fcntl
except ImportError:
@@ -25,6 +34,7 @@ from collections.abc import (
Iterable,
MutableMapping,
MutableSequence,
+ Sequence,
)
from datetime import datetime
from email.utils import parsedate_to_datetime
@@ -43,13 +53,11 @@ from typing import (
TypeAlias,
TypedDict,
Union,
- cast,
)
import requests
from cachecontrol import CacheControl
from cachecontrol.caches import FileCache
-from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType
from mypy_extensions import mypyc_attr
from schema_salad.exceptions import ValidationException
@@ -85,7 +93,6 @@ ResolverType: TypeAlias = Callable[["Loader", str], Optional[str]]
DestinationsType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]]
ScatterDestinationsType: TypeAlias = MutableMapping[str, list[Optional[CWLOutputType]]]
ScatterOutputCallbackType: TypeAlias = Callable[[Optional[ScatterDestinationsType], str], None]
-SinkType: TypeAlias = Union[CWLOutputType, CWLObjectType]
JSONType: TypeAlias = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None]
@@ -257,16 +264,16 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No
def dedup(
listing: MutableSequence[CWLFileType | CWLDirectoryType],
-) -> list[CWLFileType | CWLDirectoryType]:
- """Remove duplicated items iin a list of CWL File/Directory objects."""
+) -> MutableSequence[CWLFileType | CWLDirectoryType]:
+ """Remove duplicate entries from a CWL Directory 'listing'."""
marksub = set()
def mark(d: dict[str, str]) -> None:
marksub.add(d["location"])
for entry in listing:
- if entry["class"] == "Directory":
- for e in cast(list[CWLObjectType], entry.get("listing", [])):
+ if is_directory(entry):
+ for e in entry.get("listing", []):
adjustFileObjs(e, mark)
adjustDirObjs(e, mark)
@@ -284,20 +291,17 @@ def get_listing(
fs_access: "StdFsAccess", rec: CWLObjectType | CWLDirectoryType, recursive: bool = True
) -> None:
"""Expand, recursively, any 'listing' fields in a Directory."""
- if rec.get("class") != "Directory":
- finddirs: list[CWLObjectType] = []
+ if not is_directory(rec):
+ finddirs: list[CWLDirectoryType] = []
visit_class(rec, ("Directory",), finddirs.append)
for f in finddirs:
- _get_listing(fs_access, cast(CWLDirectoryType, f), recursive)
+ get_listing(fs_access, f, recursive=recursive)
return
- _get_listing(fs_access, cast(CWLDirectoryType, rec), recursive)
-
-
-def _get_listing(fs_access: "StdFsAccess", rec: CWLDirectoryType, recursive: bool = True) -> None:
if "listing" in rec:
return
listing: MutableSequence[CWLFileType | CWLDirectoryType] = []
- for ld in fs_access.listdir(rec["location"]):
+ loc = rec["location"]
+ for ld in fs_access.listdir(loc):
parse = urllib.parse.urlparse(ld)
bn = os.path.basename(urllib.request.url2pathname(parse.path))
if fs_access.isdir(ld):
@@ -309,7 +313,7 @@ def _get_listing(fs_access: "StdFsAccess", rec: CWLDirectoryType, recursive: boo
}
)
if recursive:
- _get_listing(fs_access, ent, recursive)
+ get_listing(fs_access, ent, recursive)
listing.append(ent)
else:
listing.append(CWLFileType(**{"class": "File", "location": ld, "basename": bn}))
@@ -412,21 +416,24 @@ def ensure_non_writable(path: str) -> None:
def normalizeFilesDirs(
- job: (
- None
- | MutableSequence[CWLFileType | CWLDirectoryType]
- | CWLFileType
- | CWLDirectoryType
- | CWLObjectType
- ),
+ job: Sequence[CWLObjectType | CWLOutputType | None] | CWLObjectType | CWLOutputType | None,
) -> None:
- def addLocation(d: dict[str, Any]) -> None:
+ """
+ Add missing `location`s and `basename`s to CWL File and Directory objects.
+
+ :raises ValidationException: if anonymous objects are missing required fields,
+ or if the location ends in '/' but the object isn't
+ a directory
+
+ """
+
+ def addLocation(d: CWLFileType | CWLDirectoryType) -> None:
if "location" not in d:
- if d["class"] == "File" and ("contents" not in d):
+ if is_file(d) and ("contents" not in d):
raise ValidationException(
"Anonymous file object must have 'contents' and 'basename' fields."
)
- if d["class"] == "Directory" and ("listing" not in d or "basename" not in d):
+ if is_directory(d) and ("listing" not in d or "basename" not in d):
raise ValidationException(
"Anonymous directory object must have 'listing' and 'basename' fields."
)
@@ -438,7 +445,7 @@ def normalizeFilesDirs(
path = parse.path
# strip trailing slash
if path.endswith("/"):
- if d["class"] != "Directory":
+ if not is_directory(d):
raise ValidationException(
"location '%s' ends with '/' but is not a Directory" % d["location"]
)
@@ -460,7 +467,7 @@ def normalizeFilesDirs(
else:
d["basename"] = str(os.path.basename(urllib.request.url2pathname(path)))
- if d["class"] == "File":
+ if is_file(d):
nr, ne = os.path.splitext(d["basename"])
if d.get("nameroot") != nr:
d["nameroot"] = str(nr)
=====================================
cwltool/workflow_job.py
=====================================
@@ -7,7 +7,7 @@ from collections.abc import MutableMapping, MutableSequence, Sized
from typing import TYPE_CHECKING, Optional, Union, cast
from cwl_utils import expression
-from cwl_utils.types import CWLObjectType, CWLOutputType
+from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType
from schema_salad.sourceline import SourceLine
from schema_salad.utils import json_dumps
@@ -24,7 +24,6 @@ from .utils import (
ParametersType,
ScatterDestinationsType,
ScatterOutputCallbackType,
- SinkType,
WorkflowStateItem,
adjustDirObjs,
aslist,
@@ -403,7 +402,7 @@ def object_from_state(
a_state,
iid,
inputobj,
- linkMerge=cast(
+ cast(
Optional[str],
inp.get(
"linkMerge",
=====================================
mypy-requirements.txt
=====================================
@@ -1,4 +1,4 @@
-mypy==1.20.0 # also update pyproject.toml
+mypy==1.20.1 # also update pyproject.toml
ruamel.yaml>=0.16.0,<0.20
cwl-utils>=0.41
cwltest
=====================================
pyproject.toml
=====================================
@@ -2,7 +2,7 @@
requires = [
"setuptools>=45",
"setuptools_scm[simple]>=8.0.4,<11",
- "mypy==1.20.0", # also update mypy-requirements.txt
+ "mypy==1.20.1", # also update mypy-requirements.txt
"types-requests",
"types-psutil>=7.1.3.20251210",
"ruamel.yaml>=0.16.0,<0.20",
=====================================
tests/test_examples.py
=====================================
@@ -620,23 +620,27 @@ def test_issue_1765_print_deps_with_workflows_having_namespace_location_steps()
def test_dedupe() -> None:
not_deduped: MutableSequence[CWLFileType | CWLDirectoryType] = [
- {"class": "File", "location": "file:///example/a"},
- {"class": "File", "location": "file:///example/a"},
- {"class": "File", "location": "file:///example/d"},
- {
- "class": "Directory",
- "location": "file:///example/c",
- "listing": [{"class": "File", "location": "file:///example/d"}],
- },
+ CWLFileType(**{"class": "File", "location": "file:///example/a"}),
+ CWLFileType(**{"class": "File", "location": "file:///example/a"}),
+ CWLFileType(**{"class": "File", "location": "file:///example/d"}),
+ CWLDirectoryType(
+ **{
+ "class": "Directory",
+ "location": "file:///example/c",
+ "listing": [{"class": "File", "location": "file:///example/d"}],
+ }
+ ),
]
expected = [
- {"class": "File", "location": "file:///example/a"},
- {
- "class": "Directory",
- "location": "file:///example/c",
- "listing": [{"class": "File", "location": "file:///example/d"}],
- },
+ CWLFileType(**{"class": "File", "location": "file:///example/a"}),
+ CWLDirectoryType(
+ **{
+ "class": "Directory",
+ "location": "file:///example/c",
+ "listing": [{"class": "File", "location": "file:///example/d"}],
+ }
+ ),
]
assert dedup(not_deduped) == expected
=====================================
tests/test_http_input.py
=====================================
@@ -14,12 +14,14 @@ def test_http_path_mapping(tmp_path: Path) -> None:
"https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta"
)
base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [
- {
- "class": "File",
- "location": "https://raw.githubusercontent.com/common-workflow-language/"
- "cwltool/main/tests/2.fasta",
- "basename": "chr20.fa",
- }
+ CWLFileType(
+ **{
+ "class": "File",
+ "location": "https://raw.githubusercontent.com/common-workflow-language/"
+ "cwltool/main/tests/2.fasta",
+ "basename": "chr20.fa",
+ }
+ )
]
pathmap = PathMapper(base_file, os.getcwd(), str(tmp_path))._pathmap
@@ -56,11 +58,13 @@ def test_modification_date(tmp_path: Path) -> None:
location = httpserver.url_for(f"/{remote_file_name}")
base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [
- {
- "class": "File",
- "location": location,
- "basename": remote_file_name,
- }
+ CWLFileType(
+ **{
+ "class": "File",
+ "location": location,
+ "basename": remote_file_name,
+ }
+ )
]
date_now = datetime.now()
=====================================
tests/test_js_sandbox.py
=====================================
@@ -9,6 +9,7 @@ from typing import Any
import pytest
from cwl_utils import sandboxjs
+from cwl_utils.types import CWLFileType
from cwltool.factory import Factory
from cwltool.loghandler import _logger, configure_logging
@@ -41,7 +42,7 @@ def test_value_from_two_concatenated_expressions() -> None:
js_engine.localdata = threading.local() # type: ignore[attr-defined]
factory = Factory()
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
- file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
+ file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")})
assert echo(file1=file) == {"out": "a string\n"}
@@ -87,7 +88,7 @@ def test_value_from_two_concatenated_expressions_podman(
with monkeypatch.context() as m:
m.setenv("PATH", new_paths)
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
- file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
+ file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")})
assert echo(file1=file) == {"out": "a string\n"}
@@ -112,7 +113,7 @@ def test_value_from_two_concatenated_expressions_singularity(
m.setenv("CWL_SINGULARITY_CACHE", str(singularity_cache))
m.setenv("PATH", new_paths)
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
- file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
+ file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")})
assert echo(file1=file) == {"out": "a string\n"}
=====================================
tests/test_parallel.py
=====================================
@@ -1,6 +1,8 @@
import json
from pathlib import Path
+from cwl_utils.types import CWLFileType
+
from cwltool.context import RuntimeContext
from cwltool.executors import MultithreadedJobExecutor
from cwltool.factory import Factory
@@ -17,7 +19,7 @@ def test_sequential_workflow(tmp_path: Path) -> None:
runtime_context.select_resources = executor.select_resources
factory = Factory(executor, None, runtime_context)
echo = factory.make(get_data(test_file))
- file_contents = {"class": "File", "location": get_data("tests/wf/whale.txt")}
+ file_contents = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")})
assert echo(file1=file_contents) == {"count_output": 16}
=====================================
tests/test_secrets.py
=====================================
@@ -2,9 +2,10 @@ import shutil
import tempfile
from collections.abc import Callable, MutableMapping
from io import StringIO
+from typing import cast
import pytest
-from cwl_utils.types import CWLObjectType
+from cwl_utils.types import CWLObjectType, CWLOutputType
from cwltool.main import main
from cwltool.secrets import SecretStore
@@ -27,8 +28,8 @@ def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None:
storage, obscured = secrets
assert obscured["foo"] != "bar"
assert obscured["baz"] == "quux"
- result = storage.retrieve(obscured)
- assert isinstance(result, MutableMapping) and result["foo"] == "bar"
+ result = cast(MutableMapping[str, CWLOutputType], storage.retrieve(obscured))
+ assert isinstance(result, dict) and result["foo"] == "bar"
obscured_factories_expected = [
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/40a00967ffa17b6c65f2f1c532c7a2059d00f403
--
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/40a00967ffa17b6c65f2f1c532c7a2059d00f403
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20260416/d2224c3f/attachment-0001.htm>
More information about the debian-med-commit
mailing list