[med-svn] [Git][med-team/cwltool][master] 4 commits: routine-update: New upstream version
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Fri Jun 21 15:18:14 BST 2024
Michael R. Crusoe pushed to branch master at Debian Med / cwltool
Commits:
96dd910b by Michael R. Crusoe at 2024-06-18T18:24:29+02:00
routine-update: New upstream version
- - - - -
80d49674 by Michael R. Crusoe at 2024-06-18T18:24:30+02:00
New upstream version 3.1.20240508115724
- - - - -
1634f594 by Michael R. Crusoe at 2024-06-18T18:24:37+02:00
Update upstream source from tag 'upstream/3.1.20240508115724'
Update to upstream version '3.1.20240508115724'
with Debian dir c9d46bc0bf63fedf88ca7b953fa202de106cb41c
- - - - -
1be37337 by Michael R. Crusoe at 2024-06-18T18:24:58+02:00
routine-update: Ready to upload to unstable
- - - - -
18 changed files:
- .github/workflows/ci-tests.yml
- PKG-INFO
- conformance-test.sh
- cwltool.egg-info/PKG-INFO
- cwltool/_version.py
- cwltool/builder.py
- cwltool/command_line_tool.py
- cwltool/cwlrdf.py
- cwltool/docker.py
- cwltool/utils.py
- debian/changelog
- mypy-requirements.txt
- pyproject.toml
- release-test.sh
- setup.py
- test-requirements.txt
- tests/test_path_checks.py
- tests/test_tmpdir.py
Changes:
=====================================
.github/workflows/ci-tests.yml
=====================================
@@ -204,7 +204,7 @@ jobs:
env:
VERSION: ${{ matrix.cwl-version }}
CONTAINER: ${{ matrix.container }}
- GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }}
+ GIT_TARGET: main
CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
run: ./conformance-test.sh
- name: Archive test results
@@ -266,7 +266,7 @@ jobs:
macos:
name: Test on macos-latest
- runs-on: macos-latest
+ runs-on: macos-13 # not latest, that is now an Apple Silicon M1, for which seqtk is not yet built on bioconda
env:
TOXENV: py312-unit
steps:
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20240404144621
+Version: 3.1.20240508115724
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Download-URL: https://github.com/common-workflow-language/cwltool
=====================================
conformance-test.sh
=====================================
@@ -13,15 +13,15 @@ venv() {
}
# Set these variables when running the script, e.g.:
-# VERSION=v1.2 GIT_TARGET=1.2.1_proposed CONTAINER=podman ./conformance_test.sh
+# VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh
# Version of the standard to test against
# Current options: v1.0, v1.1, v1.2
VERSION=${VERSION:-"v1.2"}
# Which commit of the standard's repo to use
-# Defaults to the last commit of the 1.2.1_proposed branch
-GIT_TARGET=${GIT_TARGET:-"1.2.1_proposed"}
+# Defaults to the last commit of the main branch
+GIT_TARGET=${GIT_TARGET:-"main"}
# Which container runtime to use
# Valid options: docker, singularity
=====================================
cwltool.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20240404144621
+Version: 3.1.20240508115724
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Download-URL: https://github.com/common-workflow-language/cwltool
=====================================
cwltool/_version.py
=====================================
@@ -12,5 +12,5 @@ __version__: str
__version_tuple__: VERSION_TUPLE
version_tuple: VERSION_TUPLE
-__version__ = version = '3.1.20240404144621'
-__version_tuple__ = version_tuple = (3, 1, 20240404144621)
+__version__ = version = '3.1.20240508115724'
+__version_tuple__ = version_tuple = (3, 1, 20240508115724)
=====================================
cwltool/builder.py
=====================================
@@ -86,7 +86,7 @@ def content_limit_respected_read(f: IO[bytes]) -> str:
:returns: the file contents
:raises WorkflowException: if the file is too large
"""
- return content_limit_respected_read_bytes(f).decode("utf-8")
+ return str(content_limit_respected_read_bytes(f), "utf-8")
def substitute(value: str, replace: str) -> str:
=====================================
cwltool/command_line_tool.py
=====================================
@@ -1369,8 +1369,8 @@ class CommandLineTool(Process):
else:
if binding.get("loadContents"):
with fs_access.open(cast(str, rfile["location"]), "rb") as f:
- files["contents"] = content_limit_respected_read_bytes(f).decode(
- "utf-8"
+ files["contents"] = str(
+ content_limit_respected_read_bytes(f), "utf-8"
)
if compute_checksum:
with fs_access.open(cast(str, rfile["location"]), "rb") as f:
=====================================
cwltool/cwlrdf.py
=====================================
@@ -27,7 +27,7 @@ def printrdf(wflow: Process, ctx: ContextType, style: str) -> str:
rdf = gather(wflow, ctx).serialize(format=style, encoding="utf-8")
if not rdf:
return ""
- return rdf.decode("utf-8")
+ return str(rdf, "utf-8")
def lastpart(uri: Any) -> str:
=====================================
cwltool/docker.py
=====================================
@@ -116,36 +116,38 @@ class DockerCommandLineJob(ContainerCommandLineJob):
if (docker_image_id := docker_requirement.get("dockerImageId")) is not None:
try:
manifest = json.loads(
- subprocess.check_output(
- [self.docker_exec, "inspect", docker_image_id]
- ).decode( # nosec
- "utf-8"
+ str(
+ subprocess.check_output(
+ [self.docker_exec, "inspect", docker_image_id]
+ ), # nosec
+ "utf-8",
)
)
found = manifest is not None
except (OSError, subprocess.CalledProcessError, UnicodeError):
pass
+ cmd: List[str] = []
+ if "dockerFile" in docker_requirement:
+ dockerfile_dir = create_tmp_dir(tmp_outdir_prefix)
+ with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
+ dfile.write(docker_requirement["dockerFile"])
+ cmd = [
+ self.docker_exec,
+ "build",
+ "--tag=%s" % str(docker_requirement["dockerImageId"]),
+ dockerfile_dir,
+ ]
+ _logger.info(str(cmd))
+ subprocess.check_call(cmd, stdout=sys.stderr) # nosec
+ found = True
+
if (force_pull or not found) and pull_image:
- cmd: List[str] = []
if "dockerPull" in docker_requirement:
cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])]
_logger.info(str(cmd))
subprocess.check_call(cmd, stdout=sys.stderr) # nosec
found = True
- elif "dockerFile" in docker_requirement:
- dockerfile_dir = create_tmp_dir(tmp_outdir_prefix)
- with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
- dfile.write(docker_requirement["dockerFile"])
- cmd = [
- self.docker_exec,
- "build",
- "--tag=%s" % str(docker_requirement["dockerImageId"]),
- dockerfile_dir,
- ]
- _logger.info(str(cmd))
- subprocess.check_call(cmd, stdout=sys.stderr) # nosec
- found = True
elif "dockerLoad" in docker_requirement:
cmd = [self.docker_exec, "load"]
_logger.info(str(cmd))
=====================================
cwltool/utils.py
=====================================
@@ -200,7 +200,7 @@ def bytes2str_in_dicts(
# if value is bytes, return decoded string,
elif isinstance(inp, bytes):
- return inp.decode("utf-8")
+ return str(inp, "utf-8")
# simply return elements itself
return inp
=====================================
debian/changelog
=====================================
@@ -1,3 +1,9 @@
+cwltool (3.1.20240508115724-1) unstable; urgency=medium
+
+ * New upstream version
+
+ -- Michael R. Crusoe <crusoe at debian.org> Tue, 18 Jun 2024 18:24:58 +0200
+
cwltool (3.1.20240404144621-2) unstable; urgency=medium
* No-change source-only upload.
=====================================
mypy-requirements.txt
=====================================
@@ -1,4 +1,4 @@
-mypy==1.9.0 # also update pyproject.toml
+mypy==1.10.0 # also update pyproject.toml
ruamel.yaml>=0.16.0,<0.19
cwl-utils>=0.32
types-requests
=====================================
pyproject.toml
=====================================
@@ -2,7 +2,7 @@
requires = [
"setuptools>=45",
"setuptools_scm[toml]>=8.0.4,<9",
- "mypy==1.9.0", # also update mypy-requirements.txt
+ "mypy==1.10.0", # also update mypy-requirements.txt
"types-requests",
"types-psutil",
"importlib_resources>=1.4;python_version<'3.9'",
=====================================
release-test.sh
=====================================
@@ -71,7 +71,7 @@ pushd src/${package}
pip install -rtest-requirements.txt build
make dist
#make test
-cp dist/${package}*tar.gz ../../../testenv3/
+cp dist/${module}*tar.gz ../../../testenv3/
pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module
# shellcheck disable=SC2086
@@ -87,13 +87,13 @@ source bin/activate
rm -f lib/python-wheels/setuptools* \
&& pip install --force-reinstall -U pip==${pipver} \
&& pip install setuptools==${setuptoolsver} wheel
-package_tar=$(find . -name "${package}*tar.gz")
+package_tar=$(find . -name "${module}*tar.gz")
pip install "-r${DIR}/test-requirements.txt" udocker build
pip install "${package_tar}${extras}"
udocker install
mkdir out
-tar --extract --directory=out -z -f ${package}*.tar.gz
-pushd out/${package}*
+tar --extract --directory=out -z -f ${module}*.tar.gz
+pushd out/${module}*
make dist
make test
pip install "-r${DIR}/mypy-requirements.txt"
=====================================
setup.py
=====================================
@@ -151,7 +151,7 @@ setup(
test_suite="tests",
tests_require=[
"bagit >= 1.6.4, < 1.9",
- "pytest >= 6.2, < 8.2",
+ "pytest >= 6.2, < 8.3",
"mock >= 2.0.0",
"pytest-mock >= 1.10.0",
"pytest-httpserver",
=====================================
test-requirements.txt
=====================================
@@ -1,5 +1,5 @@
bagit>=1.6.4,<1.9
-pytest>= 6.2,< 8.2
+pytest>= 6.2,< 8.3
pytest-xdist>=3.2.0 # for the worksteal scheduler
psutil # enhances pytest-xdist to allow "-n logical"
pytest-httpserver
=====================================
tests/test_path_checks.py
=====================================
@@ -12,7 +12,9 @@ from cwltool.context import LoadingContext, RuntimeContext
from cwltool.main import main
from cwltool.stdfsaccess import StdFsAccess
from cwltool.update import INTERNAL_VERSION
-from cwltool.utils import CWLObjectType
+from cwltool.utils import CWLObjectType, CONTENT_LIMIT, bytes2str_in_dicts
+from cwltool.builder import content_limit_respected_read
+from cwltool.errors import WorkflowException
from .util import needs_docker
@@ -214,3 +216,26 @@ def test_clt_returns_specialchar_names(tmp_path: Path) -> None:
result["location"]
== "keep:ae755cd1b3cff63152ff4200f4dea7e9+52/%3A%3F%23%5B%5D%40%21%24%26%27%28%29%2A%2B%2C%3B%3D"
)
+
+
+def test_content_limit_respected_read() -> None:
+ b1 = b"abcd" * 100
+ b1io = BytesIO(b1)
+
+ assert len(b1) < CONTENT_LIMIT
+ assert content_limit_respected_read(b1io) == str("abcd" * 100)
+
+ b2 = b"abcd" * 20000
+ b2io = BytesIO(b2)
+
+ assert len(b2) > CONTENT_LIMIT
+ with pytest.raises(WorkflowException):
+ content_limit_respected_read(b2io)
+
+
+def test_bytes2str_in_dicts() -> None:
+ assert bytes2str_in_dicts({"foo": b"bar"}) == {"foo": "bar"}
+
+ assert bytes2str_in_dicts({"foo": [b"bar"]}) == {"foo": ["bar"]}
+
+ assert bytes2str_in_dicts({"foo": {"foo2": b"bar"}}) == {"foo": {"foo2": "bar"}}
=====================================
tests/test_tmpdir.py
=====================================
@@ -168,6 +168,64 @@ def test_dockerfile_tmpdir_prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatc
assert (subdir / "Dockerfile").exists()
+ at needs_docker
+def test_dockerfile_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ """Test that DockerCommandLineJob.get_image builds a Dockerfile."""
+ (tmp_path / "out").mkdir()
+ tmp_outdir_prefix = tmp_path / "out" / "1"
+ (tmp_path / "3").mkdir()
+ tmpdir_prefix = str(tmp_path / "3" / "ttmp")
+ runtime_context = RuntimeContext(
+ {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None}
+ )
+ builder = Builder(
+ {},
+ [],
+ [],
+ {},
+ schema.Names(),
+ [],
+ [],
+ {},
+ None,
+ None,
+ StdFsAccess,
+ StdFsAccess(""),
+ None,
+ 0.1,
+ False,
+ False,
+ False,
+ "no_listing",
+ runtime_context.get_outdir(),
+ runtime_context.get_tmpdir(),
+ runtime_context.get_stagedir(),
+ INTERNAL_VERSION,
+ "docker",
+ )
+
+ docker_image_id = sys._getframe().f_code.co_name
+
+ assert DockerCommandLineJob(
+ builder, {}, CommandLineTool.make_path_mapper, [], [], ""
+ ).get_image(
+ {
+ "class": "DockerRequirement",
+ "dockerFile": "FROM debian:stable-slim",
+ "dockerImageId": docker_image_id,
+ },
+ pull_image=False,
+ force_pull=False,
+ tmp_outdir_prefix=str(tmp_outdir_prefix),
+ )
+ output = subprocess.check_output(
+ ["docker", "images", "--quiet", docker_image_id], stderr=subprocess.STDOUT, text=True
+ )
+
+ # If the output is empty, the image doesn't exist
+ assert output.strip(), f"Docker image {docker_image_id} does not exist"
+
+
@needs_singularity
def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Test that SingularityCommandLineJob.get_image builds a Dockerfile with Singularity."""
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/compare/2f23191d7ec8e699428f352ead4f8e0fc657be83...1be37337b522a67877e10e4236202ded30aa8dbc
--
This project does not include diff previews in email notifications.
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/compare/2f23191d7ec8e699428f352ead4f8e0fc657be83...1be37337b522a67877e10e4236202ded30aa8dbc
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20240621/fe5e088b/attachment-0001.htm>
More information about the debian-med-commit
mailing list