[med-svn] [Git][med-team/snakemake][debian-v7] Adapt to f-strings being tokenized in Python 3.12. (Closes: #1061761)
Rebecca N. Palmer (@rnpalmer-guest)
gitlab at salsa.debian.org
Thu Feb 8 22:18:13 GMT 2024
Rebecca N. Palmer pushed to branch debian-v7 at Debian Med / snakemake
Commits:
607bd710 by Rebecca N. Palmer at 2024-02-08T22:17:47+00:00
Adapt to f-strings being tokenized in Python 3.12. (Closes: #1061761)
- - - - -
3 changed files:
- debian/changelog
- + debian/patches/fstring.patch
- debian/patches/series
Changes:
=====================================
debian/changelog
=====================================
@@ -1,9 +1,10 @@
-snakemake (7.32.4-2) UNRELEASED; urgency=medium
+snakemake (7.32.4-2) unstable; urgency=medium
* Tests: don't mix tabs and spaces, as Python 3.12 rejects this.
- (partial fix for #1061761)
+ * Adapt to f-strings being tokenized in Python 3.12.
+ (together, Closes: #1061761)
- -- Rebecca N. Palmer <rebecca_palmer at zoho.com> Sat, 03 Feb 2024 22:14:06 +0000
+ -- Rebecca N. Palmer <rebecca_palmer at zoho.com> Thu, 08 Feb 2024 22:16:41 +0000
snakemake (7.32.4-1) unstable; urgency=medium
=====================================
debian/patches/fstring.patch
=====================================
@@ -0,0 +1,211 @@
+Description: Adapt to Python 3.12 tokenizing f-strings
+
+Origin: upstream 2a50dc02bb709161d62d6f7dc5d6f2733e534c09 + 87c06c0f5745f577c12db39852c6f763a2d41954 + f2c761320a5a73d6027ae3649843e6bf6a24f324
+Author: Hocnonsense, Rebecca N. Palmer <rebecca_palmer at zoho.com>
+
+--- a/snakemake/parser.py
++++ b/snakemake/parser.py
+@@ -3,9 +3,10 @@ __copyright__ = "Copyright 2022, Johanne
+ __email__ = "johannes.koester at uni-due.de"
+ __license__ = "MIT"
+
++import sys
+ import textwrap
+ import tokenize
+-from typing import Any, Dict, Generator, List, Optional
++from typing import Any, Callable, Dict, Generator, List, Optional
+
+ import snakemake
+ from snakemake import common, sourcecache, workflow
+@@ -55,6 +56,10 @@ def is_string(token):
+ return token.type == tokenize.STRING
+
+
++def is_fstring_start(token):
++ return sys.version_info >= (3, 12) and token.type == tokenize.FSTRING_START
++
++
+ def is_eof(token):
+ return token.type == tokenize.ENDMARKER
+
+@@ -74,7 +79,7 @@ class TokenAutomaton:
+ def __init__(self, snakefile: "Snakefile", base_indent=0, dedent=0, root=True):
+ self.root = root
+ self.snakefile = snakefile
+- self.state = None
++ self.state: Callable[[tokenize.TokenInfo], Generator] = None # type: ignore
+ self.base_indent = base_indent
+ self.line = 0
+ self.indent = 0
+@@ -95,11 +100,37 @@ class TokenAutomaton:
+ self.indent = token.end[1] - self.base_indent
+ self.was_indented |= self.indent > 0
+
++ def parse_fstring(self, token: tokenize.TokenInfo):
++ # only for python >= 3.12, since then python changed the
++ # parsing manner of f-string, see
++ # [pep-0701](https://peps.python.org/pep-0701)
++ isin_fstring = 1
++ t = token.string
++ for t1 in self.snakefile:
++ if t1.type == tokenize.FSTRING_START:
++ isin_fstring += 1
++ t += t1.string
++ elif t1.type == tokenize.FSTRING_END:
++ isin_fstring -= 1
++ t += t1.string
++ elif t1.type == tokenize.FSTRING_MIDDLE:
++ t += t1.string.replace("{", "{{").replace("}", "}}")
++ else:
++ t += t1.string
++ if isin_fstring == 0:
++ break
++ if hasattr(self, "cmd") and self.cmd[-1][1] == token:
++ self.cmd[-1] = t, token
++ return t
++
+ def consume(self):
+ for token in self.snakefile:
+ self.indentation(token)
+ try:
+ for t, orig in self.state(token):
++ # python >= 3.12 only
++ if is_fstring_start(token):
++ t = self.parse_fstring(token)
+ if self.lasttoken == "\n" and not t.isspace():
+ yield INDENT * self.effective_indent, orig
+ yield t, orig
+@@ -125,6 +156,7 @@ class TokenAutomaton:
+
+ class KeywordState(TokenAutomaton):
+ prefix = ""
++ start: Callable[[], Generator[str, None, None]]
+
+ def __init__(self, snakefile, base_indent=0, dedent=0, root=True):
+ super().__init__(snakefile, base_indent=base_indent, dedent=dedent, root=root)
+@@ -569,10 +601,10 @@ class AbstractCmd(Run):
+ super().__init__(
+ snakefile, rulename, base_indent=base_indent, dedent=dedent, root=root
+ )
+- self.cmd = list()
++ self.cmd: list[tuple[str, tokenize.TokenInfo]] = []
+ self.token = None
+ if self.overwrite_cmd is not None:
+- self.block_content = self.overwrite_block_content
++ self.block_content = self.overwrite_block_content # type: ignore
+
+ def is_block_end(self, token):
+ return (self.line and self.indent <= 0) or is_eof(token)
+@@ -597,7 +629,7 @@ class AbstractCmd(Run):
+ yield INDENT * (self.effective_indent + 1)
+ yield self.end_func
+ yield "("
+- yield "\n".join(self.cmd)
++ yield from self.cmd
+ yield from self.args()
+ yield "\n"
+ yield ")"
+@@ -610,19 +642,18 @@ class AbstractCmd(Run):
+ self.error(
+ "Command must be given as string after the shell keyword.", token
+ )
+- for t in self.end():
+- yield t, self.token
++ yield from super().decorate_end(self.token)
+
+ def block_content(self, token):
+ self.token = token
+- self.cmd.append(token.string)
++ self.cmd.append((token.string, token))
+ yield token.string, token
+
+ def overwrite_block_content(self, token):
+ if self.token is None:
+ self.token = token
+ cmd = repr(self.overwrite_cmd)
+- self.cmd.append(cmd)
++ self.cmd.append((cmd, token))
+ yield cmd, token
+
+
+@@ -1290,8 +1321,8 @@ def parse(path, workflow, overwrite_shel
+ )
+ snakefile.lines += t.count("\n")
+ compilation.append(t)
+- compilation = "".join(format_tokens(compilation))
+- if linemap:
+- last = max(linemap)
+- linemap[last + 1] = linemap[last]
+- return compilation, linemap, snakefile.rulecount
++ join_compilation = "".join(format_tokens(compilation))
++ if linemap:
++ last = max(linemap)
++ linemap[last + 1] = linemap[last]
++ return join_compilation, snakefile.rulecount
+--- /dev/null
++++ b/tests/test_fstring/Snakefile
+@@ -0,0 +1,51 @@
++shell.executable("bash")
++
++PREFIX = "SID23454678"
++mid = ".t"
++
++rule unit1:
++ output:
++ f"{PREFIX}{mid}xt",
++ shell:
++ "echo '>'{output}'<'; touch {output}; sleep 1"
++
++
++rule unit2:
++ shell:
++ f"ls"
++
++assert (
++ f"""
++{
++ "hello, snakemake"
++}
++"""
++ == """
++hello, snakemake
++"""
++)
++assert (
++ f"""
++ {
++ "hello, snakemake"
++}
++"""
++ == """
++ hello, snakemake
++"""
++)
++
++if 1:
++ assert (
++ f"""
++{
++ "hello, snakemake"
++}
++"""
++ == """
++hello, snakemake
++"""
++ )
++
++assert f"FORMAT['{PREFIX}']['{{}}']" == "FORMAT['SID23454678']['{}']"
++assert f"FORMAT['{PREFIX}'][}}'{{'{{]" == "FORMAT['SID23454678'][}'{'{]"
+--- a/tests/tests.py
++++ b/tests/tests.py
+@@ -1986,6 +1986,10 @@ def test_ensure_checksum_fail():
+ run(dpath("test_ensure"), targets=["d"], shouldfail=True)
+
+
++def test_fstring():
++ run(dpath("test_fstring"), targets=["SID23454678.txt"])
++
++
+ @skip_on_windows
+ def test_github_issue1261():
+ run(dpath("test_github_issue1261"), shouldfail=True, check_results=True)
=====================================
debian/patches/series
=====================================
@@ -16,3 +16,4 @@ skip-tests-with-slurm.patch
skip-test-lint_long_run.patch
skip-test_workflow_calling.patch
no_mix_tabs_spaces.patch
+fstring.patch
View it on GitLab: https://salsa.debian.org/med-team/snakemake/-/commit/607bd710a51b77502c641443cf68a2dae025c695
--
View it on GitLab: https://salsa.debian.org/med-team/snakemake/-/commit/607bd710a51b77502c641443cf68a2dae025c695
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20240208/fa973280/attachment-0001.htm>
More information about the debian-med-commit
mailing list