aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/tools/python3/Lib/tokenize.py
diff options
context:
space:
mode:
authorshadchin <shadchin@yandex-team.com>2025-02-16 15:28:01 +0300
committershadchin <shadchin@yandex-team.com>2025-02-16 16:03:50 +0300
commita27b6a96fdc5ca444428ddef4823d0486dcdccb9 (patch)
treeadde5c24d9ea37e1634e7972e27e682a820ab941 /contrib/tools/python3/Lib/tokenize.py
parent7a3958c3c6de324baab9dba4bd4eb808c1b839a6 (diff)
downloadydb-a27b6a96fdc5ca444428ddef4823d0486dcdccb9.tar.gz
Update Python 3 to 3.12.9
commit_hash:c8651982d81e18f18e037fb247cc6ae53c4fa7f1
Diffstat (limited to 'contrib/tools/python3/Lib/tokenize.py')
-rw-r--r--contrib/tools/python3/Lib/tokenize.py14
1 files changed, 4 insertions, 10 deletions
diff --git a/contrib/tools/python3/Lib/tokenize.py b/contrib/tools/python3/Lib/tokenize.py
index b2dff8e696..553c1ca438 100644
--- a/contrib/tools/python3/Lib/tokenize.py
+++ b/contrib/tools/python3/Lib/tokenize.py
@@ -320,16 +320,10 @@ def untokenize(iterable):
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
- Round-trip invariant for full input:
- Untokenized source will match input source exactly
-
- Round-trip invariant for limited input:
- # Output bytes will tokenize back to the input
- t1 = [tok[:2] for tok in tokenize(f.readline)]
- newcode = untokenize(t1)
- readline = BytesIO(newcode).readline
- t2 = [tok[:2] for tok in tokenize(readline)]
- assert t1 == t2
+ The result is guaranteed to tokenize back to match the input so
+ that the conversion is lossless and round-trips are assured.
+ The guarantee applies only to the token type and token string as
+ the spacing between tokens (column positions) may change.
"""
ut = Untokenizer()
out = ut.untokenize(iterable)