diff options
author | shadchin <shadchin@yandex-team.com> | 2025-02-16 15:28:01 +0300 |
---|---|---|
committer | shadchin <shadchin@yandex-team.com> | 2025-02-16 16:03:50 +0300 |
commit | a27b6a96fdc5ca444428ddef4823d0486dcdccb9 (patch) | |
tree | adde5c24d9ea37e1634e7972e27e682a820ab941 /contrib/tools/python3/Lib/tokenize.py | |
parent | 7a3958c3c6de324baab9dba4bd4eb808c1b839a6 (diff) | |
download | ydb-a27b6a96fdc5ca444428ddef4823d0486dcdccb9.tar.gz |
Update Python 3 to 3.12.9
commit_hash:c8651982d81e18f18e037fb247cc6ae53c4fa7f1
Diffstat (limited to 'contrib/tools/python3/Lib/tokenize.py')
-rw-r--r-- | contrib/tools/python3/Lib/tokenize.py | 14 |
1 files changed, 4 insertions, 10 deletions
diff --git a/contrib/tools/python3/Lib/tokenize.py b/contrib/tools/python3/Lib/tokenize.py index b2dff8e696..553c1ca438 100644 --- a/contrib/tools/python3/Lib/tokenize.py +++ b/contrib/tools/python3/Lib/tokenize.py @@ -320,16 +320,10 @@ def untokenize(iterable): with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output bytes will tokenize back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. """ ut = Untokenizer() out = ut.untokenize(iterable) |