Library import 240819-0942 (#7994)

Co-authored-by: robot-piglet <robot-piglet@yandex-team.com>
Co-authored-by: hiddenpath <hiddenpath@yandex-team.com>
Co-authored-by: bulatman <bulatman@yandex-team.com>
Co-authored-by: robot-contrib <robot-contrib@yandex-team.com>
Co-authored-by: osidorkin <osidorkin@yandex-team.com>
Co-authored-by: akhropov <akhropov@yandex-team.com>
Co-authored-by: akozhikhov <akozhikhov@yandex-team.com>
Co-authored-by: orlovorlov <orlovorlov@yandex-team.com>
Co-authored-by: babenko <babenko@yandex-team.com>
Co-authored-by: shadchin <shadchin@yandex-team.com>
Co-authored-by: dmasloff <dmasloff@yandex-team.com>
Co-authored-by: aleksei-le <aleksei-le@yandex-team.com>
Co-authored-by: coteeq <coteeq@yandex-team.com>
Co-authored-by: dimdim11 <dimdim11@yandex-team.com>
Co-authored-by: robot-ya-builder <robot-ya-builder@yandex-team.com>
Co-authored-by: iaz1607 <iaz1607@yandex-team.com>

1 files changed, 12 insertions, 1 deletions

diff --git a/contrib/tools/python3/Python/Python-tokenize.c b/contrib/tools/python3/Python/Python-tokenize.c
index 664e7d8a50..baad836d7a 100644
--- a/contrib/tools/python3/Python/Python-tokenize.c
+++ b/contrib/tools/python3/Python/Python-tokenize.c
@@ -35,6 +35,7 @@ typedef struct
     /* Needed to cache line for performance */
     PyObject *last_line;
     Py_ssize_t last_lineno;
+    Py_ssize_t last_end_lineno;
     Py_ssize_t byte_col_offset_diff;
 } tokenizeriterobject;
 
@@ -76,6 +77,7 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
     self->last_line = NULL;
     self->byte_col_offset_diff = 0;
     self->last_lineno = 0;
+    self->last_end_lineno = 0;
 
     return (PyObject *)self;
 }
@@ -212,6 +214,7 @@ tokenizeriter_next(tokenizeriterobject *it)
 
     const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
     PyObject* line = NULL;
+    int line_changed = 1;
     if (it->tok->tok_extra_tokens && is_trailing_token) {
         line = PyUnicode_FromString("");
     } else {
@@ -230,6 +233,7 @@ tokenizeriter_next(tokenizeriterobject *it)
         } else {
             // Line hasn't changed so we reuse the cached one.
             line = it->last_line;
+            line_changed = 0;
         }
     }
     if (line == NULL) {
@@ -240,13 +244,20 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
     Py_ssize_t end_lineno = it->tok->lineno;
     it->last_lineno = lineno;
+    it->last_end_lineno = end_lineno;
 
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;
     Py_ssize_t byte_offset = -1;
     if (token.start != NULL && token.start >= line_start) {
         byte_offset = token.start - line_start;
-        col_offset = byte_offset - it->byte_col_offset_diff;
+        if (line_changed) {
+            col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
+            it->byte_col_offset_diff = byte_offset - col_offset;
+        }
+        else {
+            col_offset = byte_offset - it->byte_col_offset_diff;
+        }
     }
     if (token.end != NULL && token.end >= it->tok->line_start) {
         Py_ssize_t end_byte_offset = token.end - it->tok->line_start;