Restoring authorship annotation for <[email protected]>. Commit 2 of 2.

10 files changed, 3133 insertions, 3133 deletions

diff --git a/contrib/tools/python3/src/Parser/acceler.c b/contrib/tools/python3/src/Parser/acceler.c
index 339fe00e5ad..e515833e1dd 100644
--- a/contrib/tools/python3/src/Parser/acceler.c
+++ b/contrib/tools/python3/src/Parser/acceler.c
@@ -1,123 +1,123 @@
- 
-/* Parser accelerator module */ 
- 
-/* The parser as originally conceived had disappointing performance. 
-   This module does some precomputation that speeds up the selection 
-   of a DFA based upon a token, turning a search through an array 
-   into a simple indexing operation.  The parser now cannot work 
-   without the accelerators installed.  Note that the accelerators 
-   are installed dynamically when the parser is initialized, they 
-   are not part of the static data structure written on graminit.[ch] 
-   by the parser generator. */ 
- 
+
+/* Parser accelerator module */
+
+/* The parser as originally conceived had disappointing performance.
+   This module does some precomputation that speeds up the selection
+   of a DFA based upon a token, turning a search through an array
+   into a simple indexing operation.  The parser now cannot work
+   without the accelerators installed.  Note that the accelerators
+   are installed dynamically when the parser is initialized, they
+   are not part of the static data structure written on graminit.[ch]
+   by the parser generator. */
+
 #include "Python.h"
-#include "grammar.h" 
-#include "node.h" 
-#include "token.h" 
-#include "parser.h" 
- 
-/* Forward references */ 
+#include "grammar.h"
+#include "node.h"
+#include "token.h"
+#include "parser.h"
+
+/* Forward references */
 static void fixdfa(grammar *, const dfa *);
-static void fixstate(grammar *, state *); 
- 
-void 
-PyGrammar_AddAccelerators(grammar *g) 
-{ 
-    int i; 
+static void fixstate(grammar *, state *);
+
+void
+PyGrammar_AddAccelerators(grammar *g)
+{
+    int i;
     const dfa *d = g->g_dfa;
-    for (i = g->g_ndfas; --i >= 0; d++) 
-        fixdfa(g, d); 
-    g->g_accel = 1; 
-} 
- 
-void 
-PyGrammar_RemoveAccelerators(grammar *g) 
-{ 
-    int i; 
-    g->g_accel = 0; 
+    for (i = g->g_ndfas; --i >= 0; d++)
+        fixdfa(g, d);
+    g->g_accel = 1;
+}
+
+void
+PyGrammar_RemoveAccelerators(grammar *g)
+{
+    int i;
+    g->g_accel = 0;
     const dfa *d = g->g_dfa;
-    for (i = g->g_ndfas; --i >= 0; d++) { 
-        state *s; 
-        int j; 
-        s = d->d_state; 
-        for (j = 0; j < d->d_nstates; j++, s++) { 
-            if (s->s_accel) 
-                PyObject_FREE(s->s_accel); 
-            s->s_accel = NULL; 
-        } 
-    } 
-} 
- 
-static void 
+    for (i = g->g_ndfas; --i >= 0; d++) {
+        state *s;
+        int j;
+        s = d->d_state;
+        for (j = 0; j < d->d_nstates; j++, s++) {
+            if (s->s_accel)
+                PyObject_FREE(s->s_accel);
+            s->s_accel = NULL;
+        }
+    }
+}
+
+static void
 fixdfa(grammar *g, const dfa *d)
-{ 
-    state *s; 
-    int j; 
-    s = d->d_state; 
-    for (j = 0; j < d->d_nstates; j++, s++) 
-        fixstate(g, s); 
-} 
- 
-static void 
-fixstate(grammar *g, state *s) 
-{ 
+{
+    state *s;
+    int j;
+    s = d->d_state;
+    for (j = 0; j < d->d_nstates; j++, s++)
+        fixstate(g, s);
+}
+
+static void
+fixstate(grammar *g, state *s)
+{
     const arc *a;
-    int k; 
-    int *accel; 
-    int nl = g->g_ll.ll_nlabels; 
-    s->s_accept = 0; 
-    accel = (int *) PyObject_MALLOC(nl * sizeof(int)); 
-    if (accel == NULL) { 
-        fprintf(stderr, "no mem to build parser accelerators\n"); 
-        exit(1); 
-    } 
-    for (k = 0; k < nl; k++) 
-        accel[k] = -1; 
-    a = s->s_arc; 
-    for (k = s->s_narcs; --k >= 0; a++) { 
-        int lbl = a->a_lbl; 
+    int k;
+    int *accel;
+    int nl = g->g_ll.ll_nlabels;
+    s->s_accept = 0;
+    accel = (int *) PyObject_MALLOC(nl * sizeof(int));
+    if (accel == NULL) {
+        fprintf(stderr, "no mem to build parser accelerators\n");
+        exit(1);
+    }
+    for (k = 0; k < nl; k++)
+        accel[k] = -1;
+    a = s->s_arc;
+    for (k = s->s_narcs; --k >= 0; a++) {
+        int lbl = a->a_lbl;
         const label *l = &g->g_ll.ll_label[lbl];
-        int type = l->lb_type; 
-        if (a->a_arrow >= (1 << 7)) { 
-            printf("XXX too many states!\n"); 
-            continue; 
-        } 
-        if (ISNONTERMINAL(type)) { 
+        int type = l->lb_type;
+        if (a->a_arrow >= (1 << 7)) {
+            printf("XXX too many states!\n");
+            continue;
+        }
+        if (ISNONTERMINAL(type)) {
             const dfa *d1 = PyGrammar_FindDFA(g, type);
-            int ibit; 
-            if (type - NT_OFFSET >= (1 << 7)) { 
-                printf("XXX too high nonterminal number!\n"); 
-                continue; 
-            } 
-            for (ibit = 0; ibit < g->g_ll.ll_nlabels; ibit++) { 
-                if (testbit(d1->d_first, ibit)) { 
-                    if (accel[ibit] != -1) 
-                        printf("XXX ambiguity!\n"); 
-                    accel[ibit] = a->a_arrow | (1 << 7) | 
-                        ((type - NT_OFFSET) << 8); 
-                } 
-            } 
-        } 
-        else if (lbl == EMPTY) 
-            s->s_accept = 1; 
-        else if (lbl >= 0 && lbl < nl) 
-            accel[lbl] = a->a_arrow; 
-    } 
-    while (nl > 0 && accel[nl-1] == -1) 
-        nl--; 
-    for (k = 0; k < nl && accel[k] == -1;) 
-        k++; 
-    if (k < nl) { 
-        int i; 
-        s->s_accel = (int *) PyObject_MALLOC((nl-k) * sizeof(int)); 
-        if (s->s_accel == NULL) { 
-            fprintf(stderr, "no mem to add parser accelerators\n"); 
-            exit(1); 
-        } 
-        s->s_lower = k; 
-        s->s_upper = nl; 
-        for (i = 0; k < nl; i++, k++) 
-            s->s_accel[i] = accel[k]; 
-    } 
-    PyObject_FREE(accel); 
-} 
+            int ibit;
+            if (type - NT_OFFSET >= (1 << 7)) {
+                printf("XXX too high nonterminal number!\n");
+                continue;
+            }
+            for (ibit = 0; ibit < g->g_ll.ll_nlabels; ibit++) {
+                if (testbit(d1->d_first, ibit)) {
+                    if (accel[ibit] != -1)
+                        printf("XXX ambiguity!\n");
+                    accel[ibit] = a->a_arrow | (1 << 7) |
+                        ((type - NT_OFFSET) << 8);
+                }
+            }
+        }
+        else if (lbl == EMPTY)
+            s->s_accept = 1;
+        else if (lbl >= 0 && lbl < nl)
+            accel[lbl] = a->a_arrow;
+    }
+    while (nl > 0 && accel[nl-1] == -1)
+        nl--;
+    for (k = 0; k < nl && accel[k] == -1;)
+        k++;
+    if (k < nl) {
+        int i;
+        s->s_accel = (int *) PyObject_MALLOC((nl-k) * sizeof(int));
+        if (s->s_accel == NULL) {
+            fprintf(stderr, "no mem to add parser accelerators\n");
+            exit(1);
+        }
+        s->s_lower = k;
+        s->s_upper = nl;
+        for (i = 0; k < nl; i++, k++)
+            s->s_accel[i] = accel[k];
+    }
+    PyObject_FREE(accel);
+}
diff --git a/contrib/tools/python3/src/Parser/grammar1.c b/contrib/tools/python3/src/Parser/grammar1.c
index 2c6ecd6b98d..c702040efdf 100644
--- a/contrib/tools/python3/src/Parser/grammar1.c
+++ b/contrib/tools/python3/src/Parser/grammar1.c
@@ -1,47 +1,47 @@
- 
-/* Grammar subroutines needed by parser */ 
- 
-#include "Python.h" 
-#include "grammar.h" 
-#include "token.h" 
- 
-/* Return the DFA for the given type */ 
- 
+
+/* Grammar subroutines needed by parser */
+
+#include "Python.h"
+#include "grammar.h"
+#include "token.h"
+
+/* Return the DFA for the given type */
+
 const dfa *
-PyGrammar_FindDFA(grammar *g, int type) 
-{ 
-    /* Massive speed-up */ 
+PyGrammar_FindDFA(grammar *g, int type)
+{
+    /* Massive speed-up */
     const dfa *d = &g->g_dfa[type - NT_OFFSET];
-    assert(d->d_type == type); 
-    return d; 
-} 
- 
-const char * 
-PyGrammar_LabelRepr(label *lb) 
-{ 
-    static char buf[100]; 
- 
-    if (lb->lb_type == ENDMARKER) 
-        return "EMPTY"; 
-    else if (ISNONTERMINAL(lb->lb_type)) { 
-        if (lb->lb_str == NULL) { 
-            PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type); 
-            return buf; 
-        } 
-        else 
-            return lb->lb_str; 
-    } 
-    else if (lb->lb_type < N_TOKENS) { 
-        if (lb->lb_str == NULL) 
-            return _PyParser_TokenNames[lb->lb_type]; 
-        else { 
-            PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)", 
-                _PyParser_TokenNames[lb->lb_type], lb->lb_str); 
-            return buf; 
-        } 
-    } 
-    else { 
+    assert(d->d_type == type);
+    return d;
+}
+
+const char *
+PyGrammar_LabelRepr(label *lb)
+{
+    static char buf[100];
+
+    if (lb->lb_type == ENDMARKER)
+        return "EMPTY";
+    else if (ISNONTERMINAL(lb->lb_type)) {
+        if (lb->lb_str == NULL) {
+            PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type);
+            return buf;
+        }
+        else
+            return lb->lb_str;
+    }
+    else if (lb->lb_type < N_TOKENS) {
+        if (lb->lb_str == NULL)
+            return _PyParser_TokenNames[lb->lb_type];
+        else {
+            PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)",
+                _PyParser_TokenNames[lb->lb_type], lb->lb_str);
+            return buf;
+        }
+    }
+    else {
         Py_FatalError("invalid grammar label");
-        return NULL; 
-    } 
-} 
+        return NULL;
+    }
+}
diff --git a/contrib/tools/python3/src/Parser/listnode.c b/contrib/tools/python3/src/Parser/listnode.c
index e8844e8a685..c806b98e48c 100644
--- a/contrib/tools/python3/src/Parser/listnode.c
+++ b/contrib/tools/python3/src/Parser/listnode.c
@@ -1,71 +1,71 @@
- 
-/* List a node on a file */ 
- 
+
+/* List a node on a file */
+
 #include "Python.h"
 #include "pycore_interp.h"        // PyInterpreterState.parser
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "token.h" 
-#include "node.h" 
- 
-/* Forward */ 
-static void list1node(FILE *, node *); 
-static void listnode(FILE *, node *); 
- 
-void 
-PyNode_ListTree(node *n) 
-{ 
-    listnode(stdout, n); 
-} 
- 
-static void 
-listnode(FILE *fp, node *n) 
-{ 
+#include "token.h"
+#include "node.h"
+
+/* Forward */
+static void list1node(FILE *, node *);
+static void listnode(FILE *, node *);
+
+void
+PyNode_ListTree(node *n)
+{
+    listnode(stdout, n);
+}
+
+static void
+listnode(FILE *fp, node *n)
+{
     PyInterpreterState *interp = _PyInterpreterState_GET();
 
     interp->parser.listnode.level = 0;
     interp->parser.listnode.atbol = 1;
-    list1node(fp, n); 
-} 
- 
-static void 
-list1node(FILE *fp, node *n) 
-{ 
+    list1node(fp, n);
+}
+
+static void
+list1node(FILE *fp, node *n)
+{
     PyInterpreterState *interp;
 
-    if (n == NULL) 
-        return; 
-    if (ISNONTERMINAL(TYPE(n))) { 
-        int i; 
-        for (i = 0; i < NCH(n); i++) 
-            list1node(fp, CHILD(n, i)); 
-    } 
-    else if (ISTERMINAL(TYPE(n))) { 
+    if (n == NULL)
+        return;
+    if (ISNONTERMINAL(TYPE(n))) {
+        int i;
+        for (i = 0; i < NCH(n); i++)
+            list1node(fp, CHILD(n, i));
+    }
+    else if (ISTERMINAL(TYPE(n))) {
         interp = _PyInterpreterState_GET();
-        switch (TYPE(n)) { 
-        case INDENT: 
+        switch (TYPE(n)) {
+        case INDENT:
             interp->parser.listnode.level++;
-            break; 
-        case DEDENT: 
+            break;
+        case DEDENT:
             interp->parser.listnode.level--;
-            break; 
-        default: 
+            break;
+        default:
             if (interp->parser.listnode.atbol) {
-                int i; 
+                int i;
                 for (i = 0; i < interp->parser.listnode.level; ++i)
-                    fprintf(fp, "\t"); 
+                    fprintf(fp, "\t");
                 interp->parser.listnode.atbol = 0;
-            } 
-            if (TYPE(n) == NEWLINE) { 
-                if (STR(n) != NULL) 
-                    fprintf(fp, "%s", STR(n)); 
-                fprintf(fp, "\n"); 
+            }
+            if (TYPE(n) == NEWLINE) {
+                if (STR(n) != NULL)
+                    fprintf(fp, "%s", STR(n));
+                fprintf(fp, "\n");
                 interp->parser.listnode.atbol = 1;
-            } 
-            else 
-                fprintf(fp, "%s ", STR(n)); 
-            break; 
-        } 
-    } 
-    else 
-        fprintf(fp, "? "); 
-} 
+            }
+            else
+                fprintf(fp, "%s ", STR(n));
+            break;
+        }
+    }
+    else
+        fprintf(fp, "? ");
+}
diff --git a/contrib/tools/python3/src/Parser/myreadline.c b/contrib/tools/python3/src/Parser/myreadline.c
index f775284d843..143b41f1eab 100644
--- a/contrib/tools/python3/src/Parser/myreadline.c
+++ b/contrib/tools/python3/src/Parser/myreadline.c
@@ -1,35 +1,35 @@
- 
-/* Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c. 
-   By default, or when stdin is not a tty device, we have a super 
-   simple my_readline function using fgets. 
-   Optionally, we can use the GNU readline library. 
-   my_readline() has a different return value from GNU readline(): 
-   - NULL if an interrupt occurred or if an error occurred 
-   - a malloc'ed empty string if EOF was read 
-   - a malloc'ed string ending in \n normally 
-*/ 
- 
-#include "Python.h" 
+
+/* Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
+   By default, or when stdin is not a tty device, we have a super
+   simple my_readline function using fgets.
+   Optionally, we can use the GNU readline library.
+   my_readline() has a different return value from GNU readline():
+   - NULL if an interrupt occurred or if an error occurred
+   - a malloc'ed empty string if EOF was read
+   - a malloc'ed string ending in \n normally
+*/
+
+#include "Python.h"
 #include "pycore_pystate.h"   // _PyThreadState_GET()
-#ifdef MS_WINDOWS 
+#ifdef MS_WINDOWS
 #  define WIN32_LEAN_AND_MEAN
 #  include "windows.h"
-#endif /* MS_WINDOWS */ 
- 
- 
-PyThreadState* _PyOS_ReadlineTState = NULL; 
- 
-static PyThread_type_lock _PyOS_ReadlineLock = NULL; 
- 
-int (*PyOS_InputHook)(void) = NULL; 
- 
-/* This function restarts a fgets() after an EINTR error occurred 
+#endif /* MS_WINDOWS */
+
+
+PyThreadState* _PyOS_ReadlineTState = NULL;
+
+static PyThread_type_lock _PyOS_ReadlineLock = NULL;
+
+int (*PyOS_InputHook)(void) = NULL;
+
+/* This function restarts a fgets() after an EINTR error occurred
    except if _PyOS_InterruptOccurred() returns true. */
- 
-static int 
+
+static int
 my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp)
-{ 
-#ifdef MS_WINDOWS 
+{
+#ifdef MS_WINDOWS
     HANDLE handle;
     _Py_BEGIN_SUPPRESS_IPH
     handle = (HANDLE)_get_osfhandle(fileno(fp));
@@ -39,284 +39,284 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp)
     if (handle == INVALID_HANDLE_VALUE) {
         return -1; /* EOF */
     }
-#endif 
+#endif
 
-    while (1) { 
+    while (1) {
         if (PyOS_InputHook != NULL) {
-            (void)(PyOS_InputHook)(); 
+            (void)(PyOS_InputHook)();
         }
 
-        errno = 0; 
-        clearerr(fp); 
+        errno = 0;
+        clearerr(fp);
         char *p = fgets(buf, len, fp);
         if (p != NULL) {
-            return 0; /* No error */ 
+            return 0; /* No error */
         }
         int err = errno;
 
-#ifdef MS_WINDOWS 
-        /* Ctrl-C anywhere on the line or Ctrl-Z if the only character 
-           on a line will set ERROR_OPERATION_ABORTED. Under normal 
-           circumstances Ctrl-C will also have caused the SIGINT handler 
-           to fire which will have set the event object returned by 
-           _PyOS_SigintEvent. This signal fires in another thread and 
-           is not guaranteed to have occurred before this point in the 
-           code. 
- 
-           Therefore: check whether the event is set with a small timeout. 
-           If it is, assume this is a Ctrl-C and reset the event. If it 
-           isn't set assume that this is a Ctrl-Z on its own and drop 
-           through to check for EOF. 
-        */ 
-        if (GetLastError()==ERROR_OPERATION_ABORTED) { 
+#ifdef MS_WINDOWS
+        /* Ctrl-C anywhere on the line or Ctrl-Z if the only character
+           on a line will set ERROR_OPERATION_ABORTED. Under normal
+           circumstances Ctrl-C will also have caused the SIGINT handler
+           to fire which will have set the event object returned by
+           _PyOS_SigintEvent. This signal fires in another thread and
+           is not guaranteed to have occurred before this point in the
+           code.
+
+           Therefore: check whether the event is set with a small timeout.
+           If it is, assume this is a Ctrl-C and reset the event. If it
+           isn't set assume that this is a Ctrl-Z on its own and drop
+           through to check for EOF.
+        */
+        if (GetLastError()==ERROR_OPERATION_ABORTED) {
             HANDLE hInterruptEvent = _PyOS_SigintEvent();
-            switch (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE)) { 
-            case WAIT_OBJECT_0: 
-                ResetEvent(hInterruptEvent); 
-                return 1; /* Interrupt */ 
-            case WAIT_FAILED: 
-                return -2; /* Error */ 
-            } 
-        } 
-#endif /* MS_WINDOWS */ 
+            switch (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE)) {
+            case WAIT_OBJECT_0:
+                ResetEvent(hInterruptEvent);
+                return 1; /* Interrupt */
+            case WAIT_FAILED:
+                return -2; /* Error */
+            }
+        }
+#endif /* MS_WINDOWS */
 
-        if (feof(fp)) { 
-            clearerr(fp); 
-            return -1; /* EOF */ 
-        } 
+        if (feof(fp)) {
+            clearerr(fp);
+            return -1; /* EOF */
+        }
 
-#ifdef EINTR 
-        if (err == EINTR) { 
+#ifdef EINTR
+        if (err == EINTR) {
             PyEval_RestoreThread(tstate);
             int s = PyErr_CheckSignals();
-            PyEval_SaveThread(); 
+            PyEval_SaveThread();
 
             if (s < 0) {
                 return 1;
             }
             /* try again */
-            continue; 
-        } 
-#endif 
+            continue;
+        }
+#endif
 
         if (_PyOS_InterruptOccurred(tstate)) {
-            return 1; /* Interrupt */ 
-        } 
-        return -2; /* Error */ 
-    } 
-    /* NOTREACHED */ 
-} 
- 
-#ifdef MS_WINDOWS 
-/* Readline implementation using ReadConsoleW */ 
- 
-extern char _get_console_type(HANDLE handle); 
- 
-char * 
+            return 1; /* Interrupt */
+        }
+        return -2; /* Error */
+    }
+    /* NOTREACHED */
+}
+
+#ifdef MS_WINDOWS
+/* Readline implementation using ReadConsoleW */
+
+extern char _get_console_type(HANDLE handle);
+
+char *
 _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn)
-{ 
-    static wchar_t wbuf_local[1024 * 16]; 
-    const DWORD chunk_size = 1024; 
- 
-    DWORD n_read, total_read, wbuflen, u8len; 
-    wchar_t *wbuf; 
-    char *buf = NULL; 
-    int err = 0; 
- 
-    n_read = (DWORD)-1; 
-    total_read = 0; 
-    wbuf = wbuf_local; 
-    wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1; 
-    while (1) { 
-        if (PyOS_InputHook != NULL) { 
-            (void)(PyOS_InputHook)(); 
-        } 
-        if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) { 
-            err = GetLastError(); 
-            goto exit; 
-        } 
-        if (n_read == (DWORD)-1 && (err = GetLastError()) == ERROR_OPERATION_ABORTED) { 
-            break; 
-        } 
-        if (n_read == 0) { 
-            int s; 
-            err = GetLastError(); 
-            if (err != ERROR_OPERATION_ABORTED) 
-                goto exit; 
-            err = 0; 
-            HANDLE hInterruptEvent = _PyOS_SigintEvent(); 
-            if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) 
-                    == WAIT_OBJECT_0) { 
-                ResetEvent(hInterruptEvent); 
+{
+    static wchar_t wbuf_local[1024 * 16];
+    const DWORD chunk_size = 1024;
+
+    DWORD n_read, total_read, wbuflen, u8len;
+    wchar_t *wbuf;
+    char *buf = NULL;
+    int err = 0;
+
+    n_read = (DWORD)-1;
+    total_read = 0;
+    wbuf = wbuf_local;
+    wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1;
+    while (1) {
+        if (PyOS_InputHook != NULL) {
+            (void)(PyOS_InputHook)();
+        }
+        if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) {
+            err = GetLastError();
+            goto exit;
+        }
+        if (n_read == (DWORD)-1 && (err = GetLastError()) == ERROR_OPERATION_ABORTED) {
+            break;
+        }
+        if (n_read == 0) {
+            int s;
+            err = GetLastError();
+            if (err != ERROR_OPERATION_ABORTED)
+                goto exit;
+            err = 0;
+            HANDLE hInterruptEvent = _PyOS_SigintEvent();
+            if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE)
+                    == WAIT_OBJECT_0) {
+                ResetEvent(hInterruptEvent);
                 PyEval_RestoreThread(tstate);
-                s = PyErr_CheckSignals(); 
-                PyEval_SaveThread(); 
+                s = PyErr_CheckSignals();
+                PyEval_SaveThread();
                 if (s < 0) {
-                    goto exit; 
+                    goto exit;
                 }
-            } 
-            break; 
-        } 
- 
-        total_read += n_read; 
-        if (total_read == 0 || wbuf[total_read - 1] == L'\n') { 
-            break; 
-        } 
-        wbuflen += chunk_size; 
-        if (wbuf == wbuf_local) { 
-            wbuf[total_read] = '\0'; 
-            wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t)); 
+            }
+            break;
+        }
+
+        total_read += n_read;
+        if (total_read == 0 || wbuf[total_read - 1] == L'\n') {
+            break;
+        }
+        wbuflen += chunk_size;
+        if (wbuf == wbuf_local) {
+            wbuf[total_read] = '\0';
+            wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t));
             if (wbuf) {
-                wcscpy_s(wbuf, wbuflen, wbuf_local); 
+                wcscpy_s(wbuf, wbuflen, wbuf_local);
             }
-            else { 
+            else {
                 PyEval_RestoreThread(tstate);
-                PyErr_NoMemory(); 
+                PyErr_NoMemory();
                 PyEval_SaveThread();
-                goto exit; 
-            } 
-        } 
-        else { 
-            wchar_t *tmp = PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t)); 
-            if (tmp == NULL) { 
+                goto exit;
+            }
+        }
+        else {
+            wchar_t *tmp = PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t));
+            if (tmp == NULL) {
                 PyEval_RestoreThread(tstate);
-                PyErr_NoMemory(); 
+                PyErr_NoMemory();
                 PyEval_SaveThread();
-                goto exit; 
-            } 
-            wbuf = tmp; 
-        } 
-    } 
- 
-    if (wbuf[0] == '\x1a') { 
-        buf = PyMem_RawMalloc(1); 
+                goto exit;
+            }
+            wbuf = tmp;
+        }
+    }
+
+    if (wbuf[0] == '\x1a') {
+        buf = PyMem_RawMalloc(1);
         if (buf) {
-            buf[0] = '\0'; 
+            buf[0] = '\0';
         }
-        else { 
+        else {
             PyEval_RestoreThread(tstate);
-            PyErr_NoMemory(); 
+            PyErr_NoMemory();
             PyEval_SaveThread();
-        } 
-        goto exit; 
-    } 
- 
+        }
+        goto exit;
+    }
+
     u8len = WideCharToMultiByte(CP_UTF8, 0,
                                 wbuf, total_read,
                                 NULL, 0,
                                 NULL, NULL);
-    buf = PyMem_RawMalloc(u8len + 1); 
-    if (buf == NULL) { 
+    buf = PyMem_RawMalloc(u8len + 1);
+    if (buf == NULL) {
         PyEval_RestoreThread(tstate);
-        PyErr_NoMemory(); 
+        PyErr_NoMemory();
         PyEval_SaveThread();
-        goto exit; 
-    } 
+        goto exit;
+    }
 
     u8len = WideCharToMultiByte(CP_UTF8, 0,
                                 wbuf, total_read,
                                 buf, u8len,
                                 NULL, NULL);
-    buf[u8len] = '\0'; 
- 
-exit: 
+    buf[u8len] = '\0';
+
+exit:
     if (wbuf != wbuf_local) {
-        PyMem_RawFree(wbuf); 
+        PyMem_RawFree(wbuf);
     }
- 
-    if (err) { 
+
+    if (err) {
         PyEval_RestoreThread(tstate);
-        PyErr_SetFromWindowsErr(err); 
-        PyEval_SaveThread(); 
-    } 
-    return buf; 
-} 
- 
-#endif 
- 
- 
-/* Readline implementation using fgets() */ 
- 
-char * 
-PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) 
-{ 
-    size_t n; 
-    char *p, *pr; 
+        PyErr_SetFromWindowsErr(err);
+        PyEval_SaveThread();
+    }
+    return buf;
+}
+
+#endif
+
+
+/* Readline implementation using fgets() */
+
+char *
+PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
+{
+    size_t n;
+    char *p, *pr;
     PyThreadState *tstate = _PyOS_ReadlineTState;
     assert(tstate != NULL);
- 
-#ifdef MS_WINDOWS 
-    if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) { 
-        HANDLE hStdIn, hStdErr; 
- 
-        _Py_BEGIN_SUPPRESS_IPH 
-        hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin)); 
-        hStdErr = (HANDLE)_get_osfhandle(fileno(stderr)); 
-        _Py_END_SUPPRESS_IPH 
- 
-        if (_get_console_type(hStdIn) == 'r') { 
-            fflush(sys_stdout); 
-            if (prompt) { 
-                if (_get_console_type(hStdErr) == 'w') { 
-                    wchar_t *wbuf; 
-                    int wlen; 
-                    wlen = MultiByteToWideChar(CP_UTF8, 0, prompt, -1, 
-                            NULL, 0); 
-                    if (wlen) { 
-                        wbuf = PyMem_RawMalloc(wlen * sizeof(wchar_t)); 
-                        if (wbuf == NULL) { 
+
+#ifdef MS_WINDOWS
+    if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) {
+        HANDLE hStdIn, hStdErr;
+
+        _Py_BEGIN_SUPPRESS_IPH
+        hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin));
+        hStdErr = (HANDLE)_get_osfhandle(fileno(stderr));
+        _Py_END_SUPPRESS_IPH
+
+        if (_get_console_type(hStdIn) == 'r') {
+            fflush(sys_stdout);
+            if (prompt) {
+                if (_get_console_type(hStdErr) == 'w') {
+                    wchar_t *wbuf;
+                    int wlen;
+                    wlen = MultiByteToWideChar(CP_UTF8, 0, prompt, -1,
+                            NULL, 0);
+                    if (wlen) {
+                        wbuf = PyMem_RawMalloc(wlen * sizeof(wchar_t));
+                        if (wbuf == NULL) {
                             PyEval_RestoreThread(tstate);
-                            PyErr_NoMemory(); 
+                            PyErr_NoMemory();
                             PyEval_SaveThread();
-                            return NULL; 
-                        } 
-                        wlen = MultiByteToWideChar(CP_UTF8, 0, prompt, -1, 
-                                wbuf, wlen); 
-                        if (wlen) { 
-                            DWORD n; 
-                            fflush(stderr); 
-                            /* wlen includes null terminator, so subtract 1 */ 
-                            WriteConsoleW(hStdErr, wbuf, wlen - 1, &n, NULL); 
-                        } 
-                        PyMem_RawFree(wbuf); 
-                    } 
-                } else { 
-                    fprintf(stderr, "%s", prompt); 
-                    fflush(stderr); 
-                } 
-            } 
-            clearerr(sys_stdin); 
+                            return NULL;
+                        }
+                        wlen = MultiByteToWideChar(CP_UTF8, 0, prompt, -1,
+                                wbuf, wlen);
+                        if (wlen) {
+                            DWORD n;
+                            fflush(stderr);
+                            /* wlen includes null terminator, so subtract 1 */
+                            WriteConsoleW(hStdErr, wbuf, wlen - 1, &n, NULL);
+                        }
+                        PyMem_RawFree(wbuf);
+                    }
+                } else {
+                    fprintf(stderr, "%s", prompt);
+                    fflush(stderr);
+                }
+            }
+            clearerr(sys_stdin);
             return _PyOS_WindowsConsoleReadline(tstate, hStdIn);
-        } 
-    } 
-#endif 
- 
-    fflush(sys_stdout); 
+        }
+    }
+#endif
+
+    fflush(sys_stdout);
     if (prompt) {
-        fprintf(stderr, "%s", prompt); 
+        fprintf(stderr, "%s", prompt);
     }
-    fflush(stderr); 
- 
+    fflush(stderr);
+
     n = 0;
     p = NULL;
     do {
         size_t incr = (n > 0) ? n + 2 : 100;
-        if (incr > INT_MAX) { 
-            PyMem_RawFree(p); 
+        if (incr > INT_MAX) {
+            PyMem_RawFree(p);
             PyEval_RestoreThread(tstate);
-            PyErr_SetString(PyExc_OverflowError, "input line too long"); 
+            PyErr_SetString(PyExc_OverflowError, "input line too long");
             PyEval_SaveThread();
-            return NULL; 
-        } 
-        pr = (char *)PyMem_RawRealloc(p, n + incr); 
-        if (pr == NULL) { 
-            PyMem_RawFree(p); 
+            return NULL;
+        }
+        pr = (char *)PyMem_RawRealloc(p, n + incr);
+        if (pr == NULL) {
+            PyMem_RawFree(p);
             PyEval_RestoreThread(tstate);
-            PyErr_NoMemory(); 
+            PyErr_NoMemory();
             PyEval_SaveThread();
-            return NULL; 
-        } 
-        p = pr; 
+            return NULL;
+        }
+        p = pr;
         int err = my_fgets(tstate, p + n, (int)incr, sys_stdin);
         if (err == 1) {
             // Interrupt
@@ -325,91 +325,91 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
         } else if (err != 0) {
             // EOF or error
             p[n] = '\0';
-            break; 
+            break;
         }
         n += strlen(p + n);
     } while (p[n-1] != '\n');
 
-    pr = (char *)PyMem_RawRealloc(p, n+1); 
-    if (pr == NULL) { 
-        PyMem_RawFree(p); 
+    pr = (char *)PyMem_RawRealloc(p, n+1);
+    if (pr == NULL) {
+        PyMem_RawFree(p);
         PyEval_RestoreThread(tstate);
-        PyErr_NoMemory(); 
+        PyErr_NoMemory();
         PyEval_SaveThread();
-        return NULL; 
-    } 
-    return pr; 
-} 
- 
- 
-/* By initializing this function pointer, systems embedding Python can 
-   override the readline function. 
- 
-   Note: Python expects in return a buffer allocated with PyMem_Malloc. */ 
- 
-char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) = NULL; 
- 
- 
-/* Interface used by tokenizer.c and bltinmodule.c */ 
- 
-char * 
-PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) 
-{ 
-    char *rv, *res; 
-    size_t len; 
- 
+        return NULL;
+    }
+    return pr;
+}
+
+
+/* By initializing this function pointer, systems embedding Python can
+   override the readline function.
+
+   Note: Python expects in return a buffer allocated with PyMem_Malloc. */
+
+char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) = NULL;
+
+
+/* Interface used by tokenizer.c and bltinmodule.c */
+
+char *
+PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
+{
+    char *rv, *res;
+    size_t len;
+
     PyThreadState *tstate = _PyThreadState_GET();
     if (_PyOS_ReadlineTState == tstate) {
-        PyErr_SetString(PyExc_RuntimeError, 
-                        "can't re-enter readline"); 
-        return NULL; 
-    } 
- 
- 
-    if (PyOS_ReadlineFunctionPointer == NULL) { 
-        PyOS_ReadlineFunctionPointer = PyOS_StdioReadline; 
-    } 
- 
-    if (_PyOS_ReadlineLock == NULL) { 
-        _PyOS_ReadlineLock = PyThread_allocate_lock(); 
-        if (_PyOS_ReadlineLock == NULL) { 
-            PyErr_SetString(PyExc_MemoryError, "can't allocate lock"); 
-            return NULL; 
-        } 
-    } 
- 
+        PyErr_SetString(PyExc_RuntimeError,
+                        "can't re-enter readline");
+        return NULL;
+    }
+
+
+    if (PyOS_ReadlineFunctionPointer == NULL) {
+        PyOS_ReadlineFunctionPointer = PyOS_StdioReadline;
+    }
+
+    if (_PyOS_ReadlineLock == NULL) {
+        _PyOS_ReadlineLock = PyThread_allocate_lock();
+        if (_PyOS_ReadlineLock == NULL) {
+            PyErr_SetString(PyExc_MemoryError, "can't allocate lock");
+            return NULL;
+        }
+    }
+
     _PyOS_ReadlineTState = tstate;
-    Py_BEGIN_ALLOW_THREADS 
-    PyThread_acquire_lock(_PyOS_ReadlineLock, 1); 
- 
-    /* This is needed to handle the unlikely case that the 
-     * interpreter is in interactive mode *and* stdin/out are not 
-     * a tty.  This can happen, for example if python is run like 
-     * this: python -i < test1.py 
-     */ 
-    if (!isatty (fileno (sys_stdin)) || !isatty (fileno (sys_stdout))) 
-        rv = PyOS_StdioReadline (sys_stdin, sys_stdout, prompt); 
-    else 
-        rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, 
-                                             prompt); 
-    Py_END_ALLOW_THREADS 
- 
-    PyThread_release_lock(_PyOS_ReadlineLock); 
- 
-    _PyOS_ReadlineTState = NULL; 
- 
-    if (rv == NULL) 
-        return NULL; 
- 
-    len = strlen(rv) + 1; 
-    res = PyMem_Malloc(len); 
-    if (res != NULL) { 
-        memcpy(res, rv, len); 
-    } 
-    else { 
-        PyErr_NoMemory(); 
-    } 
-    PyMem_RawFree(rv); 
- 
-    return res; 
-} 
+    Py_BEGIN_ALLOW_THREADS
+    PyThread_acquire_lock(_PyOS_ReadlineLock, 1);
+
+    /* This is needed to handle the unlikely case that the
+     * interpreter is in interactive mode *and* stdin/out are not
+     * a tty.  This can happen, for example if python is run like
+     * this: python -i < test1.py
+     */
+    if (!isatty (fileno (sys_stdin)) || !isatty (fileno (sys_stdout)))
+        rv = PyOS_StdioReadline (sys_stdin, sys_stdout, prompt);
+    else
+        rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout,
+                                             prompt);
+    Py_END_ALLOW_THREADS
+
+    PyThread_release_lock(_PyOS_ReadlineLock);
+
+    _PyOS_ReadlineTState = NULL;
+
+    if (rv == NULL)
+        return NULL;
+
+    len = strlen(rv) + 1;
+    res = PyMem_Malloc(len);
+    if (res != NULL) {
+        memcpy(res, rv, len);
+    }
+    else {
+        PyErr_NoMemory();
+    }
+    PyMem_RawFree(rv);
+
+    return res;
+}
diff --git a/contrib/tools/python3/src/Parser/node.c b/contrib/tools/python3/src/Parser/node.c
index fb25147fcfa..8789e01e9b8 100644
--- a/contrib/tools/python3/src/Parser/node.c
+++ b/contrib/tools/python3/src/Parser/node.c
@@ -1,83 +1,83 @@
-/* Parse tree node implementation */ 
- 
-#include "Python.h" 
-#include "node.h" 
-#include "errcode.h" 
- 
-node * 
-PyNode_New(int type) 
-{ 
-    node *n = (node *) PyObject_MALLOC(1 * sizeof(node)); 
-    if (n == NULL) 
-        return NULL; 
-    n->n_type = type; 
-    n->n_str = NULL; 
-    n->n_lineno = 0; 
+/* Parse tree node implementation */
+
+#include "Python.h"
+#include "node.h"
+#include "errcode.h"
+
+node *
+PyNode_New(int type)
+{
+    node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
+    if (n == NULL)
+        return NULL;
+    n->n_type = type;
+    n->n_str = NULL;
+    n->n_lineno = 0;
     n->n_end_lineno = 0;
     n->n_col_offset = 0;
     n->n_end_col_offset = -1;
-    n->n_nchildren = 0; 
-    n->n_child = NULL; 
-    return n; 
-} 
- 
-/* See comments at XXXROUNDUP below.  Returns -1 on overflow. */ 
-static int 
-fancy_roundup(int n) 
-{ 
-    /* Round up to the closest power of 2 >= n. */ 
-    int result = 256; 
-    assert(n > 128); 
-    while (result < n) { 
-        result <<= 1; 
-        if (result <= 0) 
-            return -1; 
-    } 
-    return result; 
-} 
- 
-/* A gimmick to make massive numbers of reallocs quicker.  The result is 
- * a number >= the input.  In PyNode_AddChild, it's used like so, when 
- * we're about to add child number current_size + 1: 
- * 
- *     if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1): 
- *         allocate space for XXXROUNDUP(current_size + 1) total children 
- *     else: 
- *         we already have enough space 
- * 
- * Since a node starts out empty, we must have 
- * 
- *     XXXROUNDUP(0) < XXXROUNDUP(1) 
- * 
- * so that we allocate space for the first child.  One-child nodes are very 
- * common (presumably that would change if we used a more abstract form 
- * of syntax tree), so to avoid wasting memory it's desirable that 
- * XXXROUNDUP(1) == 1.  That in turn forces XXXROUNDUP(0) == 0. 
- * 
- * Else for 2 <= n <= 128, we round up to the closest multiple of 4.  Why 4? 
- * Rounding up to a multiple of an exact power of 2 is very efficient, and 
- * most nodes with more than one child have <= 4 kids. 
- * 
- * Else we call fancy_roundup() to grow proportionately to n.  We've got an 
- * extreme case then (like test_longexp.py), and on many platforms doing 
- * anything less than proportional growth leads to exorbitant runtime 
- * (e.g., MacPython), or extreme fragmentation of user address space (e.g., 
- * Win98). 
- * 
- * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre 
- * reported that, with this scheme, 89% of PyObject_REALLOC calls in 
- * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually 
- * wastes very little memory, but is very effective at sidestepping 
- * platform-realloc disasters on vulnerable platforms. 
- * 
- * Note that this would be straightforward if a node stored its current 
- * capacity.  The code is tricky to avoid that. 
- */ 
-#define XXXROUNDUP(n) ((n) <= 1 ? (n) :                         \ 
-               (n) <= 128 ? (int)_Py_SIZE_ROUND_UP((n), 4) :    \ 
-               fancy_roundup(n)) 
- 
- 
+    n->n_nchildren = 0;
+    n->n_child = NULL;
+    return n;
+}
+
+/* See comments at XXXROUNDUP below.  Returns -1 on overflow. */
+static int
+fancy_roundup(int n)
+{
+    /* Round up to the closest power of 2 >= n. */
+    int result = 256;
+    assert(n > 128);
+    while (result < n) {
+        result <<= 1;
+        if (result <= 0)
+            return -1;
+    }
+    return result;
+}
+
+/* A gimmick to make massive numbers of reallocs quicker.  The result is
+ * a number >= the input.  In PyNode_AddChild, it's used like so, when
+ * we're about to add child number current_size + 1:
+ *
+ *     if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1):
+ *         allocate space for XXXROUNDUP(current_size + 1) total children
+ *     else:
+ *         we already have enough space
+ *
+ * Since a node starts out empty, we must have
+ *
+ *     XXXROUNDUP(0) < XXXROUNDUP(1)
+ *
+ * so that we allocate space for the first child.  One-child nodes are very
+ * common (presumably that would change if we used a more abstract form
+ * of syntax tree), so to avoid wasting memory it's desirable that
+ * XXXROUNDUP(1) == 1.  That in turn forces XXXROUNDUP(0) == 0.
+ *
+ * Else for 2 <= n <= 128, we round up to the closest multiple of 4.  Why 4?
+ * Rounding up to a multiple of an exact power of 2 is very efficient, and
+ * most nodes with more than one child have <= 4 kids.
+ *
+ * Else we call fancy_roundup() to grow proportionately to n.  We've got an
+ * extreme case then (like test_longexp.py), and on many platforms doing
+ * anything less than proportional growth leads to exorbitant runtime
+ * (e.g., MacPython), or extreme fragmentation of user address space (e.g.,
+ * Win98).
+ *
+ * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
+ * reported that, with this scheme, 89% of PyObject_REALLOC calls in
+ * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually
+ * wastes very little memory, but is very effective at sidestepping
+ * platform-realloc disasters on vulnerable platforms.
+ *
+ * Note that this would be straightforward if a node stored its current
+ * capacity.  The code is tricky to avoid that.
+ */
+#define XXXROUNDUP(n) ((n) <= 1 ? (n) :                         \
+               (n) <= 128 ? (int)_Py_SIZE_ROUND_UP((n), 4) :    \
+               fancy_roundup(n))
+
+
 void
 _PyNode_FinalizeEndPos(node *n)
 {
@@ -92,98 +92,98 @@ _PyNode_FinalizeEndPos(node *n)
     n->n_end_col_offset = last->n_end_col_offset;
 }
 
-int 
+int
 PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset,
                 int end_lineno, int end_col_offset)
-{ 
-    const int nch = n1->n_nchildren; 
-    int current_capacity; 
-    int required_capacity; 
-    node *n; 
- 
+{
+    const int nch = n1->n_nchildren;
+    int current_capacity;
+    int required_capacity;
+    node *n;
+
     // finalize end position of previous node (if any)
     if (nch > 0) {
         _PyNode_FinalizeEndPos(CHILD(n1, nch - 1));
     }
 
-    if (nch == INT_MAX || nch < 0) 
-        return E_OVERFLOW; 
- 
-    current_capacity = XXXROUNDUP(nch); 
-    required_capacity = XXXROUNDUP(nch + 1); 
-    if (current_capacity < 0 || required_capacity < 0) 
-        return E_OVERFLOW; 
-    if (current_capacity < required_capacity) { 
-        if ((size_t)required_capacity > SIZE_MAX / sizeof(node)) { 
-            return E_NOMEM; 
-        } 
-        n = n1->n_child; 
-        n = (node *) PyObject_REALLOC(n, 
-                                      required_capacity * sizeof(node)); 
-        if (n == NULL) 
-            return E_NOMEM; 
-        n1->n_child = n; 
-    } 
- 
-    n = &n1->n_child[n1->n_nchildren++]; 
-    n->n_type = type; 
-    n->n_str = str; 
-    n->n_lineno = lineno; 
-    n->n_col_offset = col_offset; 
+    if (nch == INT_MAX || nch < 0)
+        return E_OVERFLOW;
+
+    current_capacity = XXXROUNDUP(nch);
+    required_capacity = XXXROUNDUP(nch + 1);
+    if (current_capacity < 0 || required_capacity < 0)
+        return E_OVERFLOW;
+    if (current_capacity < required_capacity) {
+        if ((size_t)required_capacity > SIZE_MAX / sizeof(node)) {
+            return E_NOMEM;
+        }
+        n = n1->n_child;
+        n = (node *) PyObject_REALLOC(n,
+                                      required_capacity * sizeof(node));
+        if (n == NULL)
+            return E_NOMEM;
+        n1->n_child = n;
+    }
+
+    n = &n1->n_child[n1->n_nchildren++];
+    n->n_type = type;
+    n->n_str = str;
+    n->n_lineno = lineno;
+    n->n_col_offset = col_offset;
     n->n_end_lineno = end_lineno;  // this and below will be updates after all children are added.
     n->n_end_col_offset = end_col_offset;
-    n->n_nchildren = 0; 
-    n->n_child = NULL; 
-    return 0; 
-} 
- 
-/* Forward */ 
-static void freechildren(node *); 
-static Py_ssize_t sizeofchildren(node *n); 
- 
- 
-void 
-PyNode_Free(node *n) 
-{ 
-    if (n != NULL) { 
-        freechildren(n); 
-        PyObject_FREE(n); 
-    } 
-} 
- 
-Py_ssize_t 
-_PyNode_SizeOf(node *n) 
-{ 
-    Py_ssize_t res = 0; 
- 
-    if (n != NULL) 
-        res = sizeof(node) + sizeofchildren(n); 
-    return res; 
-} 
- 
-static void 
-freechildren(node *n) 
-{ 
-    int i; 
-    for (i = NCH(n); --i >= 0; ) 
-        freechildren(CHILD(n, i)); 
-    if (n->n_child != NULL) 
-        PyObject_FREE(n->n_child); 
-    if (STR(n) != NULL) 
-        PyObject_FREE(STR(n)); 
-} 
- 
-static Py_ssize_t 
-sizeofchildren(node *n) 
-{ 
-    Py_ssize_t res = 0; 
-    int i; 
-    for (i = NCH(n); --i >= 0; ) 
-        res += sizeofchildren(CHILD(n, i)); 
-    if (n->n_child != NULL) 
-        /* allocated size of n->n_child array */ 
-        res += XXXROUNDUP(NCH(n)) * sizeof(node); 
-    if (STR(n) != NULL) 
-        res += strlen(STR(n)) + 1; 
-    return res; 
-} 
+    n->n_nchildren = 0;
+    n->n_child = NULL;
+    return 0;
+}
+
+/* Forward */
+static void freechildren(node *);
+static Py_ssize_t sizeofchildren(node *n);
+
+
+void
+PyNode_Free(node *n)
+{
+    if (n != NULL) {
+        freechildren(n);
+        PyObject_FREE(n);
+    }
+}
+
+Py_ssize_t
+_PyNode_SizeOf(node *n)
+{
+    Py_ssize_t res = 0;
+
+    if (n != NULL)
+        res = sizeof(node) + sizeofchildren(n);
+    return res;
+}
+
+static void
+freechildren(node *n)
+{
+    int i;
+    for (i = NCH(n); --i >= 0; )
+        freechildren(CHILD(n, i));
+    if (n->n_child != NULL)
+        PyObject_FREE(n->n_child);
+    if (STR(n) != NULL)
+        PyObject_FREE(STR(n));
+}
+
+static Py_ssize_t
+sizeofchildren(node *n)
+{
+    Py_ssize_t res = 0;
+    int i;
+    for (i = NCH(n); --i >= 0; )
+        res += sizeofchildren(CHILD(n, i));
+    if (n->n_child != NULL)
+        /* allocated size of n->n_child array */
+        res += XXXROUNDUP(NCH(n)) * sizeof(node);
+    if (STR(n) != NULL)
+        res += strlen(STR(n)) + 1;
+    return res;
+}
diff --git a/contrib/tools/python3/src/Parser/parser.c b/contrib/tools/python3/src/Parser/parser.c
index c25f4651a3a..a61b2f5ebf7 100644
--- a/contrib/tools/python3/src/Parser/parser.c
+++ b/contrib/tools/python3/src/Parser/parser.c
@@ -1,266 +1,266 @@
- 
-/* Parser implementation */ 
- 
-/* For a description, see the comments at end of this file */ 
- 
-/* XXX To do: error recovery */ 
- 
-#include "Python.h" 
-#include "token.h" 
-#include "grammar.h" 
-#include "node.h" 
-#include "parser.h" 
-#include "errcode.h" 
+
+/* Parser implementation */
+
+/* For a description, see the comments at end of this file */
+
+/* XXX To do: error recovery */
+
+#include "Python.h"
+#include "token.h"
+#include "grammar.h"
+#include "node.h"
+#include "parser.h"
+#include "errcode.h"
 #include "graminit.h"
- 
- 
-#ifdef Py_DEBUG 
-extern int Py_DebugFlag; 
-#define D(x) if (!Py_DebugFlag); else x 
-#else 
-#define D(x) 
-#endif 
- 
- 
-/* STACK DATA TYPE */ 
- 
-static void s_reset(stack *); 
- 
-static void 
-s_reset(stack *s) 
-{ 
-    s->s_top = &s->s_base[MAXSTACK]; 
-} 
- 
-#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK]) 
- 
-static int 
+
+
+#ifdef Py_DEBUG
+extern int Py_DebugFlag;
+#define D(x) if (!Py_DebugFlag); else x
+#else
+#define D(x)
+#endif
+
+
+/* STACK DATA TYPE */
+
+static void s_reset(stack *);
+
+static void
+s_reset(stack *s)
+{
+    s->s_top = &s->s_base[MAXSTACK];
+}
+
+#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])
+
+static int
 s_push(stack *s, const dfa *d, node *parent)
-{ 
-    stackentry *top; 
-    if (s->s_top == s->s_base) { 
-        fprintf(stderr, "s_push: parser stack overflow\n"); 
-        return E_NOMEM; 
-    } 
-    top = --s->s_top; 
-    top->s_dfa = d; 
-    top->s_parent = parent; 
-    top->s_state = 0; 
-    return 0; 
-} 
- 
-#ifdef Py_DEBUG 
- 
-static void 
-s_pop(stack *s) 
-{ 
+{
+    stackentry *top;
+    if (s->s_top == s->s_base) {
+        fprintf(stderr, "s_push: parser stack overflow\n");
+        return E_NOMEM;
+    }
+    top = --s->s_top;
+    top->s_dfa = d;
+    top->s_parent = parent;
+    top->s_state = 0;
+    return 0;
+}
+
+#ifdef Py_DEBUG
+
+static void
+s_pop(stack *s)
+{
     if (s_empty(s)) {
         Py_FatalError("parser stack underflow");
     }
-    s->s_top++; 
-} 
- 
-#else /* !Py_DEBUG */ 
- 
-#define s_pop(s) (s)->s_top++ 
- 
-#endif 
- 
- 
-/* PARSER CREATION */ 
- 
-parser_state * 
-PyParser_New(grammar *g, int start) 
-{ 
-    parser_state *ps; 
- 
-    if (!g->g_accel) 
-        PyGrammar_AddAccelerators(g); 
-    ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); 
-    if (ps == NULL) 
-        return NULL; 
-    ps->p_grammar = g; 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-    ps->p_flags = 0; 
-#endif 
-    ps->p_tree = PyNode_New(start); 
-    if (ps->p_tree == NULL) { 
-        PyMem_FREE(ps); 
-        return NULL; 
-    } 
-    s_reset(&ps->p_stack); 
-    (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); 
-    return ps; 
-} 
- 
-void 
-PyParser_Delete(parser_state *ps) 
-{ 
-    /* NB If you want to save the parse tree, 
-       you must set p_tree to NULL before calling delparser! */ 
-    PyNode_Free(ps->p_tree); 
-    PyMem_FREE(ps); 
-} 
- 
- 
-/* PARSER STACK OPERATIONS */ 
- 
-static int 
+    s->s_top++;
+}
+
+#else /* !Py_DEBUG */
+
+#define s_pop(s) (s)->s_top++
+
+#endif
+
+
+/* PARSER CREATION */
+
+parser_state *
+PyParser_New(grammar *g, int start)
+{
+    parser_state *ps;
+
+    if (!g->g_accel)
+        PyGrammar_AddAccelerators(g);
+    ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state));
+    if (ps == NULL)
+        return NULL;
+    ps->p_grammar = g;
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+    ps->p_flags = 0;
+#endif
+    ps->p_tree = PyNode_New(start);
+    if (ps->p_tree == NULL) {
+        PyMem_FREE(ps);
+        return NULL;
+    }
+    s_reset(&ps->p_stack);
+    (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree);
+    return ps;
+}
+
+void
+PyParser_Delete(parser_state *ps)
+{
+    /* NB If you want to save the parse tree,
+       you must set p_tree to NULL before calling delparser! */
+    PyNode_Free(ps->p_tree);
+    PyMem_FREE(ps);
+}
+
+
+/* PARSER STACK OPERATIONS */
+
+static int
 shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset,
       int end_lineno, int end_col_offset)
-{ 
-    int err; 
-    assert(!s_empty(s)); 
+{
+    int err;
+    assert(!s_empty(s));
     err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset,
                           end_lineno, end_col_offset);
-    if (err) 
-        return err; 
-    s->s_top->s_state = newstate; 
-    return 0; 
-} 
- 
-static int 
+    if (err)
+        return err;
+    s->s_top->s_state = newstate;
+    return 0;
+}
+
+static int
 push(stack *s, int type, const dfa *d, int newstate, int lineno, int col_offset,
      int end_lineno, int end_col_offset)
-{ 
-    int err; 
-    node *n; 
-    n = s->s_top->s_parent; 
-    assert(!s_empty(s)); 
+{
+    int err;
+    node *n;
+    n = s->s_top->s_parent;
+    assert(!s_empty(s));
     err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset,
                           end_lineno, end_col_offset);
-    if (err) 
-        return err; 
-    s->s_top->s_state = newstate; 
-    return s_push(s, d, CHILD(n, NCH(n)-1)); 
-} 
- 
- 
-/* PARSER PROPER */ 
- 
-static int 
-classify(parser_state *ps, int type, const char *str) 
-{ 
-    grammar *g = ps->p_grammar; 
-    int n = g->g_ll.ll_nlabels; 
- 
-    if (type == NAME) { 
+    if (err)
+        return err;
+    s->s_top->s_state = newstate;
+    return s_push(s, d, CHILD(n, NCH(n)-1));
+}
+
+
+/* PARSER PROPER */
+
+static int
+classify(parser_state *ps, int type, const char *str)
+{
+    grammar *g = ps->p_grammar;
+    int n = g->g_ll.ll_nlabels;
+
+    if (type == NAME) {
         const label *l = g->g_ll.ll_label;
-        int i; 
-        for (i = n; i > 0; i--, l++) { 
-            if (l->lb_type != NAME || l->lb_str == NULL || 
-                l->lb_str[0] != str[0] || 
-                strcmp(l->lb_str, str) != 0) 
-                continue; 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-#if 0 
-            /* Leaving this in as an example */ 
-            if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { 
-                if (str[0] == 'w' && strcmp(str, "with") == 0) 
-                    break; /* not a keyword yet */ 
-                else if (str[0] == 'a' && strcmp(str, "as") == 0) 
-                    break; /* not a keyword yet */ 
-            } 
-#endif 
-#endif 
-            D(printf("It's a keyword\n")); 
-            return n - i; 
-        } 
-    } 
- 
-    { 
+        int i;
+        for (i = n; i > 0; i--, l++) {
+            if (l->lb_type != NAME || l->lb_str == NULL ||
+                l->lb_str[0] != str[0] ||
+                strcmp(l->lb_str, str) != 0)
+                continue;
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+#if 0
+            /* Leaving this in as an example */
+            if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) {
+                if (str[0] == 'w' && strcmp(str, "with") == 0)
+                    break; /* not a keyword yet */
+                else if (str[0] == 'a' && strcmp(str, "as") == 0)
+                    break; /* not a keyword yet */
+            }
+#endif
+#endif
+            D(printf("It's a keyword\n"));
+            return n - i;
+        }
+    }
+
+    {
         const label *l = g->g_ll.ll_label;
-        int i; 
-        for (i = n; i > 0; i--, l++) { 
-            if (l->lb_type == type && l->lb_str == NULL) { 
-                D(printf("It's a token we know\n")); 
-                return n - i; 
-            } 
-        } 
-    } 
- 
-    D(printf("Illegal token\n")); 
-    return -1; 
-} 
- 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-#if 0 
-/* Leaving this in as an example */ 
-static void 
-future_hack(parser_state *ps) 
-{ 
-    node *n = ps->p_stack.s_top->s_parent; 
-    node *ch, *cch; 
-    int i; 
- 
-    /* from __future__ import ..., must have at least 4 children */ 
-    n = CHILD(n, 0); 
-    if (NCH(n) < 4) 
-        return; 
-    ch = CHILD(n, 0); 
-    if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0) 
-        return; 
-    ch = CHILD(n, 1); 
-    if (NCH(ch) == 1 && STR(CHILD(ch, 0)) && 
-        strcmp(STR(CHILD(ch, 0)), "__future__") != 0) 
-        return; 
-    ch = CHILD(n, 3); 
-    /* ch can be a star, a parenthesis or import_as_names */ 
-    if (TYPE(ch) == STAR) 
-        return; 
-    if (TYPE(ch) == LPAR) 
-        ch = CHILD(n, 4); 
- 
-    for (i = 0; i < NCH(ch); i += 2) { 
-        cch = CHILD(ch, i); 
-        if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) { 
-            char *str_ch = STR(CHILD(cch, 0)); 
-            if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) { 
-                ps->p_flags |= CO_FUTURE_WITH_STATEMENT; 
-            } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) { 
-                ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; 
-            } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) { 
-                ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; 
-            } 
-        } 
-    } 
-} 
-#endif 
-#endif /* future keyword */ 
- 
-int 
-PyParser_AddToken(parser_state *ps, int type, char *str, 
+        int i;
+        for (i = n; i > 0; i--, l++) {
+            if (l->lb_type == type && l->lb_str == NULL) {
+                D(printf("It's a token we know\n"));
+                return n - i;
+            }
+        }
+    }
+
+    D(printf("Illegal token\n"));
+    return -1;
+}
+
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+#if 0
+/* Leaving this in as an example */
+static void
+future_hack(parser_state *ps)
+{
+    node *n = ps->p_stack.s_top->s_parent;
+    node *ch, *cch;
+    int i;
+
+    /* from __future__ import ..., must have at least 4 children */
+    n = CHILD(n, 0);
+    if (NCH(n) < 4)
+        return;
+    ch = CHILD(n, 0);
+    if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0)
+        return;
+    ch = CHILD(n, 1);
+    if (NCH(ch) == 1 && STR(CHILD(ch, 0)) &&
+        strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
+        return;
+    ch = CHILD(n, 3);
+    /* ch can be a star, a parenthesis or import_as_names */
+    if (TYPE(ch) == STAR)
+        return;
+    if (TYPE(ch) == LPAR)
+        ch = CHILD(n, 4);
+
+    for (i = 0; i < NCH(ch); i += 2) {
+        cch = CHILD(ch, i);
+        if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
+            char *str_ch = STR(CHILD(cch, 0));
+            if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
+                ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
+            } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
+                ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
+            } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
+                ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
+            }
+        }
+    }
+}
+#endif
+#endif /* future keyword */
+
+int
+PyParser_AddToken(parser_state *ps, int type, char *str,
                   int lineno, int col_offset,
                   int end_lineno, int end_col_offset,
                   int *expected_ret)
-{ 
-    int ilabel; 
-    int err; 
- 
-    D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); 
- 
-    /* Find out which label this token is */ 
-    ilabel = classify(ps, type, str); 
-    if (ilabel < 0) 
-        return E_SYNTAX; 
- 
-    /* Loop until the token is shifted or an error occurred */ 
-    for (;;) { 
-        /* Fetch the current dfa and state */ 
+{
+    int ilabel;
+    int err;
+
+    D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str));
+
+    /* Find out which label this token is */
+    ilabel = classify(ps, type, str);
+    if (ilabel < 0)
+        return E_SYNTAX;
+
+    /* Loop until the token is shifted or an error occurred */
+    for (;;) {
+        /* Fetch the current dfa and state */
         const dfa *d = ps->p_stack.s_top->s_dfa;
-        state *s = &d->d_state[ps->p_stack.s_top->s_state]; 
- 
-        D(printf(" DFA '%s', state %d:", 
-            d->d_name, ps->p_stack.s_top->s_state)); 
- 
-        /* Check accelerator */ 
-        if (s->s_lower <= ilabel && ilabel < s->s_upper) { 
-            int x = s->s_accel[ilabel - s->s_lower]; 
-            if (x != -1) { 
-                if (x & (1<<7)) { 
-                    /* Push non-terminal */ 
-                    int nt = (x >> 8) + NT_OFFSET; 
-                    int arrow = x & ((1<<7)-1); 
+        state *s = &d->d_state[ps->p_stack.s_top->s_state];
+
+        D(printf(" DFA '%s', state %d:",
+            d->d_name, ps->p_stack.s_top->s_state));
+
+        /* Check accelerator */
+        if (s->s_lower <= ilabel && ilabel < s->s_upper) {
+            int x = s->s_accel[ilabel - s->s_lower];
+            if (x != -1) {
+                if (x & (1<<7)) {
+                    /* Push non-terminal */
+                    int nt = (x >> 8) + NT_OFFSET;
+                    int arrow = x & ((1<<7)-1);
                     if (nt == func_body_suite && !(ps->p_flags & PyCF_TYPE_COMMENTS)) {
                         /* When parsing type comments is not requested,
                            we can provide better errors about bad indentation
@@ -269,195 +269,195 @@ PyParser_AddToken(parser_state *ps, int type, char *str,
                         nt = suite;
                     }
                     const dfa *d1 = PyGrammar_FindDFA(
-                        ps->p_grammar, nt); 
-                    if ((err = push(&ps->p_stack, nt, d1, 
+                        ps->p_grammar, nt);
+                    if ((err = push(&ps->p_stack, nt, d1,
                         arrow, lineno, col_offset,
                         end_lineno, end_col_offset)) > 0) {
-                        D(printf(" MemError: push\n")); 
-                        return err; 
-                    } 
+                        D(printf(" MemError: push\n"));
+                        return err;
+                    }
                     D(printf(" Push '%s'\n", d1->d_name));
-                    continue; 
-                } 
- 
-                /* Shift the token */ 
-                if ((err = shift(&ps->p_stack, type, str, 
+                    continue;
+                }
+
+                /* Shift the token */
+                if ((err = shift(&ps->p_stack, type, str,
                                 x, lineno, col_offset,
                                 end_lineno, end_col_offset)) > 0) {
-                    D(printf(" MemError: shift.\n")); 
-                    return err; 
-                } 
-                D(printf(" Shift.\n")); 
-                /* Pop while we are in an accept-only state */ 
-                while (s = &d->d_state 
-                                [ps->p_stack.s_top->s_state], 
-                    s->s_accept && s->s_narcs == 1) { 
-                    D(printf("  DFA '%s', state %d: " 
-                             "Direct pop.\n", 
-                             d->d_name, 
-                             ps->p_stack.s_top->s_state)); 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-#if 0 
-                    if (d->d_name[0] == 'i' && 
-                        strcmp(d->d_name, 
-                           "import_stmt") == 0) 
-                        future_hack(ps); 
-#endif 
-#endif 
-                    s_pop(&ps->p_stack); 
-                    if (s_empty(&ps->p_stack)) { 
-                        D(printf("  ACCEPT.\n")); 
-                        return E_DONE; 
-                    } 
-                    d = ps->p_stack.s_top->s_dfa; 
-                } 
-                return E_OK; 
-            } 
-        } 
- 
-        if (s->s_accept) { 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-#if 0 
-            if (d->d_name[0] == 'i' && 
-                strcmp(d->d_name, "import_stmt") == 0) 
-                future_hack(ps); 
-#endif 
-#endif 
-            /* Pop this dfa and try again */ 
-            s_pop(&ps->p_stack); 
-            D(printf(" Pop ...\n")); 
-            if (s_empty(&ps->p_stack)) { 
-                D(printf(" Error: bottom of stack.\n")); 
-                return E_SYNTAX; 
-            } 
-            continue; 
-        } 
- 
-        /* Stuck, report syntax error */ 
-        D(printf(" Error.\n")); 
-        if (expected_ret) { 
-            if (s->s_lower == s->s_upper - 1) { 
-                /* Only one possible expected token */ 
-                *expected_ret = ps->p_grammar-> 
-                    g_ll.ll_label[s->s_lower].lb_type; 
-            } 
-            else 
-                *expected_ret = -1; 
-        } 
-        return E_SYNTAX; 
-    } 
-} 
- 
- 
-#ifdef Py_DEBUG 
- 
-/* DEBUG OUTPUT */ 
- 
-void 
-dumptree(grammar *g, node *n) 
-{ 
-    int i; 
- 
-    if (n == NULL) 
-        printf("NIL"); 
-    else { 
-        label l; 
-        l.lb_type = TYPE(n); 
-        l.lb_str = STR(n); 
-        printf("%s", PyGrammar_LabelRepr(&l)); 
-        if (ISNONTERMINAL(TYPE(n))) { 
-            printf("("); 
-            for (i = 0; i < NCH(n); i++) { 
-                if (i > 0) 
-                    printf(","); 
-                dumptree(g, CHILD(n, i)); 
-            } 
-            printf(")"); 
-        } 
-    } 
-} 
- 
-void 
-showtree(grammar *g, node *n) 
-{ 
-    int i; 
- 
-    if (n == NULL) 
-        return; 
-    if (ISNONTERMINAL(TYPE(n))) { 
-        for (i = 0; i < NCH(n); i++) 
-            showtree(g, CHILD(n, i)); 
-    } 
-    else if (ISTERMINAL(TYPE(n))) { 
-        printf("%s", _PyParser_TokenNames[TYPE(n)]); 
-        if (TYPE(n) == NUMBER || TYPE(n) == NAME) 
-            printf("(%s)", STR(n)); 
-        printf(" "); 
-    } 
-    else 
-        printf("? "); 
-} 
- 
-void 
-printtree(parser_state *ps) 
-{ 
-    if (Py_DebugFlag) { 
-        printf("Parse tree:\n"); 
-        dumptree(ps->p_grammar, ps->p_tree); 
-        printf("\n"); 
-        printf("Tokens:\n"); 
-        showtree(ps->p_grammar, ps->p_tree); 
-        printf("\n"); 
-    } 
-    printf("Listing:\n"); 
-    PyNode_ListTree(ps->p_tree); 
-    printf("\n"); 
-} 
- 
-#endif /* Py_DEBUG */ 
- 
-/* 
- 
-Description 
------------ 
- 
-The parser's interface is different than usual: the function addtoken() 
-must be called for each token in the input.  This makes it possible to 
-turn it into an incremental parsing system later.  The parsing system 
-constructs a parse tree as it goes. 
- 
-A parsing rule is represented as a Deterministic Finite-state Automaton 
-(DFA).  A node in a DFA represents a state of the parser; an arc represents 
-a transition.  Transitions are either labeled with terminal symbols or 
-with non-terminals.  When the parser decides to follow an arc labeled 
-with a non-terminal, it is invoked recursively with the DFA representing 
-the parsing rule for that as its initial state; when that DFA accepts, 
-the parser that invoked it continues.  The parse tree constructed by the 
-recursively called parser is inserted as a child in the current parse tree. 
- 
-The DFA's can be constructed automatically from a more conventional 
-language description.  An extended LL(1) grammar (ELL(1)) is suitable. 
-Certain restrictions make the parser's life easier: rules that can produce 
-the empty string should be outlawed (there are other ways to put loops 
-or optional parts in the language).  To avoid the need to construct 
-FIRST sets, we can require that all but the last alternative of a rule 
-(really: arc going out of a DFA's state) must begin with a terminal 
-symbol. 
- 
-As an example, consider this grammar: 
- 
-expr:   term (OP term)* 
-term:   CONSTANT | '(' expr ')' 
- 
-The DFA corresponding to the rule for expr is: 
- 
-------->.---term-->.-------> 
-    ^          | 
-    |          | 
-    \----OP----/ 
- 
-The parse tree generated for the input a+b is: 
- 
-(expr: (term: (NAME: a)), (OP: +), (term: (NAME: b))) 
- 
-*/ 
+                    D(printf(" MemError: shift.\n"));
+                    return err;
+                }
+                D(printf(" Shift.\n"));
+                /* Pop while we are in an accept-only state */
+                while (s = &d->d_state
+                                [ps->p_stack.s_top->s_state],
+                    s->s_accept && s->s_narcs == 1) {
+                    D(printf("  DFA '%s', state %d: "
+                             "Direct pop.\n",
+                             d->d_name,
+                             ps->p_stack.s_top->s_state));
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+#if 0
+                    if (d->d_name[0] == 'i' &&
+                        strcmp(d->d_name,
+                           "import_stmt") == 0)
+                        future_hack(ps);
+#endif
+#endif
+                    s_pop(&ps->p_stack);
+                    if (s_empty(&ps->p_stack)) {
+                        D(printf("  ACCEPT.\n"));
+                        return E_DONE;
+                    }
+                    d = ps->p_stack.s_top->s_dfa;
+                }
+                return E_OK;
+            }
+        }
+
+        if (s->s_accept) {
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+#if 0
+            if (d->d_name[0] == 'i' &&
+                strcmp(d->d_name, "import_stmt") == 0)
+                future_hack(ps);
+#endif
+#endif
+            /* Pop this dfa and try again */
+            s_pop(&ps->p_stack);
+            D(printf(" Pop ...\n"));
+            if (s_empty(&ps->p_stack)) {
+                D(printf(" Error: bottom of stack.\n"));
+                return E_SYNTAX;
+            }
+            continue;
+        }
+
+        /* Stuck, report syntax error */
+        D(printf(" Error.\n"));
+        if (expected_ret) {
+            if (s->s_lower == s->s_upper - 1) {
+                /* Only one possible expected token */
+                *expected_ret = ps->p_grammar->
+                    g_ll.ll_label[s->s_lower].lb_type;
+            }
+            else
+                *expected_ret = -1;
+        }
+        return E_SYNTAX;
+    }
+}
+
+
+#ifdef Py_DEBUG
+
+/* DEBUG OUTPUT */
+
+void
+dumptree(grammar *g, node *n)
+{
+    int i;
+
+    if (n == NULL)
+        printf("NIL");
+    else {
+        label l;
+        l.lb_type = TYPE(n);
+        l.lb_str = STR(n);
+        printf("%s", PyGrammar_LabelRepr(&l));
+        if (ISNONTERMINAL(TYPE(n))) {
+            printf("(");
+            for (i = 0; i < NCH(n); i++) {
+                if (i > 0)
+                    printf(",");
+                dumptree(g, CHILD(n, i));
+            }
+            printf(")");
+        }
+    }
+}
+
+void
+showtree(grammar *g, node *n)
+{
+    int i;
+
+    if (n == NULL)
+        return;
+    if (ISNONTERMINAL(TYPE(n))) {
+        for (i = 0; i < NCH(n); i++)
+            showtree(g, CHILD(n, i));
+    }
+    else if (ISTERMINAL(TYPE(n))) {
+        printf("%s", _PyParser_TokenNames[TYPE(n)]);
+        if (TYPE(n) == NUMBER || TYPE(n) == NAME)
+            printf("(%s)", STR(n));
+        printf(" ");
+    }
+    else
+        printf("? ");
+}
+
+void
+printtree(parser_state *ps)
+{
+    if (Py_DebugFlag) {
+        printf("Parse tree:\n");
+        dumptree(ps->p_grammar, ps->p_tree);
+        printf("\n");
+        printf("Tokens:\n");
+        showtree(ps->p_grammar, ps->p_tree);
+        printf("\n");
+    }
+    printf("Listing:\n");
+    PyNode_ListTree(ps->p_tree);
+    printf("\n");
+}
+
+#endif /* Py_DEBUG */
+
+/*
+
+Description
+-----------
+
+The parser's interface is different than usual: the function addtoken()
+must be called for each token in the input.  This makes it possible to
+turn it into an incremental parsing system later.  The parsing system
+constructs a parse tree as it goes.
+
+A parsing rule is represented as a Deterministic Finite-state Automaton
+(DFA).  A node in a DFA represents a state of the parser; an arc represents
+a transition.  Transitions are either labeled with terminal symbols or
+with non-terminals.  When the parser decides to follow an arc labeled
+with a non-terminal, it is invoked recursively with the DFA representing
+the parsing rule for that as its initial state; when that DFA accepts,
+the parser that invoked it continues.  The parse tree constructed by the
+recursively called parser is inserted as a child in the current parse tree.
+
+The DFA's can be constructed automatically from a more conventional
+language description.  An extended LL(1) grammar (ELL(1)) is suitable.
+Certain restrictions make the parser's life easier: rules that can produce
+the empty string should be outlawed (there are other ways to put loops
+or optional parts in the language).  To avoid the need to construct
+FIRST sets, we can require that all but the last alternative of a rule
+(really: arc going out of a DFA's state) must begin with a terminal
+symbol.
+
+As an example, consider this grammar:
+
+expr:   term (OP term)*
+term:   CONSTANT | '(' expr ')'
+
+The DFA corresponding to the rule for expr is:
+
+------->.---term-->.------->
+    ^          |
+    |          |
+    \----OP----/
+
+The parse tree generated for the input a+b is:
+
+(expr: (term: (NAME: a)), (OP: +), (term: (NAME: b)))
+
+*/
diff --git a/contrib/tools/python3/src/Parser/parser.h b/contrib/tools/python3/src/Parser/parser.h
index 1098b17de78..b16075e7f29 100644
--- a/contrib/tools/python3/src/Parser/parser.h
+++ b/contrib/tools/python3/src/Parser/parser.h
@@ -1,49 +1,49 @@
-#ifndef Py_PARSER_H 
-#define Py_PARSER_H 
-#ifdef __cplusplus 
-extern "C" { 
-#endif 
- 
- 
-/* Parser interface */ 
- 
+#ifndef Py_PARSER_H
+#define Py_PARSER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Parser interface */
+
 #define MAXSTACK 1700
- 
-typedef struct { 
-    int              s_state;       /* State in current DFA */ 
+
+typedef struct {
+    int              s_state;       /* State in current DFA */
     const dfa       *s_dfa;         /* Current DFA */
-    struct _node    *s_parent;      /* Where to add next node */ 
-} stackentry; 
- 
-typedef struct { 
-    stackentry      *s_top;         /* Top entry */ 
-    stackentry       s_base[MAXSTACK];/* Array of stack entries */ 
-                                    /* NB The stack grows down */ 
-} stack; 
- 
-typedef struct { 
-    stack           p_stack;        /* Stack of parser states */ 
-    grammar         *p_grammar;     /* Grammar to use */ 
-    node            *p_tree;        /* Top of parse tree */ 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-    unsigned long   p_flags;        /* see co_flags in Include/code.h */ 
-#endif 
-} parser_state; 
- 
-parser_state *PyParser_New(grammar *g, int start); 
-void PyParser_Delete(parser_state *ps); 
+    struct _node    *s_parent;      /* Where to add next node */
+} stackentry;
+
+typedef struct {
+    stackentry      *s_top;         /* Top entry */
+    stackentry       s_base[MAXSTACK];/* Array of stack entries */
+                                    /* NB The stack grows down */
+} stack;
+
+typedef struct {
+    stack           p_stack;        /* Stack of parser states */
+    grammar         *p_grammar;     /* Grammar to use */
+    node            *p_tree;        /* Top of parse tree */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+    unsigned long   p_flags;        /* see co_flags in Include/code.h */
+#endif
+} parser_state;
+
+parser_state *PyParser_New(grammar *g, int start);
+void PyParser_Delete(parser_state *ps);
 int PyParser_AddToken(parser_state *ps, int type, char *str,
                       int lineno, int col_offset,
                       int end_lineno, int end_col_offset,
-                      int *expected_ret); 
-void PyGrammar_AddAccelerators(grammar *g); 
- 
+                      int *expected_ret);
+void PyGrammar_AddAccelerators(grammar *g);
+
 
 #define showtree _Py_showtree
 #define printtree _Py_printtree
 #define dumptree _Py_dumptree
 
-#ifdef __cplusplus 
-} 
-#endif 
-#endif /* !Py_PARSER_H */ 
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PARSER_H */
diff --git a/contrib/tools/python3/src/Parser/parsetok.c b/contrib/tools/python3/src/Parser/parsetok.c
index 51aefd1635e..1ecb2c4a16d 100644
--- a/contrib/tools/python3/src/Parser/parsetok.c
+++ b/contrib/tools/python3/src/Parser/parsetok.c
@@ -1,20 +1,20 @@
- 
-/* Parser-tokenizer link implementation */ 
- 
+
+/* Parser-tokenizer link implementation */
+
 #include "Python.h"
-#include "tokenizer.h" 
-#include "node.h" 
-#include "grammar.h" 
-#include "parser.h" 
-#include "parsetok.h" 
-#include "errcode.h" 
-#include "graminit.h" 
- 
- 
-/* Forward */ 
-static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); 
-static int initerr(perrdetail *err_ret, PyObject * filename); 
- 
+#include "tokenizer.h"
+#include "node.h"
+#include "grammar.h"
+#include "parser.h"
+#include "parsetok.h"
+#include "errcode.h"
+#include "graminit.h"
+
+
+/* Forward */
+static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
+static int initerr(perrdetail *err_ret, PyObject * filename);
+
 typedef struct {
     struct {
         int lineno;
@@ -60,225 +60,225 @@ growable_comment_array_deallocate(growable_comment_array *arr) {
     free(arr->items);
 }
 
-/* Parse input coming from a string.  Return error code, print some errors. */ 
-node * 
-PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) 
-{ 
-    return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); 
-} 
- 
-node * 
-PyParser_ParseStringFlags(const char *s, grammar *g, int start, 
-                          perrdetail *err_ret, int flags) 
-{ 
-    return PyParser_ParseStringFlagsFilename(s, NULL, 
-                                             g, start, err_ret, flags); 
-} 
- 
-node * 
-PyParser_ParseStringFlagsFilename(const char *s, const char *filename, 
-                          grammar *g, int start, 
-                          perrdetail *err_ret, int flags) 
-{ 
-    int iflags = flags; 
-    return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start, 
-                                               err_ret, &iflags); 
-} 
- 
-node * 
-PyParser_ParseStringObject(const char *s, PyObject *filename, 
-                           grammar *g, int start, 
-                           perrdetail *err_ret, int *flags) 
-{ 
-    struct tok_state *tok; 
-    int exec_input = start == file_input; 
- 
-    if (initerr(err_ret, filename) < 0) 
-        return NULL; 
- 
+/* Parse input coming from a string.  Return error code, print some errors. */
+node *
+PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
+{
+    return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
+}
+
+node *
+PyParser_ParseStringFlags(const char *s, grammar *g, int start,
+                          perrdetail *err_ret, int flags)
+{
+    return PyParser_ParseStringFlagsFilename(s, NULL,
+                                             g, start, err_ret, flags);
+}
+
+node *
+PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
+                          grammar *g, int start,
+                          perrdetail *err_ret, int flags)
+{
+    int iflags = flags;
+    return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
+                                               err_ret, &iflags);
+}
+
+node *
+PyParser_ParseStringObject(const char *s, PyObject *filename,
+                           grammar *g, int start,
+                           perrdetail *err_ret, int *flags)
+{
+    struct tok_state *tok;
+    int exec_input = start == file_input;
+
+    if (initerr(err_ret, filename) < 0)
+        return NULL;
+
     if (PySys_Audit("compile", "yO", s, err_ret->filename) < 0) {
         err_ret->error = E_ERROR;
         return NULL;
     }
 
-    if (*flags & PyPARSE_IGNORE_COOKIE) 
-        tok = PyTokenizer_FromUTF8(s, exec_input); 
-    else 
-        tok = PyTokenizer_FromString(s, exec_input); 
-    if (tok == NULL) { 
-        err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM; 
-        return NULL; 
-    } 
+    if (*flags & PyPARSE_IGNORE_COOKIE)
+        tok = PyTokenizer_FromUTF8(s, exec_input);
+    else
+        tok = PyTokenizer_FromString(s, exec_input);
+    if (tok == NULL) {
+        err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
+        return NULL;
+    }
     if (*flags & PyPARSE_TYPE_COMMENTS) {
         tok->type_comments = 1;
     }
- 
-    Py_INCREF(err_ret->filename); 
-    tok->filename = err_ret->filename; 
+
+    Py_INCREF(err_ret->filename);
+    tok->filename = err_ret->filename;
     if (*flags & PyPARSE_ASYNC_HACKS)
         tok->async_hacks = 1;
-    return parsetok(tok, g, start, err_ret, flags); 
-} 
- 
-node * 
-PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str, 
-                          grammar *g, int start, 
-                          perrdetail *err_ret, int *flags) 
-{ 
-    node *n; 
-    PyObject *filename = NULL; 
-    if (filename_str != NULL) { 
-        filename = PyUnicode_DecodeFSDefault(filename_str); 
-        if (filename == NULL) { 
-            err_ret->error = E_ERROR; 
-            return NULL; 
-        } 
-    } 
-    n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags); 
-    Py_XDECREF(filename); 
-    return n; 
-} 
- 
-/* Parse input coming from a file.  Return error code, print some errors. */ 
- 
-node * 
-PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start, 
-                   const char *ps1, const char *ps2, 
-                   perrdetail *err_ret) 
-{ 
-    return PyParser_ParseFileFlags(fp, filename, NULL, 
-                                   g, start, ps1, ps2, err_ret, 0); 
-} 
- 
-node * 
-PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc, 
-                        grammar *g, int start, 
-                        const char *ps1, const char *ps2, 
-                        perrdetail *err_ret, int flags) 
-{ 
-    int iflags = flags; 
-    return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1, 
-                                     ps2, err_ret, &iflags); 
-} 
- 
-node * 
-PyParser_ParseFileObject(FILE *fp, PyObject *filename, 
-                         const char *enc, grammar *g, int start, 
-                         const char *ps1, const char *ps2, 
-                         perrdetail *err_ret, int *flags) 
-{ 
-    struct tok_state *tok; 
- 
-    if (initerr(err_ret, filename) < 0) 
-        return NULL; 
- 
+    return parsetok(tok, g, start, err_ret, flags);
+}
+
+node *
+PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str,
+                          grammar *g, int start,
+                          perrdetail *err_ret, int *flags)
+{
+    node *n;
+    PyObject *filename = NULL;
+    if (filename_str != NULL) {
+        filename = PyUnicode_DecodeFSDefault(filename_str);
+        if (filename == NULL) {
+            err_ret->error = E_ERROR;
+            return NULL;
+        }
+    }
+    n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags);
+    Py_XDECREF(filename);
+    return n;
+}
+
+/* Parse input coming from a file.  Return error code, print some errors. */
+
+node *
+PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
+                   const char *ps1, const char *ps2,
+                   perrdetail *err_ret)
+{
+    return PyParser_ParseFileFlags(fp, filename, NULL,
+                                   g, start, ps1, ps2, err_ret, 0);
+}
+
+node *
+PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
+                        grammar *g, int start,
+                        const char *ps1, const char *ps2,
+                        perrdetail *err_ret, int flags)
+{
+    int iflags = flags;
+    return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
+                                     ps2, err_ret, &iflags);
+}
+
+node *
+PyParser_ParseFileObject(FILE *fp, PyObject *filename,
+                         const char *enc, grammar *g, int start,
+                         const char *ps1, const char *ps2,
+                         perrdetail *err_ret, int *flags)
+{
+    struct tok_state *tok;
+
+    if (initerr(err_ret, filename) < 0)
+        return NULL;
+
     if (PySys_Audit("compile", "OO", Py_None, err_ret->filename) < 0) {
         return NULL;
     }
 
-    if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) { 
-        err_ret->error = E_NOMEM; 
-        return NULL; 
-    } 
+    if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
+        err_ret->error = E_NOMEM;
+        return NULL;
+    }
     if (*flags & PyPARSE_TYPE_COMMENTS) {
         tok->type_comments = 1;
     }
-    Py_INCREF(err_ret->filename); 
-    tok->filename = err_ret->filename; 
-    return parsetok(tok, g, start, err_ret, flags); 
-} 
- 
-node * 
-PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, 
-                          const char *enc, grammar *g, int start, 
-                          const char *ps1, const char *ps2, 
-                          perrdetail *err_ret, int *flags) 
-{ 
-    node *n; 
-    PyObject *fileobj = NULL; 
-    if (filename != NULL) { 
-        fileobj = PyUnicode_DecodeFSDefault(filename); 
-        if (fileobj == NULL) { 
-            err_ret->error = E_ERROR; 
-            return NULL; 
-        } 
-    } 
-    n = PyParser_ParseFileObject(fp, fileobj, enc, g, 
-                                 start, ps1, ps2, err_ret, flags); 
-    Py_XDECREF(fileobj); 
-    return n; 
-} 
- 
-/* Parse input coming from the given tokenizer structure. 
-   Return error code. */ 
- 
-static node * 
-parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, 
-         int *flags) 
-{ 
-    parser_state *ps; 
-    node *n; 
-    int started = 0; 
+    Py_INCREF(err_ret->filename);
+    tok->filename = err_ret->filename;
+    return parsetok(tok, g, start, err_ret, flags);
+}
+
+node *
+PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
+                          const char *enc, grammar *g, int start,
+                          const char *ps1, const char *ps2,
+                          perrdetail *err_ret, int *flags)
+{
+    node *n;
+    PyObject *fileobj = NULL;
+    if (filename != NULL) {
+        fileobj = PyUnicode_DecodeFSDefault(filename);
+        if (fileobj == NULL) {
+            err_ret->error = E_ERROR;
+            return NULL;
+        }
+    }
+    n = PyParser_ParseFileObject(fp, fileobj, enc, g,
+                                 start, ps1, ps2, err_ret, flags);
+    Py_XDECREF(fileobj);
+    return n;
+}
+
+/* Parse input coming from the given tokenizer structure.
+   Return error code. */
+
+static node *
+parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
+         int *flags)
+{
+    parser_state *ps;
+    node *n;
+    int started = 0;
     int col_offset, end_col_offset;
     growable_comment_array type_ignores;
- 
+
     if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         PyTokenizer_Free(tok);
         return NULL;
     }
 
-    if ((ps = PyParser_New(g, start)) == NULL) { 
-        err_ret->error = E_NOMEM; 
+    if ((ps = PyParser_New(g, start)) == NULL) {
+        err_ret->error = E_NOMEM;
         growable_comment_array_deallocate(&type_ignores);
-        PyTokenizer_Free(tok); 
-        return NULL; 
-    } 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-    if (*flags & PyPARSE_BARRY_AS_BDFL) 
-        ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL; 
+        PyTokenizer_Free(tok);
+        return NULL;
+    }
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+    if (*flags & PyPARSE_BARRY_AS_BDFL)
+        ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
     if (*flags & PyPARSE_TYPE_COMMENTS)
         ps->p_flags |= PyCF_TYPE_COMMENTS;
-#endif 
- 
-    for (;;) { 
+#endif
+
+    for (;;) {
         const char *a, *b;
-        int type; 
-        size_t len; 
-        char *str; 
+        int type;
+        size_t len;
+        char *str;
         col_offset = -1;
         int lineno;
         const char *line_start;
- 
-        type = PyTokenizer_Get(tok, &a, &b); 
 
-        len = (a != NULL && b != NULL) ? b - a : 0; 
-        str = (char *) PyObject_MALLOC(len + 1); 
-        if (str == NULL) { 
-            err_ret->error = E_NOMEM; 
-            break; 
-        } 
-        if (len > 0) 
-            strncpy(str, a, len); 
-        str[len] = '\0'; 
- 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-        if (type == NOTEQUAL) { 
-            if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 
-                            strcmp(str, "!=")) { 
-                PyObject_FREE(str); 
-                err_ret->error = E_SYNTAX; 
-                break; 
-            } 
-            else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 
-                            strcmp(str, "<>")) { 
-                PyObject_FREE(str); 
-                err_ret->expected = NOTEQUAL; 
-                err_ret->error = E_SYNTAX; 
-                break; 
-            } 
-        } 
-#endif 
+        type = PyTokenizer_Get(tok, &a, &b);
+
+        len = (a != NULL && b != NULL) ? b - a : 0;
+        str = (char *) PyObject_MALLOC(len + 1);
+        if (str == NULL) {
+            err_ret->error = E_NOMEM;
+            break;
+        }
+        if (len > 0)
+            strncpy(str, a, len);
+        str[len] = '\0';
+
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+        if (type == NOTEQUAL) {
+            if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
+                            strcmp(str, "!=")) {
+                PyObject_FREE(str);
+                err_ret->error = E_SYNTAX;
+                break;
+            }
+            else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
+                            strcmp(str, "<>")) {
+                PyObject_FREE(str);
+                err_ret->expected = NOTEQUAL;
+                err_ret->error = E_SYNTAX;
+                break;
+            }
+        }
+#endif
 
         /* Nodes of type STRING, especially multi line strings
            must be handled differently in order to get both
@@ -288,12 +288,12 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         line_start = type == STRING ? tok->multi_line_start : tok->line_start;
         if (a != NULL && a >= line_start) {
             col_offset = Py_SAFE_DOWNCAST(a - line_start,
-                                          intptr_t, int); 
-        } 
-        else { 
-            col_offset = -1; 
-        } 
- 
+                                          intptr_t, int);
+        }
+        else {
+            col_offset = -1;
+        }
+
         if (b != NULL && b >= tok->line_start) {
             end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start,
                                               intptr_t, int);
@@ -331,25 +331,25 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             started = 1;
         }
 
-        if ((err_ret->error = 
-             PyParser_AddToken(ps, (int)type, str, 
+        if ((err_ret->error =
+             PyParser_AddToken(ps, (int)type, str,
                                lineno, col_offset, tok->lineno, end_col_offset,
-                               &(err_ret->expected))) != E_OK) { 
+                               &(err_ret->expected))) != E_OK) {
             if (tok->done == E_EOF && !ISWHITESPACE(type)) {
                 tok->done = E_SYNTAX;
             }
-            if (err_ret->error != E_DONE) { 
-                PyObject_FREE(str); 
-                err_ret->token = type; 
-            } 
-            break; 
-        } 
-    } 
- 
-    if (err_ret->error == E_DONE) { 
-        n = ps->p_tree; 
-        ps->p_tree = NULL; 
- 
+            if (err_ret->error != E_DONE) {
+                PyObject_FREE(str);
+                err_ret->token = type;
+            }
+            break;
+        }
+    }
+
+    if (err_ret->error == E_DONE) {
+        n = ps->p_tree;
+        ps->p_tree = NULL;
+
         if (n->n_type == file_input) {
             /* Put type_ignore nodes in the ENDMARKER of file_input. */
             int num;
@@ -374,113 +374,113 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             }
         }
 
-        /* Check that the source for a single input statement really 
-           is a single statement by looking at what is left in the 
-           buffer after parsing.  Trailing whitespace and comments 
-           are OK.  */ 
+        /* Check that the source for a single input statement really
+           is a single statement by looking at what is left in the
+           buffer after parsing.  Trailing whitespace and comments
+           are OK.  */
         if (err_ret->error == E_DONE && start == single_input) {
             const char *cur = tok->cur;
-            char c = *tok->cur; 
- 
-            for (;;) { 
-                while (c == ' ' || c == '\t' || c == '\n' || c == '\014') 
-                    c = *++cur; 
- 
-                if (!c) 
-                    break; 
- 
-                if (c != '#') { 
-                    err_ret->error = E_BADSINGLE; 
-                    PyNode_Free(n); 
-                    n = NULL; 
-                    break; 
-                } 
- 
-                /* Suck up comment. */ 
-                while (c && c != '\n') 
-                    c = *++cur; 
-            } 
-        } 
-    } 
-    else 
-        n = NULL; 
- 
+            char c = *tok->cur;
+
+            for (;;) {
+                while (c == ' ' || c == '\t' || c == '\n' || c == '\014')
+                    c = *++cur;
+
+                if (!c)
+                    break;
+
+                if (c != '#') {
+                    err_ret->error = E_BADSINGLE;
+                    PyNode_Free(n);
+                    n = NULL;
+                    break;
+                }
+
+                /* Suck up comment. */
+                while (c && c != '\n')
+                    c = *++cur;
+            }
+        }
+    }
+    else
+        n = NULL;
+
     growable_comment_array_deallocate(&type_ignores);
 
-#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 
-    *flags = ps->p_flags; 
-#endif 
-    PyParser_Delete(ps); 
- 
-    if (n == NULL) { 
-        if (tok->done == E_EOF) 
-            err_ret->error = E_EOF; 
-        err_ret->lineno = tok->lineno; 
-        if (tok->buf != NULL) { 
-            size_t len; 
-            assert(tok->cur - tok->buf < INT_MAX); 
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+    *flags = ps->p_flags;
+#endif
+    PyParser_Delete(ps);
+
+    if (n == NULL) {
+        if (tok->done == E_EOF)
+            err_ret->error = E_EOF;
+        err_ret->lineno = tok->lineno;
+        if (tok->buf != NULL) {
+            size_t len;
+            assert(tok->cur - tok->buf < INT_MAX);
             /* if we've managed to parse a token, point the offset to its start,
              * else use the current reading position of the tokenizer
              */
             err_ret->offset = col_offset != -1 ? col_offset + 1 : ((int)(tok->cur - tok->buf));
-            len = tok->inp - tok->buf; 
-            err_ret->text = (char *) PyObject_MALLOC(len + 1); 
-            if (err_ret->text != NULL) { 
-                if (len > 0) 
-                    strncpy(err_ret->text, tok->buf, len); 
-                err_ret->text[len] = '\0'; 
-            } 
-        } 
-    } else if (tok->encoding != NULL) { 
-        /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was 
-         * allocated using PyMem_ 
-         */ 
-        node* r = PyNode_New(encoding_decl); 
-        if (r) 
-            r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1); 
-        if (!r || !r->n_str) { 
-            err_ret->error = E_NOMEM; 
-            if (r) 
-                PyObject_FREE(r); 
-            n = NULL; 
-            goto done; 
-        } 
-        strcpy(r->n_str, tok->encoding); 
-        PyMem_FREE(tok->encoding); 
-        tok->encoding = NULL; 
-        r->n_nchildren = 1; 
-        r->n_child = n; 
-        n = r; 
-    } 
- 
-done: 
-    PyTokenizer_Free(tok); 
- 
+            len = tok->inp - tok->buf;
+            err_ret->text = (char *) PyObject_MALLOC(len + 1);
+            if (err_ret->text != NULL) {
+                if (len > 0)
+                    strncpy(err_ret->text, tok->buf, len);
+                err_ret->text[len] = '\0';
+            }
+        }
+    } else if (tok->encoding != NULL) {
+        /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
+         * allocated using PyMem_
+         */
+        node* r = PyNode_New(encoding_decl);
+        if (r)
+            r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
+        if (!r || !r->n_str) {
+            err_ret->error = E_NOMEM;
+            if (r)
+                PyObject_FREE(r);
+            n = NULL;
+            goto done;
+        }
+        strcpy(r->n_str, tok->encoding);
+        PyMem_FREE(tok->encoding);
+        tok->encoding = NULL;
+        r->n_nchildren = 1;
+        r->n_child = n;
+        n = r;
+    }
+
+done:
+    PyTokenizer_Free(tok);
+
     if (n != NULL) {
         _PyNode_FinalizeEndPos(n);
     }
-    return n; 
-} 
- 
-static int 
-initerr(perrdetail *err_ret, PyObject *filename) 
-{ 
-    err_ret->error = E_OK; 
-    err_ret->lineno = 0; 
-    err_ret->offset = 0; 
-    err_ret->text = NULL; 
-    err_ret->token = -1; 
-    err_ret->expected = -1; 
-    if (filename) { 
-        Py_INCREF(filename); 
-        err_ret->filename = filename; 
-    } 
-    else { 
-        err_ret->filename = PyUnicode_FromString("<string>"); 
-        if (err_ret->filename == NULL) { 
-            err_ret->error = E_ERROR; 
-            return -1; 
-        } 
-    } 
-    return 0; 
-} 
+    return n;
+}
+
+static int
+initerr(perrdetail *err_ret, PyObject *filename)
+{
+    err_ret->error = E_OK;
+    err_ret->lineno = 0;
+    err_ret->offset = 0;
+    err_ret->text = NULL;
+    err_ret->token = -1;
+    err_ret->expected = -1;
+    if (filename) {
+        Py_INCREF(filename);
+        err_ret->filename = filename;
+    }
+    else {
+        err_ret->filename = PyUnicode_FromString("<string>");
+        if (err_ret->filename == NULL) {
+            err_ret->error = E_ERROR;
+            return -1;
+        }
+    }
+    return 0;
+}
diff --git a/contrib/tools/python3/src/Parser/tokenizer.c b/contrib/tools/python3/src/Parser/tokenizer.c
index caf3d3a9ac1..41bfdb853c7 100644
--- a/contrib/tools/python3/src/Parser/tokenizer.c
+++ b/contrib/tools/python3/src/Parser/tokenizer.c
@@ -1,1045 +1,1045 @@
- 
-/* Tokenizer implementation */ 
- 
+
+/* Tokenizer implementation */
+
 #define PY_SSIZE_T_CLEAN
-#include "Python.h" 
- 
-#include <ctype.h> 
-#include <assert.h> 
- 
-#include "tokenizer.h" 
-#include "errcode.h" 
- 
-#include "unicodeobject.h" 
-#include "bytesobject.h" 
-#include "fileobject.h" 
-#include "codecs.h" 
-#include "abstract.h" 
- 
-/* Alternate tab spacing */ 
-#define ALTTABSIZE 1 
- 
-#define is_potential_identifier_start(c) (\ 
-              (c >= 'a' && c <= 'z')\ 
-               || (c >= 'A' && c <= 'Z')\ 
-               || c == '_'\ 
-               || (c >= 128)) 
- 
-#define is_potential_identifier_char(c) (\ 
-              (c >= 'a' && c <= 'z')\ 
-               || (c >= 'A' && c <= 'Z')\ 
-               || (c >= '0' && c <= '9')\ 
-               || c == '_'\ 
-               || (c >= 128)) 
- 
- 
-/* Don't ever change this -- it would break the portability of Python code */ 
-#define TABSIZE 8 
- 
-/* Forward */ 
-static struct tok_state *tok_new(void); 
-static int tok_nextc(struct tok_state *tok); 
-static void tok_backup(struct tok_state *tok, int c); 
- 
- 
+#include "Python.h"
+
+#include <ctype.h>
+#include <assert.h>
+
+#include "tokenizer.h"
+#include "errcode.h"
+
+#include "unicodeobject.h"
+#include "bytesobject.h"
+#include "fileobject.h"
+#include "codecs.h"
+#include "abstract.h"
+
+/* Alternate tab spacing */
+#define ALTTABSIZE 1
+
+#define is_potential_identifier_start(c) (\
+              (c >= 'a' && c <= 'z')\
+               || (c >= 'A' && c <= 'Z')\
+               || c == '_'\
+               || (c >= 128))
+
+#define is_potential_identifier_char(c) (\
+              (c >= 'a' && c <= 'z')\
+               || (c >= 'A' && c <= 'Z')\
+               || (c >= '0' && c <= '9')\
+               || c == '_'\
+               || (c >= 128))
+
+
+/* Don't ever change this -- it would break the portability of Python code */
+#define TABSIZE 8
+
+/* Forward */
+static struct tok_state *tok_new(void);
+static int tok_nextc(struct tok_state *tok);
+static void tok_backup(struct tok_state *tok, int c);
+
+
 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
    tokenizing. */
 static const char* type_comment_prefix = "# type: ";
- 
-/* Create and initialize a new tok_state structure */ 
- 
-static struct tok_state * 
-tok_new(void) 
-{ 
-    struct tok_state *tok = (struct tok_state *)PyMem_MALLOC( 
-                                            sizeof(struct tok_state)); 
-    if (tok == NULL) 
-        return NULL; 
+
+/* Create and initialize a new tok_state structure */
+
+static struct tok_state *
+tok_new(void)
+{
+    struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
+                                            sizeof(struct tok_state));
+    if (tok == NULL)
+        return NULL;
     tok->buf = tok->cur = tok->inp = NULL;
     tok->start = NULL;
     tok->end = NULL;
-    tok->done = E_OK; 
-    tok->fp = NULL; 
-    tok->input = NULL; 
-    tok->tabsize = TABSIZE; 
-    tok->indent = 0; 
-    tok->indstack[0] = 0; 
- 
-    tok->atbol = 1; 
-    tok->pendin = 0; 
-    tok->prompt = tok->nextprompt = NULL; 
-    tok->lineno = 0; 
-    tok->level = 0; 
-    tok->altindstack[0] = 0; 
-    tok->decoding_state = STATE_INIT; 
-    tok->decoding_erred = 0; 
-    tok->read_coding_spec = 0; 
-    tok->enc = NULL; 
-    tok->encoding = NULL; 
-    tok->cont_line = 0; 
-    tok->filename = NULL; 
-    tok->decoding_readline = NULL; 
-    tok->decoding_buffer = NULL; 
+    tok->done = E_OK;
+    tok->fp = NULL;
+    tok->input = NULL;
+    tok->tabsize = TABSIZE;
+    tok->indent = 0;
+    tok->indstack[0] = 0;
+
+    tok->atbol = 1;
+    tok->pendin = 0;
+    tok->prompt = tok->nextprompt = NULL;
+    tok->lineno = 0;
+    tok->level = 0;
+    tok->altindstack[0] = 0;
+    tok->decoding_state = STATE_INIT;
+    tok->decoding_erred = 0;
+    tok->read_coding_spec = 0;
+    tok->enc = NULL;
+    tok->encoding = NULL;
+    tok->cont_line = 0;
+    tok->filename = NULL;
+    tok->decoding_readline = NULL;
+    tok->decoding_buffer = NULL;
     tok->type_comments = 0;
- 
+
     tok->async_hacks = 0;
     tok->async_def = 0;
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
 
-    return tok; 
-} 
- 
-static char * 
-new_string(const char *s, Py_ssize_t len, struct tok_state *tok) 
-{ 
-    char* result = (char *)PyMem_MALLOC(len + 1); 
-    if (!result) { 
-        tok->done = E_NOMEM; 
-        return NULL; 
-    } 
-    memcpy(result, s, len); 
-    result[len] = '\0'; 
-    return result; 
-} 
- 
-static char * 
-error_ret(struct tok_state *tok) /* XXX */ 
-{ 
-    tok->decoding_erred = 1; 
-    if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ 
-        PyMem_FREE(tok->buf); 
+    return tok;
+}
+
+static char *
+new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
+{
+    char* result = (char *)PyMem_MALLOC(len + 1);
+    if (!result) {
+        tok->done = E_NOMEM;
+        return NULL;
+    }
+    memcpy(result, s, len);
+    result[len] = '\0';
+    return result;
+}
+
+static char *
+error_ret(struct tok_state *tok) /* XXX */
+{
+    tok->decoding_erred = 1;
+    if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
+        PyMem_FREE(tok->buf);
     tok->buf = tok->cur = tok->inp = NULL;
     tok->start = NULL;
     tok->end = NULL;
-    tok->done = E_DECODE; 
-    return NULL;                /* as if it were EOF */ 
-} 
- 
- 
-static const char * 
-get_normal_name(const char *s)  /* for utf-8 and latin-1 */ 
-{ 
-    char buf[13]; 
-    int i; 
-    for (i = 0; i < 12; i++) { 
-        int c = s[i]; 
-        if (c == '\0') 
-            break; 
-        else if (c == '_') 
-            buf[i] = '-'; 
-        else 
-            buf[i] = tolower(c); 
-    } 
-    buf[i] = '\0'; 
-    if (strcmp(buf, "utf-8") == 0 || 
-        strncmp(buf, "utf-8-", 6) == 0) 
-        return "utf-8"; 
-    else if (strcmp(buf, "latin-1") == 0 || 
-             strcmp(buf, "iso-8859-1") == 0 || 
-             strcmp(buf, "iso-latin-1") == 0 || 
-             strncmp(buf, "latin-1-", 8) == 0 || 
-             strncmp(buf, "iso-8859-1-", 11) == 0 || 
-             strncmp(buf, "iso-latin-1-", 12) == 0) 
-        return "iso-8859-1"; 
-    else 
-        return s; 
-} 
- 
-/* Return the coding spec in S, or NULL if none is found.  */ 
- 
-static int 
-get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok) 
-{ 
-    Py_ssize_t i; 
-    *spec = NULL; 
-    /* Coding spec must be in a comment, and that comment must be 
-     * the only statement on the source code line. */ 
-    for (i = 0; i < size - 6; i++) { 
-        if (s[i] == '#') 
-            break; 
-        if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014') 
-            return 1; 
-    } 
-    for (; i < size - 6; i++) { /* XXX inefficient search */ 
-        const char* t = s + i; 
-        if (strncmp(t, "coding", 6) == 0) { 
-            const char* begin = NULL; 
-            t += 6; 
-            if (t[0] != ':' && t[0] != '=') 
-                continue; 
-            do { 
-                t++; 
-            } while (t[0] == '\x20' || t[0] == '\t'); 
- 
-            begin = t; 
-            while (Py_ISALNUM(t[0]) || 
-                   t[0] == '-' || t[0] == '_' || t[0] == '.') 
-                t++; 
- 
-            if (begin < t) { 
-                char* r = new_string(begin, t - begin, tok); 
-                const char* q; 
-                if (!r) 
-                    return 0; 
-                q = get_normal_name(r); 
-                if (r != q) { 
-                    PyMem_FREE(r); 
-                    r = new_string(q, strlen(q), tok); 
-                    if (!r) 
-                        return 0; 
-                } 
-                *spec = r; 
-                break; 
-            } 
-        } 
-    } 
-    return 1; 
-} 
- 
-/* Check whether the line contains a coding spec. If it does, 
-   invoke the set_readline function for the new encoding. 
-   This function receives the tok_state and the new encoding. 
-   Return 1 on success, 0 on failure.  */ 
- 
-static int 
-check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, 
-                  int set_readline(struct tok_state *, const char *)) 
-{ 
-    char *cs; 
-    int r = 1; 
- 
-    if (tok->cont_line) { 
-        /* It's a continuation line, so it can't be a coding spec. */ 
-        tok->read_coding_spec = 1; 
-        return 1; 
-    } 
-    if (!get_coding_spec(line, &cs, size, tok)) 
-        return 0; 
-    if (!cs) { 
-        Py_ssize_t i; 
-        for (i = 0; i < size; i++) { 
-            if (line[i] == '#' || line[i] == '\n' || line[i] == '\r') 
-                break; 
-            if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') { 
-                /* Stop checking coding spec after a line containing 
-                 * anything except a comment. */ 
-                tok->read_coding_spec = 1; 
-                break; 
-            } 
-        } 
-        return 1; 
-    } 
-    tok->read_coding_spec = 1; 
-    if (tok->encoding == NULL) { 
-        assert(tok->decoding_state == STATE_RAW); 
-        if (strcmp(cs, "utf-8") == 0) { 
-            tok->encoding = cs; 
-        } else { 
-            r = set_readline(tok, cs); 
-            if (r) { 
-                tok->encoding = cs; 
-                tok->decoding_state = STATE_NORMAL; 
-            } 
-            else { 
-                PyErr_Format(PyExc_SyntaxError, 
-                             "encoding problem: %s", cs); 
-                PyMem_FREE(cs); 
-            } 
-        } 
-    } else {                /* then, compare cs with BOM */ 
-        r = (strcmp(tok->encoding, cs) == 0); 
-        if (!r) 
-            PyErr_Format(PyExc_SyntaxError, 
-                         "encoding problem: %s with BOM", cs); 
-        PyMem_FREE(cs); 
-    } 
-    return r; 
-} 
- 
-/* See whether the file starts with a BOM. If it does, 
-   invoke the set_readline function with the new encoding. 
-   Return 1 on success, 0 on failure.  */ 
- 
-static int 
-check_bom(int get_char(struct tok_state *), 
-          void unget_char(int, struct tok_state *), 
-          int set_readline(struct tok_state *, const char *), 
-          struct tok_state *tok) 
-{ 
-    int ch1, ch2, ch3; 
-    ch1 = get_char(tok); 
-    tok->decoding_state = STATE_RAW; 
-    if (ch1 == EOF) { 
-        return 1; 
-    } else if (ch1 == 0xEF) { 
-        ch2 = get_char(tok); 
-        if (ch2 != 0xBB) { 
-            unget_char(ch2, tok); 
-            unget_char(ch1, tok); 
-            return 1; 
-        } 
-        ch3 = get_char(tok); 
-        if (ch3 != 0xBF) { 
-            unget_char(ch3, tok); 
-            unget_char(ch2, tok); 
-            unget_char(ch1, tok); 
-            return 1; 
-        } 
-#if 0 
-    /* Disable support for UTF-16 BOMs until a decision 
-       is made whether this needs to be supported.  */ 
-    } else if (ch1 == 0xFE) { 
-        ch2 = get_char(tok); 
-        if (ch2 != 0xFF) { 
-            unget_char(ch2, tok); 
-            unget_char(ch1, tok); 
-            return 1; 
-        } 
-        if (!set_readline(tok, "utf-16-be")) 
-            return 0; 
-        tok->decoding_state = STATE_NORMAL; 
-    } else if (ch1 == 0xFF) { 
-        ch2 = get_char(tok); 
-        if (ch2 != 0xFE) { 
-            unget_char(ch2, tok); 
-            unget_char(ch1, tok); 
-            return 1; 
-        } 
-        if (!set_readline(tok, "utf-16-le")) 
-            return 0; 
-        tok->decoding_state = STATE_NORMAL; 
-#endif 
-    } else { 
-        unget_char(ch1, tok); 
-        return 1; 
-    } 
-    if (tok->encoding != NULL) 
-        PyMem_FREE(tok->encoding); 
-    tok->encoding = new_string("utf-8", 5, tok); 
-    if (!tok->encoding) 
-        return 0; 
-    /* No need to set_readline: input is already utf-8 */ 
-    return 1; 
-} 
- 
-/* Read a line of text from TOK into S, using the stream in TOK. 
-   Return NULL on failure, else S. 
- 
-   On entry, tok->decoding_buffer will be one of: 
-     1) NULL: need to call tok->decoding_readline to get a new line 
-     2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and 
-       stored the result in tok->decoding_buffer 
-     3) PyByteArrayObject *: previous call to fp_readl did not have enough room 
-       (in the s buffer) to copy entire contents of the line read 
-       by tok->decoding_readline.  tok->decoding_buffer has the overflow. 
-       In this case, fp_readl is called in a loop (with an expanded buffer) 
-       until the buffer ends with a '\n' (or until the end of the file is 
-       reached): see tok_nextc and its calls to decoding_fgets. 
-*/ 
- 
-static char * 
-fp_readl(char *s, int size, struct tok_state *tok) 
-{ 
-    PyObject* bufobj; 
-    const char *buf; 
-    Py_ssize_t buflen; 
- 
-    /* Ask for one less byte so we can terminate it */ 
-    assert(size > 0); 
-    size--; 
- 
-    if (tok->decoding_buffer) { 
-        bufobj = tok->decoding_buffer; 
-        Py_INCREF(bufobj); 
-    } 
-    else 
-    { 
-        bufobj = _PyObject_CallNoArg(tok->decoding_readline); 
-        if (bufobj == NULL) 
-            goto error; 
-    } 
-    if (PyUnicode_CheckExact(bufobj)) 
-    { 
-        buf = PyUnicode_AsUTF8AndSize(bufobj, &buflen); 
-        if (buf == NULL) { 
-            goto error; 
-        } 
-    } 
-    else 
-    { 
-        buf = PyByteArray_AsString(bufobj); 
-        if (buf == NULL) { 
-            goto error; 
-        } 
-        buflen = PyByteArray_GET_SIZE(bufobj); 
-    } 
- 
-    Py_XDECREF(tok->decoding_buffer); 
-    if (buflen > size) { 
-        /* Too many chars, the rest goes into tok->decoding_buffer */ 
-        tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size, 
-                                                         buflen-size); 
-        if (tok->decoding_buffer == NULL) 
-            goto error; 
-        buflen = size; 
-    } 
-    else 
-        tok->decoding_buffer = NULL; 
- 
-    memcpy(s, buf, buflen); 
-    s[buflen] = '\0'; 
-    if (buflen == 0) /* EOF */ 
-        s = NULL; 
-    Py_DECREF(bufobj); 
-    return s; 
- 
-error: 
-    Py_XDECREF(bufobj); 
-    return error_ret(tok); 
-} 
- 
-/* Set the readline function for TOK to a StreamReader's 
-   readline function. The StreamReader is named ENC. 
- 
-   This function is called from check_bom and check_coding_spec. 
- 
-   ENC is usually identical to the future value of tok->encoding, 
-   except for the (currently unsupported) case of UTF-16. 
- 
-   Return 1 on success, 0 on failure. */ 
- 
-static int 
-fp_setreadl(struct tok_state *tok, const char* enc) 
-{ 
-    PyObject *readline, *io, *stream; 
-    _Py_IDENTIFIER(open); 
-    _Py_IDENTIFIER(readline); 
-    int fd; 
-    long pos; 
- 
-    fd = fileno(tok->fp); 
-    /* Due to buffering the file offset for fd can be different from the file 
-     * position of tok->fp.  If tok->fp was opened in text mode on Windows, 
-     * its file position counts CRLF as one char and can't be directly mapped 
-     * to the file offset for fd.  Instead we step back one byte and read to 
-     * the end of line.*/ 
-    pos = ftell(tok->fp); 
-    if (pos == -1 || 
-        lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) { 
-        PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL); 
-        return 0; 
-    } 
- 
-    io = PyImport_ImportModuleNoBlock("io"); 
-    if (io == NULL) 
-        return 0; 
- 
-    stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO", 
-                    fd, "r", -1, enc, Py_None, Py_None, Py_False); 
-    Py_DECREF(io); 
-    if (stream == NULL) 
-        return 0; 
- 
-    readline = _PyObject_GetAttrId(stream, &PyId_readline); 
-    Py_DECREF(stream); 
-    if (readline == NULL) 
-        return 0; 
-    Py_XSETREF(tok->decoding_readline, readline); 
- 
-    if (pos > 0) { 
-        PyObject *bufobj = _PyObject_CallNoArg(readline); 
-        if (bufobj == NULL) 
-            return 0; 
-        Py_DECREF(bufobj); 
-    } 
- 
-    return 1; 
-} 
- 
-/* Fetch the next byte from TOK. */ 
- 
-static int fp_getc(struct tok_state *tok) { 
-    return getc(tok->fp); 
-} 
- 
-/* Unfetch the last byte back into TOK.  */ 
- 
-static void fp_ungetc(int c, struct tok_state *tok) { 
-    ungetc(c, tok->fp); 
-} 
- 
-/* Check whether the characters at s start a valid 
-   UTF-8 sequence. Return the number of characters forming 
-   the sequence if yes, 0 if not.  */ 
-static int valid_utf8(const unsigned char* s) 
-{ 
-    int expected = 0; 
-    int length; 
-    if (*s < 0x80) 
-        /* single-byte code */ 
-        return 1; 
-    if (*s < 0xc0) 
-        /* following byte */ 
-        return 0; 
-    if (*s < 0xE0) 
-        expected = 1; 
-    else if (*s < 0xF0) 
-        expected = 2; 
-    else if (*s < 0xF8) 
-        expected = 3; 
-    else 
-        return 0; 
-    length = expected + 1; 
-    for (; expected; expected--) 
-        if (s[expected] < 0x80 || s[expected] >= 0xC0) 
-            return 0; 
-    return length; 
-} 
- 
-/* Read a line of input from TOK. Determine encoding 
-   if necessary.  */ 
- 
-static char * 
-decoding_fgets(char *s, int size, struct tok_state *tok) 
-{ 
-    char *line = NULL; 
-    int badchar = 0; 
-    for (;;) { 
-        if (tok->decoding_state == STATE_NORMAL) { 
-            /* We already have a codec associated with 
-               this input. */ 
-            line = fp_readl(s, size, tok); 
-            break; 
-        } else if (tok->decoding_state == STATE_RAW) { 
-            /* We want a 'raw' read. */ 
-            line = Py_UniversalNewlineFgets(s, size, 
-                                            tok->fp, NULL); 
-            break; 
-        } else { 
-            /* We have not yet determined the encoding. 
-               If an encoding is found, use the file-pointer 
-               reader functions from now on. */ 
-            if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) 
-                return error_ret(tok); 
-            assert(tok->decoding_state != STATE_INIT); 
-        } 
-    } 
-    if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) { 
-        if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) { 
-            return error_ret(tok); 
-        } 
-    } 
-    /* The default encoding is UTF-8, so make sure we don't have any 
-       non-UTF-8 sequences in it. */ 
-    if (line && !tok->encoding) { 
-        unsigned char *c; 
-        int length; 
-        for (c = (unsigned char *)line; *c; c += length) 
-            if (!(length = valid_utf8(c))) { 
-                badchar = *c; 
-                break; 
-            } 
-    } 
-    if (badchar) { 
-        /* Need to add 1 to the line number, since this line 
-           has not been counted, yet.  */ 
-        PyErr_Format(PyExc_SyntaxError, 
-                "Non-UTF-8 code starting with '\\x%.2x' " 
-                "in file %U on line %i, " 
-                "but no encoding declared; " 
+    tok->done = E_DECODE;
+    return NULL;                /* as if it were EOF */
+}
+
+
+static const char *
+get_normal_name(const char *s)  /* for utf-8 and latin-1 */
+{
+    char buf[13];
+    int i;
+    for (i = 0; i < 12; i++) {
+        int c = s[i];
+        if (c == '\0')
+            break;
+        else if (c == '_')
+            buf[i] = '-';
+        else
+            buf[i] = tolower(c);
+    }
+    buf[i] = '\0';
+    if (strcmp(buf, "utf-8") == 0 ||
+        strncmp(buf, "utf-8-", 6) == 0)
+        return "utf-8";
+    else if (strcmp(buf, "latin-1") == 0 ||
+             strcmp(buf, "iso-8859-1") == 0 ||
+             strcmp(buf, "iso-latin-1") == 0 ||
+             strncmp(buf, "latin-1-", 8) == 0 ||
+             strncmp(buf, "iso-8859-1-", 11) == 0 ||
+             strncmp(buf, "iso-latin-1-", 12) == 0)
+        return "iso-8859-1";
+    else
+        return s;
+}
+
+/* Return the coding spec in S, or NULL if none is found.  */
+
+static int
+get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)
+{
+    Py_ssize_t i;
+    *spec = NULL;
+    /* Coding spec must be in a comment, and that comment must be
+     * the only statement on the source code line. */
+    for (i = 0; i < size - 6; i++) {
+        if (s[i] == '#')
+            break;
+        if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
+            return 1;
+    }
+    for (; i < size - 6; i++) { /* XXX inefficient search */
+        const char* t = s + i;
+        if (strncmp(t, "coding", 6) == 0) {
+            const char* begin = NULL;
+            t += 6;
+            if (t[0] != ':' && t[0] != '=')
+                continue;
+            do {
+                t++;
+            } while (t[0] == '\x20' || t[0] == '\t');
+
+            begin = t;
+            while (Py_ISALNUM(t[0]) ||
+                   t[0] == '-' || t[0] == '_' || t[0] == '.')
+                t++;
+
+            if (begin < t) {
+                char* r = new_string(begin, t - begin, tok);
+                const char* q;
+                if (!r)
+                    return 0;
+                q = get_normal_name(r);
+                if (r != q) {
+                    PyMem_FREE(r);
+                    r = new_string(q, strlen(q), tok);
+                    if (!r)
+                        return 0;
+                }
+                *spec = r;
+                break;
+            }
+        }
+    }
+    return 1;
+}
+
+/* Check whether the line contains a coding spec. If it does,
+   invoke the set_readline function for the new encoding.
+   This function receives the tok_state and the new encoding.
+   Return 1 on success, 0 on failure.  */
+
+static int
+check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
+                  int set_readline(struct tok_state *, const char *))
+{
+    char *cs;
+    int r = 1;
+
+    if (tok->cont_line) {
+        /* It's a continuation line, so it can't be a coding spec. */
+        tok->read_coding_spec = 1;
+        return 1;
+    }
+    if (!get_coding_spec(line, &cs, size, tok))
+        return 0;
+    if (!cs) {
+        Py_ssize_t i;
+        for (i = 0; i < size; i++) {
+            if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
+                break;
+            if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
+                /* Stop checking coding spec after a line containing
+                 * anything except a comment. */
+                tok->read_coding_spec = 1;
+                break;
+            }
+        }
+        return 1;
+    }
+    tok->read_coding_spec = 1;
+    if (tok->encoding == NULL) {
+        assert(tok->decoding_state == STATE_RAW);
+        if (strcmp(cs, "utf-8") == 0) {
+            tok->encoding = cs;
+        } else {
+            r = set_readline(tok, cs);
+            if (r) {
+                tok->encoding = cs;
+                tok->decoding_state = STATE_NORMAL;
+            }
+            else {
+                PyErr_Format(PyExc_SyntaxError,
+                             "encoding problem: %s", cs);
+                PyMem_FREE(cs);
+            }
+        }
+    } else {                /* then, compare cs with BOM */
+        r = (strcmp(tok->encoding, cs) == 0);
+        if (!r)
+            PyErr_Format(PyExc_SyntaxError,
+                         "encoding problem: %s with BOM", cs);
+        PyMem_FREE(cs);
+    }
+    return r;
+}
+
+/* See whether the file starts with a BOM. If it does,
+   invoke the set_readline function with the new encoding.
+   Return 1 on success, 0 on failure.  */
+
+static int
+check_bom(int get_char(struct tok_state *),
+          void unget_char(int, struct tok_state *),
+          int set_readline(struct tok_state *, const char *),
+          struct tok_state *tok)
+{
+    int ch1, ch2, ch3;
+    ch1 = get_char(tok);
+    tok->decoding_state = STATE_RAW;
+    if (ch1 == EOF) {
+        return 1;
+    } else if (ch1 == 0xEF) {
+        ch2 = get_char(tok);
+        if (ch2 != 0xBB) {
+            unget_char(ch2, tok);
+            unget_char(ch1, tok);
+            return 1;
+        }
+        ch3 = get_char(tok);
+        if (ch3 != 0xBF) {
+            unget_char(ch3, tok);
+            unget_char(ch2, tok);
+            unget_char(ch1, tok);
+            return 1;
+        }
+#if 0
+    /* Disable support for UTF-16 BOMs until a decision
+       is made whether this needs to be supported.  */
+    } else if (ch1 == 0xFE) {
+        ch2 = get_char(tok);
+        if (ch2 != 0xFF) {
+            unget_char(ch2, tok);
+            unget_char(ch1, tok);
+            return 1;
+        }
+        if (!set_readline(tok, "utf-16-be"))
+            return 0;
+        tok->decoding_state = STATE_NORMAL;
+    } else if (ch1 == 0xFF) {
+        ch2 = get_char(tok);
+        if (ch2 != 0xFE) {
+            unget_char(ch2, tok);
+            unget_char(ch1, tok);
+            return 1;
+        }
+        if (!set_readline(tok, "utf-16-le"))
+            return 0;
+        tok->decoding_state = STATE_NORMAL;
+#endif
+    } else {
+        unget_char(ch1, tok);
+        return 1;
+    }
+    if (tok->encoding != NULL)
+        PyMem_FREE(tok->encoding);
+    tok->encoding = new_string("utf-8", 5, tok);
+    if (!tok->encoding)
+        return 0;
+    /* No need to set_readline: input is already utf-8 */
+    return 1;
+}
+
+/* Read a line of text from TOK into S, using the stream in TOK.
+   Return NULL on failure, else S.
+
+   On entry, tok->decoding_buffer will be one of:
+     1) NULL: need to call tok->decoding_readline to get a new line
+     2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
+       stored the result in tok->decoding_buffer
+     3) PyByteArrayObject *: previous call to fp_readl did not have enough room
+       (in the s buffer) to copy entire contents of the line read
+       by tok->decoding_readline.  tok->decoding_buffer has the overflow.
+       In this case, fp_readl is called in a loop (with an expanded buffer)
+       until the buffer ends with a '\n' (or until the end of the file is
+       reached): see tok_nextc and its calls to decoding_fgets.
+*/
+
+static char *
+fp_readl(char *s, int size, struct tok_state *tok)
+{
+    PyObject* bufobj;
+    const char *buf;
+    Py_ssize_t buflen;
+
+    /* Ask for one less byte so we can terminate it */
+    assert(size > 0);
+    size--;
+
+    if (tok->decoding_buffer) {
+        bufobj = tok->decoding_buffer;
+        Py_INCREF(bufobj);
+    }
+    else
+    {
+        bufobj = _PyObject_CallNoArg(tok->decoding_readline);
+        if (bufobj == NULL)
+            goto error;
+    }
+    if (PyUnicode_CheckExact(bufobj))
+    {
+        buf = PyUnicode_AsUTF8AndSize(bufobj, &buflen);
+        if (buf == NULL) {
+            goto error;
+        }
+    }
+    else
+    {
+        buf = PyByteArray_AsString(bufobj);
+        if (buf == NULL) {
+            goto error;
+        }
+        buflen = PyByteArray_GET_SIZE(bufobj);
+    }
+
+    Py_XDECREF(tok->decoding_buffer);
+    if (buflen > size) {
+        /* Too many chars, the rest goes into tok->decoding_buffer */
+        tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
+                                                         buflen-size);
+        if (tok->decoding_buffer == NULL)
+            goto error;
+        buflen = size;
+    }
+    else
+        tok->decoding_buffer = NULL;
+
+    memcpy(s, buf, buflen);
+    s[buflen] = '\0';
+    if (buflen == 0) /* EOF */
+        s = NULL;
+    Py_DECREF(bufobj);
+    return s;
+
+error:
+    Py_XDECREF(bufobj);
+    return error_ret(tok);
+}
+
+/* Set the readline function for TOK to a StreamReader's
+   readline function. The StreamReader is named ENC.
+
+   This function is called from check_bom and check_coding_spec.
+
+   ENC is usually identical to the future value of tok->encoding,
+   except for the (currently unsupported) case of UTF-16.
+
+   Return 1 on success, 0 on failure. */
+
+static int
+fp_setreadl(struct tok_state *tok, const char* enc)
+{
+    PyObject *readline, *io, *stream;
+    _Py_IDENTIFIER(open);
+    _Py_IDENTIFIER(readline);
+    int fd;
+    long pos;
+
+    fd = fileno(tok->fp);
+    /* Due to buffering the file offset for fd can be different from the file
+     * position of tok->fp.  If tok->fp was opened in text mode on Windows,
+     * its file position counts CRLF as one char and can't be directly mapped
+     * to the file offset for fd.  Instead we step back one byte and read to
+     * the end of line.*/
+    pos = ftell(tok->fp);
+    if (pos == -1 ||
+        lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) {
+        PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
+        return 0;
+    }
+
+    io = PyImport_ImportModuleNoBlock("io");
+    if (io == NULL)
+        return 0;
+
+    stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO",
+                    fd, "r", -1, enc, Py_None, Py_None, Py_False);
+    Py_DECREF(io);
+    if (stream == NULL)
+        return 0;
+
+    readline = _PyObject_GetAttrId(stream, &PyId_readline);
+    Py_DECREF(stream);
+    if (readline == NULL)
+        return 0;
+    Py_XSETREF(tok->decoding_readline, readline);
+
+    if (pos > 0) {
+        PyObject *bufobj = _PyObject_CallNoArg(readline);
+        if (bufobj == NULL)
+            return 0;
+        Py_DECREF(bufobj);
+    }
+
+    return 1;
+}
+
+/* Fetch the next byte from TOK. */
+
+static int fp_getc(struct tok_state *tok) {
+    return getc(tok->fp);
+}
+
+/* Unfetch the last byte back into TOK.  */
+
+static void fp_ungetc(int c, struct tok_state *tok) {
+    ungetc(c, tok->fp);
+}
+
+/* Check whether the characters at s start a valid
+   UTF-8 sequence. Return the number of characters forming
+   the sequence if yes, 0 if not.  */
+static int valid_utf8(const unsigned char* s)
+{
+    int expected = 0;
+    int length;
+    if (*s < 0x80)
+        /* single-byte code */
+        return 1;
+    if (*s < 0xc0)
+        /* following byte */
+        return 0;
+    if (*s < 0xE0)
+        expected = 1;
+    else if (*s < 0xF0)
+        expected = 2;
+    else if (*s < 0xF8)
+        expected = 3;
+    else
+        return 0;
+    length = expected + 1;
+    for (; expected; expected--)
+        if (s[expected] < 0x80 || s[expected] >= 0xC0)
+            return 0;
+    return length;
+}
+
+/* Read a line of input from TOK. Determine encoding
+   if necessary.  */
+
+static char *
+decoding_fgets(char *s, int size, struct tok_state *tok)
+{
+    char *line = NULL;
+    int badchar = 0;
+    for (;;) {
+        if (tok->decoding_state == STATE_NORMAL) {
+            /* We already have a codec associated with
+               this input. */
+            line = fp_readl(s, size, tok);
+            break;
+        } else if (tok->decoding_state == STATE_RAW) {
+            /* We want a 'raw' read. */
+            line = Py_UniversalNewlineFgets(s, size,
+                                            tok->fp, NULL);
+            break;
+        } else {
+            /* We have not yet determined the encoding.
+               If an encoding is found, use the file-pointer
+               reader functions from now on. */
+            if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
+                return error_ret(tok);
+            assert(tok->decoding_state != STATE_INIT);
+        }
+    }
+    if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
+        if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
+            return error_ret(tok);
+        }
+    }
+    /* The default encoding is UTF-8, so make sure we don't have any
+       non-UTF-8 sequences in it. */
+    if (line && !tok->encoding) {
+        unsigned char *c;
+        int length;
+        for (c = (unsigned char *)line; *c; c += length)
+            if (!(length = valid_utf8(c))) {
+                badchar = *c;
+                break;
+            }
+    }
+    if (badchar) {
+        /* Need to add 1 to the line number, since this line
+           has not been counted, yet.  */
+        PyErr_Format(PyExc_SyntaxError,
+                "Non-UTF-8 code starting with '\\x%.2x' "
+                "in file %U on line %i, "
+                "but no encoding declared; "
                 "see https://python.org/dev/peps/pep-0263/ for details",
-                badchar, tok->filename, tok->lineno + 1); 
-        return error_ret(tok); 
-    } 
-    return line; 
-} 
- 
-static int 
-decoding_feof(struct tok_state *tok) 
-{ 
-    if (tok->decoding_state != STATE_NORMAL) { 
-        return feof(tok->fp); 
-    } else { 
-        PyObject* buf = tok->decoding_buffer; 
-        if (buf == NULL) { 
-            buf = _PyObject_CallNoArg(tok->decoding_readline); 
-            if (buf == NULL) { 
-                error_ret(tok); 
-                return 1; 
-            } else { 
-                tok->decoding_buffer = buf; 
-            } 
-        } 
-        return PyObject_Length(buf) == 0; 
-    } 
-} 
- 
-/* Fetch a byte from TOK, using the string buffer. */ 
- 
-static int 
-buf_getc(struct tok_state *tok) { 
-    return Py_CHARMASK(*tok->str++); 
-} 
- 
-/* Unfetch a byte from TOK, using the string buffer. */ 
- 
-static void 
-buf_ungetc(int c, struct tok_state *tok) { 
-    tok->str--; 
-    assert(Py_CHARMASK(*tok->str) == c);        /* tok->cur may point to read-only segment */ 
-} 
- 
-/* Set the readline function for TOK to ENC. For the string-based 
-   tokenizer, this means to just record the encoding. */ 
- 
-static int 
-buf_setreadl(struct tok_state *tok, const char* enc) { 
-    tok->enc = enc; 
-    return 1; 
-} 
- 
-/* Return a UTF-8 encoding Python string object from the 
-   C byte string STR, which is encoded with ENC. */ 
- 
-static PyObject * 
-translate_into_utf8(const char* str, const char* enc) { 
-    PyObject *utf8; 
-    PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL); 
-    if (buf == NULL) 
-        return NULL; 
-    utf8 = PyUnicode_AsUTF8String(buf); 
-    Py_DECREF(buf); 
-    return utf8; 
-} 
- 
- 
-static char * 
-translate_newlines(const char *s, int exec_input, struct tok_state *tok) { 
-    int skip_next_lf = 0; 
-    size_t needed_length = strlen(s) + 2, final_length; 
-    char *buf, *current; 
-    char c = '\0'; 
-    buf = PyMem_MALLOC(needed_length); 
-    if (buf == NULL) { 
-        tok->done = E_NOMEM; 
-        return NULL; 
-    } 
-    for (current = buf; *s; s++, current++) { 
-        c = *s; 
-        if (skip_next_lf) { 
-            skip_next_lf = 0; 
-            if (c == '\n') { 
-                c = *++s; 
-                if (!c) 
-                    break; 
-            } 
-        } 
-        if (c == '\r') { 
-            skip_next_lf = 1; 
-            c = '\n'; 
-        } 
-        *current = c; 
-    } 
-    /* If this is exec input, add a newline to the end of the string if 
-       there isn't one already. */ 
-    if (exec_input && c != '\n') { 
-        *current = '\n'; 
-        current++; 
-    } 
-    *current = '\0'; 
-    final_length = current - buf + 1; 
+                badchar, tok->filename, tok->lineno + 1);
+        return error_ret(tok);
+    }
+    return line;
+}
+
+static int
+decoding_feof(struct tok_state *tok)
+{
+    if (tok->decoding_state != STATE_NORMAL) {
+        return feof(tok->fp);
+    } else {
+        PyObject* buf = tok->decoding_buffer;
+        if (buf == NULL) {
+            buf = _PyObject_CallNoArg(tok->decoding_readline);
+            if (buf == NULL) {
+                error_ret(tok);
+                return 1;
+            } else {
+                tok->decoding_buffer = buf;
+            }
+        }
+        return PyObject_Length(buf) == 0;
+    }
+}
+
+/* Fetch a byte from TOK, using the string buffer. */
+
+static int
+buf_getc(struct tok_state *tok) {
+    return Py_CHARMASK(*tok->str++);
+}
+
+/* Unfetch a byte from TOK, using the string buffer. */
+
+static void
+buf_ungetc(int c, struct tok_state *tok) {
+    tok->str--;
+    assert(Py_CHARMASK(*tok->str) == c);        /* tok->cur may point to read-only segment */
+}
+
+/* Set the readline function for TOK to ENC. For the string-based
+   tokenizer, this means to just record the encoding. */
+
+static int
+buf_setreadl(struct tok_state *tok, const char* enc) {
+    tok->enc = enc;
+    return 1;
+}
+
+/* Return a UTF-8 encoding Python string object from the
+   C byte string STR, which is encoded with ENC. */
+
+static PyObject *
+translate_into_utf8(const char* str, const char* enc) {
+    PyObject *utf8;
+    PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
+    if (buf == NULL)
+        return NULL;
+    utf8 = PyUnicode_AsUTF8String(buf);
+    Py_DECREF(buf);
+    return utf8;
+}
+
+
+static char *
+translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
+    int skip_next_lf = 0;
+    size_t needed_length = strlen(s) + 2, final_length;
+    char *buf, *current;
+    char c = '\0';
+    buf = PyMem_MALLOC(needed_length);
+    if (buf == NULL) {
+        tok->done = E_NOMEM;
+        return NULL;
+    }
+    for (current = buf; *s; s++, current++) {
+        c = *s;
+        if (skip_next_lf) {
+            skip_next_lf = 0;
+            if (c == '\n') {
+                c = *++s;
+                if (!c)
+                    break;
+            }
+        }
+        if (c == '\r') {
+            skip_next_lf = 1;
+            c = '\n';
+        }
+        *current = c;
+    }
+    /* If this is exec input, add a newline to the end of the string if
+       there isn't one already. */
+    if (exec_input && c != '\n') {
+        *current = '\n';
+        current++;
+    }
+    *current = '\0';
+    final_length = current - buf + 1;
     if (final_length < needed_length && final_length) {
-        /* should never fail */ 
+        /* should never fail */
         char* result = PyMem_REALLOC(buf, final_length);
         if (result == NULL) {
             PyMem_FREE(buf);
         }
         buf = result;
     }
-    return buf; 
-} 
- 
-/* Decode a byte string STR for use as the buffer of TOK. 
-   Look for encoding declarations inside STR, and record them 
-   inside TOK.  */ 
- 
+    return buf;
+}
+
+/* Decode a byte string STR for use as the buffer of TOK.
+   Look for encoding declarations inside STR, and record them
+   inside TOK.  */
+
 static char *
-decode_str(const char *input, int single, struct tok_state *tok) 
-{ 
-    PyObject* utf8 = NULL; 
+decode_str(const char *input, int single, struct tok_state *tok)
+{
+    PyObject* utf8 = NULL;
     char *str;
-    const char *s; 
-    const char *newl[2] = {NULL, NULL}; 
-    int lineno = 0; 
-    tok->input = str = translate_newlines(input, single, tok); 
-    if (str == NULL) 
-        return NULL; 
-    tok->enc = NULL; 
-    tok->str = str; 
-    if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) 
-        return error_ret(tok); 
-    str = tok->str;             /* string after BOM if any */ 
-    assert(str); 
-    if (tok->enc != NULL) { 
-        utf8 = translate_into_utf8(str, tok->enc); 
-        if (utf8 == NULL) 
-            return error_ret(tok); 
-        str = PyBytes_AsString(utf8); 
-    } 
-    for (s = str;; s++) { 
-        if (*s == '\0') break; 
-        else if (*s == '\n') { 
-            assert(lineno < 2); 
-            newl[lineno] = s; 
-            lineno++; 
-            if (lineno == 2) break; 
-        } 
-    } 
-    tok->enc = NULL; 
-    /* need to check line 1 and 2 separately since check_coding_spec 
-       assumes a single line as input */ 
-    if (newl[0]) { 
-        if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) 
-            return error_ret(tok); 
-        if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) { 
-            if (!check_coding_spec(newl[0]+1, newl[1] - newl[0], 
-                                   tok, buf_setreadl)) 
-                return error_ret(tok); 
-        } 
-    } 
-    if (tok->enc != NULL) { 
-        assert(utf8 == NULL); 
-        utf8 = translate_into_utf8(str, tok->enc); 
-        if (utf8 == NULL) 
-            return error_ret(tok); 
-        str = PyBytes_AS_STRING(utf8); 
-    } 
-    assert(tok->decoding_buffer == NULL); 
-    tok->decoding_buffer = utf8; /* CAUTION */ 
-    return str; 
-} 
- 
-/* Set up tokenizer for string */ 
- 
-struct tok_state * 
-PyTokenizer_FromString(const char *str, int exec_input) 
-{ 
-    struct tok_state *tok = tok_new(); 
+    const char *s;
+    const char *newl[2] = {NULL, NULL};
+    int lineno = 0;
+    tok->input = str = translate_newlines(input, single, tok);
+    if (str == NULL)
+        return NULL;
+    tok->enc = NULL;
+    tok->str = str;
+    if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
+        return error_ret(tok);
+    str = tok->str;             /* string after BOM if any */
+    assert(str);
+    if (tok->enc != NULL) {
+        utf8 = translate_into_utf8(str, tok->enc);
+        if (utf8 == NULL)
+            return error_ret(tok);
+        str = PyBytes_AsString(utf8);
+    }
+    for (s = str;; s++) {
+        if (*s == '\0') break;
+        else if (*s == '\n') {
+            assert(lineno < 2);
+            newl[lineno] = s;
+            lineno++;
+            if (lineno == 2) break;
+        }
+    }
+    tok->enc = NULL;
+    /* need to check line 1 and 2 separately since check_coding_spec
+       assumes a single line as input */
+    if (newl[0]) {
+        if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
+            return error_ret(tok);
+        if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) {
+            if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
+                                   tok, buf_setreadl))
+                return error_ret(tok);
+        }
+    }
+    if (tok->enc != NULL) {
+        assert(utf8 == NULL);
+        utf8 = translate_into_utf8(str, tok->enc);
+        if (utf8 == NULL)
+            return error_ret(tok);
+        str = PyBytes_AS_STRING(utf8);
+    }
+    assert(tok->decoding_buffer == NULL);
+    tok->decoding_buffer = utf8; /* CAUTION */
+    return str;
+}
+
+/* Set up tokenizer for string */
+
+struct tok_state *
+PyTokenizer_FromString(const char *str, int exec_input)
+{
+    struct tok_state *tok = tok_new();
     char *decoded;
 
-    if (tok == NULL) 
-        return NULL; 
+    if (tok == NULL)
+        return NULL;
     decoded = decode_str(str, exec_input, tok);
     if (decoded == NULL) {
-        PyTokenizer_Free(tok); 
-        return NULL; 
-    } 
- 
+        PyTokenizer_Free(tok);
+        return NULL;
+    }
+
     tok->buf = tok->cur = tok->inp = decoded;
     tok->end = decoded;
-    return tok; 
-} 
- 
-struct tok_state * 
-PyTokenizer_FromUTF8(const char *str, int exec_input) 
-{ 
-    struct tok_state *tok = tok_new(); 
+    return tok;
+}
+
+struct tok_state *
+PyTokenizer_FromUTF8(const char *str, int exec_input)
+{
+    struct tok_state *tok = tok_new();
     char *translated;
-    if (tok == NULL) 
-        return NULL; 
+    if (tok == NULL)
+        return NULL;
     tok->input = translated = translate_newlines(str, exec_input, tok);
     if (translated == NULL) {
-        PyTokenizer_Free(tok); 
-        return NULL; 
-    } 
-    tok->decoding_state = STATE_RAW; 
-    tok->read_coding_spec = 1; 
-    tok->enc = NULL; 
+        PyTokenizer_Free(tok);
+        return NULL;
+    }
+    tok->decoding_state = STATE_RAW;
+    tok->read_coding_spec = 1;
+    tok->enc = NULL;
     tok->str = translated;
-    tok->encoding = (char *)PyMem_MALLOC(6); 
-    if (!tok->encoding) { 
-        PyTokenizer_Free(tok); 
-        return NULL; 
-    } 
-    strcpy(tok->encoding, "utf-8"); 
- 
+    tok->encoding = (char *)PyMem_MALLOC(6);
+    if (!tok->encoding) {
+        PyTokenizer_Free(tok);
+        return NULL;
+    }
+    strcpy(tok->encoding, "utf-8");
+
     tok->buf = tok->cur = tok->inp = translated;
     tok->end = translated;
-    return tok; 
-} 
- 
-/* Set up tokenizer for file */ 
- 
-struct tok_state * 
-PyTokenizer_FromFile(FILE *fp, const char* enc, 
-                     const char *ps1, const char *ps2) 
-{ 
-    struct tok_state *tok = tok_new(); 
-    if (tok == NULL) 
-        return NULL; 
-    if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) { 
-        PyTokenizer_Free(tok); 
-        return NULL; 
-    } 
-    tok->cur = tok->inp = tok->buf; 
-    tok->end = tok->buf + BUFSIZ; 
-    tok->fp = fp; 
-    tok->prompt = ps1; 
-    tok->nextprompt = ps2; 
-    if (enc != NULL) { 
-        /* Must copy encoding declaration since it 
-           gets copied into the parse tree. */ 
-        tok->encoding = PyMem_MALLOC(strlen(enc)+1); 
-        if (!tok->encoding) { 
-            PyTokenizer_Free(tok); 
-            return NULL; 
-        } 
-        strcpy(tok->encoding, enc); 
-        tok->decoding_state = STATE_NORMAL; 
-    } 
-    return tok; 
-} 
- 
- 
-/* Free a tok_state structure */ 
- 
-void 
-PyTokenizer_Free(struct tok_state *tok) 
-{ 
-    if (tok->encoding != NULL) 
-        PyMem_FREE(tok->encoding); 
-    Py_XDECREF(tok->decoding_readline); 
-    Py_XDECREF(tok->decoding_buffer); 
-    Py_XDECREF(tok->filename); 
-    if (tok->fp != NULL && tok->buf != NULL) 
-        PyMem_FREE(tok->buf); 
-    if (tok->input) 
+    return tok;
+}
+
+/* Set up tokenizer for file */
+
+struct tok_state *
+PyTokenizer_FromFile(FILE *fp, const char* enc,
+                     const char *ps1, const char *ps2)
+{
+    struct tok_state *tok = tok_new();
+    if (tok == NULL)
+        return NULL;
+    if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
+        PyTokenizer_Free(tok);
+        return NULL;
+    }
+    tok->cur = tok->inp = tok->buf;
+    tok->end = tok->buf + BUFSIZ;
+    tok->fp = fp;
+    tok->prompt = ps1;
+    tok->nextprompt = ps2;
+    if (enc != NULL) {
+        /* Must copy encoding declaration since it
+           gets copied into the parse tree. */
+        tok->encoding = PyMem_MALLOC(strlen(enc)+1);
+        if (!tok->encoding) {
+            PyTokenizer_Free(tok);
+            return NULL;
+        }
+        strcpy(tok->encoding, enc);
+        tok->decoding_state = STATE_NORMAL;
+    }
+    return tok;
+}
+
+
+/* Free a tok_state structure */
+
+void
+PyTokenizer_Free(struct tok_state *tok)
+{
+    if (tok->encoding != NULL)
+        PyMem_FREE(tok->encoding);
+    Py_XDECREF(tok->decoding_readline);
+    Py_XDECREF(tok->decoding_buffer);
+    Py_XDECREF(tok->filename);
+    if (tok->fp != NULL && tok->buf != NULL)
+        PyMem_FREE(tok->buf);
+    if (tok->input)
         PyMem_FREE(tok->input);
-    PyMem_FREE(tok); 
-} 
- 
-/* Get next char, updating state; error code goes into tok->done */ 
- 
-static int 
-tok_nextc(struct tok_state *tok) 
-{ 
-    for (;;) { 
-        if (tok->cur != tok->inp) { 
-            return Py_CHARMASK(*tok->cur++); /* Fast path */ 
-        } 
-        if (tok->done != E_OK) 
-            return EOF; 
-        if (tok->fp == NULL) { 
-            char *end = strchr(tok->inp, '\n'); 
-            if (end != NULL) 
-                end++; 
-            else { 
-                end = strchr(tok->inp, '\0'); 
-                if (end == tok->inp) { 
-                    tok->done = E_EOF; 
-                    return EOF; 
-                } 
-            } 
-            if (tok->start == NULL) 
-                tok->buf = tok->cur; 
-            tok->line_start = tok->cur; 
-            tok->lineno++; 
-            tok->inp = end; 
-            return Py_CHARMASK(*tok->cur++); 
-        } 
-        if (tok->prompt != NULL) { 
-            char *newtok = PyOS_Readline(stdin, stdout, tok->prompt); 
-            if (newtok != NULL) { 
-                char *translated = translate_newlines(newtok, 0, tok); 
-                PyMem_FREE(newtok); 
-                if (translated == NULL) 
-                    return EOF; 
-                newtok = translated; 
-            } 
-            if (tok->encoding && newtok && *newtok) { 
-                /* Recode to UTF-8 */ 
-                Py_ssize_t buflen; 
-                const char* buf; 
-                PyObject *u = translate_into_utf8(newtok, tok->encoding); 
-                PyMem_FREE(newtok); 
-                if (!u) { 
-                    tok->done = E_DECODE; 
-                    return EOF; 
-                } 
-                buflen = PyBytes_GET_SIZE(u); 
-                buf = PyBytes_AS_STRING(u); 
-                newtok = PyMem_MALLOC(buflen+1); 
-                if (newtok == NULL) { 
-                    Py_DECREF(u); 
-                    tok->done = E_NOMEM; 
-                    return EOF; 
-                } 
-                strcpy(newtok, buf); 
-                Py_DECREF(u); 
-            } 
-            if (tok->nextprompt != NULL) 
-                tok->prompt = tok->nextprompt; 
-            if (newtok == NULL) 
-                tok->done = E_INTR; 
-            else if (*newtok == '\0') { 
-                PyMem_FREE(newtok); 
-                tok->done = E_EOF; 
-            } 
-            else if (tok->start != NULL) { 
-                size_t start = tok->start - tok->buf; 
-                size_t oldlen = tok->cur - tok->buf; 
-                size_t newlen = oldlen + strlen(newtok); 
+    PyMem_FREE(tok);
+}
+
+/* Get next char, updating state; error code goes into tok->done */
+
+static int
+tok_nextc(struct tok_state *tok)
+{
+    for (;;) {
+        if (tok->cur != tok->inp) {
+            return Py_CHARMASK(*tok->cur++); /* Fast path */
+        }
+        if (tok->done != E_OK)
+            return EOF;
+        if (tok->fp == NULL) {
+            char *end = strchr(tok->inp, '\n');
+            if (end != NULL)
+                end++;
+            else {
+                end = strchr(tok->inp, '\0');
+                if (end == tok->inp) {
+                    tok->done = E_EOF;
+                    return EOF;
+                }
+            }
+            if (tok->start == NULL)
+                tok->buf = tok->cur;
+            tok->line_start = tok->cur;
+            tok->lineno++;
+            tok->inp = end;
+            return Py_CHARMASK(*tok->cur++);
+        }
+        if (tok->prompt != NULL) {
+            char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
+            if (newtok != NULL) {
+                char *translated = translate_newlines(newtok, 0, tok);
+                PyMem_FREE(newtok);
+                if (translated == NULL)
+                    return EOF;
+                newtok = translated;
+            }
+            if (tok->encoding && newtok && *newtok) {
+                /* Recode to UTF-8 */
+                Py_ssize_t buflen;
+                const char* buf;
+                PyObject *u = translate_into_utf8(newtok, tok->encoding);
+                PyMem_FREE(newtok);
+                if (!u) {
+                    tok->done = E_DECODE;
+                    return EOF;
+                }
+                buflen = PyBytes_GET_SIZE(u);
+                buf = PyBytes_AS_STRING(u);
+                newtok = PyMem_MALLOC(buflen+1);
+                if (newtok == NULL) {
+                    Py_DECREF(u);
+                    tok->done = E_NOMEM;
+                    return EOF;
+                }
+                strcpy(newtok, buf);
+                Py_DECREF(u);
+            }
+            if (tok->nextprompt != NULL)
+                tok->prompt = tok->nextprompt;
+            if (newtok == NULL)
+                tok->done = E_INTR;
+            else if (*newtok == '\0') {
+                PyMem_FREE(newtok);
+                tok->done = E_EOF;
+            }
+            else if (tok->start != NULL) {
+                size_t start = tok->start - tok->buf;
+                size_t oldlen = tok->cur - tok->buf;
+                size_t newlen = oldlen + strlen(newtok);
                 Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
-                char *buf = tok->buf; 
-                buf = (char *)PyMem_REALLOC(buf, newlen+1); 
-                tok->lineno++; 
-                if (buf == NULL) { 
-                    PyMem_FREE(tok->buf); 
-                    tok->buf = NULL; 
-                    PyMem_FREE(newtok); 
-                    tok->done = E_NOMEM; 
-                    return EOF; 
-                } 
-                tok->buf = buf; 
-                tok->cur = tok->buf + oldlen; 
+                char *buf = tok->buf;
+                buf = (char *)PyMem_REALLOC(buf, newlen+1);
+                tok->lineno++;
+                if (buf == NULL) {
+                    PyMem_FREE(tok->buf);
+                    tok->buf = NULL;
+                    PyMem_FREE(newtok);
+                    tok->done = E_NOMEM;
+                    return EOF;
+                }
+                tok->buf = buf;
+                tok->cur = tok->buf + oldlen;
                 tok->multi_line_start = tok->buf + cur_multi_line_start;
-                tok->line_start = tok->cur; 
-                strcpy(tok->buf + oldlen, newtok); 
-                PyMem_FREE(newtok); 
-                tok->inp = tok->buf + newlen; 
-                tok->end = tok->inp + 1; 
-                tok->start = tok->buf + start; 
-            } 
-            else { 
-                tok->lineno++; 
-                if (tok->buf != NULL) 
-                    PyMem_FREE(tok->buf); 
-                tok->buf = newtok; 
-                tok->cur = tok->buf; 
-                tok->line_start = tok->buf; 
-                tok->inp = strchr(tok->buf, '\0'); 
-                tok->end = tok->inp + 1; 
-            } 
-        } 
-        else { 
-            int done = 0; 
-            Py_ssize_t cur = 0; 
-            char *pt; 
-            if (tok->start == NULL) { 
-                if (tok->buf == NULL) { 
-                    tok->buf = (char *) 
-                        PyMem_MALLOC(BUFSIZ); 
-                    if (tok->buf == NULL) { 
-                        tok->done = E_NOMEM; 
-                        return EOF; 
-                    } 
-                    tok->end = tok->buf + BUFSIZ; 
-                } 
-                if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf), 
-                          tok) == NULL) { 
-                    if (!tok->decoding_erred) 
-                        tok->done = E_EOF; 
-                    done = 1; 
-                } 
-                else { 
-                    tok->done = E_OK; 
-                    tok->inp = strchr(tok->buf, '\0'); 
-                    done = tok->inp == tok->buf || tok->inp[-1] == '\n'; 
-                } 
-            } 
-            else { 
-                cur = tok->cur - tok->buf; 
-                if (decoding_feof(tok)) { 
-                    tok->done = E_EOF; 
-                    done = 1; 
-                } 
-                else 
-                    tok->done = E_OK; 
-            } 
-            tok->lineno++; 
-            /* Read until '\n' or EOF */ 
-            while (!done) { 
-                Py_ssize_t curstart = tok->start == NULL ? -1 : 
-                          tok->start - tok->buf; 
+                tok->line_start = tok->cur;
+                strcpy(tok->buf + oldlen, newtok);
+                PyMem_FREE(newtok);
+                tok->inp = tok->buf + newlen;
+                tok->end = tok->inp + 1;
+                tok->start = tok->buf + start;
+            }
+            else {
+                tok->lineno++;
+                if (tok->buf != NULL)
+                    PyMem_FREE(tok->buf);
+                tok->buf = newtok;
+                tok->cur = tok->buf;
+                tok->line_start = tok->buf;
+                tok->inp = strchr(tok->buf, '\0');
+                tok->end = tok->inp + 1;
+            }
+        }
+        else {
+            int done = 0;
+            Py_ssize_t cur = 0;
+            char *pt;
+            if (tok->start == NULL) {
+                if (tok->buf == NULL) {
+                    tok->buf = (char *)
+                        PyMem_MALLOC(BUFSIZ);
+                    if (tok->buf == NULL) {
+                        tok->done = E_NOMEM;
+                        return EOF;
+                    }
+                    tok->end = tok->buf + BUFSIZ;
+                }
+                if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
+                          tok) == NULL) {
+                    if (!tok->decoding_erred)
+                        tok->done = E_EOF;
+                    done = 1;
+                }
+                else {
+                    tok->done = E_OK;
+                    tok->inp = strchr(tok->buf, '\0');
+                    done = tok->inp == tok->buf || tok->inp[-1] == '\n';
+                }
+            }
+            else {
+                cur = tok->cur - tok->buf;
+                if (decoding_feof(tok)) {
+                    tok->done = E_EOF;
+                    done = 1;
+                }
+                else
+                    tok->done = E_OK;
+            }
+            tok->lineno++;
+            /* Read until '\n' or EOF */
+            while (!done) {
+                Py_ssize_t curstart = tok->start == NULL ? -1 :
+                          tok->start - tok->buf;
                 Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
-                Py_ssize_t curvalid = tok->inp - tok->buf; 
-                Py_ssize_t newsize = curvalid + BUFSIZ; 
-                char *newbuf = tok->buf; 
-                newbuf = (char *)PyMem_REALLOC(newbuf, 
-                                               newsize); 
-                if (newbuf == NULL) { 
-                    tok->done = E_NOMEM; 
-                    tok->cur = tok->inp; 
-                    return EOF; 
-                } 
-                tok->buf = newbuf; 
-                tok->cur = tok->buf + cur; 
+                Py_ssize_t curvalid = tok->inp - tok->buf;
+                Py_ssize_t newsize = curvalid + BUFSIZ;
+                char *newbuf = tok->buf;
+                newbuf = (char *)PyMem_REALLOC(newbuf,
+                                               newsize);
+                if (newbuf == NULL) {
+                    tok->done = E_NOMEM;
+                    tok->cur = tok->inp;
+                    return EOF;
+                }
+                tok->buf = newbuf;
+                tok->cur = tok->buf + cur;
                 tok->multi_line_start = tok->buf + cur_multi_line_start;
-                tok->line_start = tok->cur; 
-                tok->inp = tok->buf + curvalid; 
-                tok->end = tok->buf + newsize; 
-                tok->start = curstart < 0 ? NULL : 
-                         tok->buf + curstart; 
-                if (decoding_fgets(tok->inp, 
-                               (int)(tok->end - tok->inp), 
-                               tok) == NULL) { 
-                    /* Break out early on decoding 
-                       errors, as tok->buf will be NULL 
-                     */ 
-                    if (tok->decoding_erred) 
-                        return EOF; 
-                    /* Last line does not end in \n, 
-                       fake one */ 
+                tok->line_start = tok->cur;
+                tok->inp = tok->buf + curvalid;
+                tok->end = tok->buf + newsize;
+                tok->start = curstart < 0 ? NULL :
+                         tok->buf + curstart;
+                if (decoding_fgets(tok->inp,
+                               (int)(tok->end - tok->inp),
+                               tok) == NULL) {
+                    /* Break out early on decoding
+                       errors, as tok->buf will be NULL
+                     */
+                    if (tok->decoding_erred)
+                        return EOF;
+                    /* Last line does not end in \n,
+                       fake one */
                     if (tok->inp[-1] != '\n')
                         strcpy(tok->inp, "\n");
-                } 
-                tok->inp = strchr(tok->inp, '\0'); 
-                done = tok->inp[-1] == '\n'; 
-            } 
-            if (tok->buf != NULL) { 
-                tok->cur = tok->buf + cur; 
-                tok->line_start = tok->cur; 
-                /* replace "\r\n" with "\n" */ 
-                /* For Mac leave the \r, giving a syntax error */ 
-                pt = tok->inp - 2; 
-                if (pt >= tok->buf && *pt == '\r') { 
-                    *pt++ = '\n'; 
-                    *pt = '\0'; 
-                    tok->inp = pt; 
-                } 
-            } 
-        } 
-        if (tok->done != E_OK) { 
-            if (tok->prompt != NULL) 
-                PySys_WriteStderr("\n"); 
-            tok->cur = tok->inp; 
-            return EOF; 
-        } 
-    } 
-    /*NOTREACHED*/ 
-} 
- 
- 
-/* Back-up one character */ 
- 
-static void 
-tok_backup(struct tok_state *tok, int c) 
-{ 
-    if (c != EOF) { 
+                }
+                tok->inp = strchr(tok->inp, '\0');
+                done = tok->inp[-1] == '\n';
+            }
+            if (tok->buf != NULL) {
+                tok->cur = tok->buf + cur;
+                tok->line_start = tok->cur;
+                /* replace "\r\n" with "\n" */
+                /* For Mac leave the \r, giving a syntax error */
+                pt = tok->inp - 2;
+                if (pt >= tok->buf && *pt == '\r') {
+                    *pt++ = '\n';
+                    *pt = '\0';
+                    tok->inp = pt;
+                }
+            }
+        }
+        if (tok->done != E_OK) {
+            if (tok->prompt != NULL)
+                PySys_WriteStderr("\n");
+            tok->cur = tok->inp;
+            return EOF;
+        }
+    }
+    /*NOTREACHED*/
+}
+
+
+/* Back-up one character */
+
+static void
+tok_backup(struct tok_state *tok, int c)
+{
+    if (c != EOF) {
         if (--tok->cur < tok->buf) {
             Py_FatalError("tokenizer beginning of buffer");
         }
         if (*tok->cur != c) {
-            *tok->cur = c; 
+            *tok->cur = c;
         }
-    } 
-} 
- 
- 
+    }
+}
+
+
 static int
 syntaxerror(struct tok_state *tok, const char *format, ...)
-{ 
+{
     PyObject *errmsg, *errtext, *args;
     va_list vargs;
 #ifdef HAVE_STDARG_PROTOTYPES
@@ -1051,8 +1051,8 @@ syntaxerror(struct tok_state *tok, const char *format, ...)
     va_end(vargs);
     if (!errmsg) {
         goto error;
-    } 
- 
+    }
+
     errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
                                    "replace");
     if (!errtext) {
@@ -1068,47 +1068,47 @@ syntaxerror(struct tok_state *tok, const char *format, ...)
     if (!errtext) {
         goto error;
     }
- 
+
     args = Py_BuildValue("(O(OiiN))", errmsg,
                          tok->filename, tok->lineno, offset, errtext);
     if (args) {
         PyErr_SetObject(PyExc_SyntaxError, args);
         Py_DECREF(args);
-    } 
- 
+    }
+
 error:
     Py_XDECREF(errmsg);
     tok->done = E_ERROR;
     return ERRORTOKEN;
-} 
- 
-static int 
-indenterror(struct tok_state *tok) 
-{ 
-    tok->done = E_TABSPACE; 
-    tok->cur = tok->inp; 
-    return ERRORTOKEN; 
-} 
- 
-/* Verify that the identifier follows PEP 3131. 
-   All identifier strings are guaranteed to be "ready" unicode objects. 
- */ 
-static int 
-verify_identifier(struct tok_state *tok) 
-{ 
-    PyObject *s; 
-    if (tok->decoding_erred) 
-        return 0; 
-    s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); 
+}
+
+static int
+indenterror(struct tok_state *tok)
+{
+    tok->done = E_TABSPACE;
+    tok->cur = tok->inp;
+    return ERRORTOKEN;
+}
+
+/* Verify that the identifier follows PEP 3131.
+   All identifier strings are guaranteed to be "ready" unicode objects.
+ */
+static int
+verify_identifier(struct tok_state *tok)
+{
+    PyObject *s;
+    if (tok->decoding_erred)
+        return 0;
+    s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
     if (s == NULL) {
-        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { 
+        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
             tok->done = E_DECODE;
         }
         else {
-            tok->done = E_ERROR; 
-        } 
-        return 0; 
-    } 
+            tok->done = E_ERROR;
+        }
+        return 0;
+    }
     Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
     if (invalid < 0) {
         Py_DECREF(s);
@@ -1142,143 +1142,143 @@ verify_identifier(struct tok_state *tok)
         }
         return 0;
     }
-    Py_DECREF(s); 
+    Py_DECREF(s);
     return 1;
-} 
- 
-static int 
-tok_decimal_tail(struct tok_state *tok) 
-{ 
-    int c; 
- 
-    while (1) { 
-        do { 
-            c = tok_nextc(tok); 
-        } while (isdigit(c)); 
-        if (c != '_') { 
-            break; 
-        } 
-        c = tok_nextc(tok); 
-        if (!isdigit(c)) { 
-            tok_backup(tok, c); 
+}
+
+static int
+tok_decimal_tail(struct tok_state *tok)
+{
+    int c;
+
+    while (1) {
+        do {
+            c = tok_nextc(tok);
+        } while (isdigit(c));
+        if (c != '_') {
+            break;
+        }
+        c = tok_nextc(tok);
+        if (!isdigit(c)) {
+            tok_backup(tok, c);
             syntaxerror(tok, "invalid decimal literal");
-            return 0; 
-        } 
-    } 
-    return c; 
-} 
- 
-/* Get next token, after space stripping etc. */ 
- 
-static int 
+            return 0;
+        }
+    }
+    return c;
+}
+
+/* Get next token, after space stripping etc. */
+
+static int
 tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
-{ 
-    int c; 
-    int blankline, nonascii; 
- 
-    *p_start = *p_end = NULL; 
-  nextline: 
-    tok->start = NULL; 
-    blankline = 0; 
- 
-    /* Get indentation level */ 
-    if (tok->atbol) { 
-        int col = 0; 
-        int altcol = 0; 
-        tok->atbol = 0; 
-        for (;;) { 
-            c = tok_nextc(tok); 
-            if (c == ' ') { 
-                col++, altcol++; 
-            } 
-            else if (c == '\t') { 
-                col = (col / tok->tabsize + 1) * tok->tabsize; 
-                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; 
-            } 
-            else if (c == '\014')  {/* Control-L (formfeed) */ 
-                col = altcol = 0; /* For Emacs users */ 
-            } 
-            else { 
-                break; 
-            } 
-        } 
-        tok_backup(tok, c); 
+{
+    int c;
+    int blankline, nonascii;
+
+    *p_start = *p_end = NULL;
+  nextline:
+    tok->start = NULL;
+    blankline = 0;
+
+    /* Get indentation level */
+    if (tok->atbol) {
+        int col = 0;
+        int altcol = 0;
+        tok->atbol = 0;
+        for (;;) {
+            c = tok_nextc(tok);
+            if (c == ' ') {
+                col++, altcol++;
+            }
+            else if (c == '\t') {
+                col = (col / tok->tabsize + 1) * tok->tabsize;
+                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
+            }
+            else if (c == '\014')  {/* Control-L (formfeed) */
+                col = altcol = 0; /* For Emacs users */
+            }
+            else {
+                break;
+            }
+        }
+        tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\\') {
-            /* Lines with only whitespace and/or comments 
+            /* Lines with only whitespace and/or comments
                and/or a line continuation character
-               shouldn't affect the indentation and are 
-               not passed to the parser as NEWLINE tokens, 
-               except *totally* empty lines in interactive 
-               mode, which signal the end of a command group. */ 
-            if (col == 0 && c == '\n' && tok->prompt != NULL) { 
-                blankline = 0; /* Let it through */ 
-            } 
+               shouldn't affect the indentation and are
+               not passed to the parser as NEWLINE tokens,
+               except *totally* empty lines in interactive
+               mode, which signal the end of a command group. */
+            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+                blankline = 0; /* Let it through */
+            }
             else if (tok->prompt != NULL && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;
                 col = altcol = 0;
             }
-            else { 
-                blankline = 1; /* Ignore completely */ 
-            } 
-            /* We can't jump back right here since we still 
-               may need to skip to the end of a comment */ 
-        } 
-        if (!blankline && tok->level == 0) { 
-            if (col == tok->indstack[tok->indent]) { 
-                /* No change */ 
-                if (altcol != tok->altindstack[tok->indent]) { 
-                    return indenterror(tok); 
-                } 
-            } 
-            else if (col > tok->indstack[tok->indent]) { 
-                /* Indent -- always one */ 
-                if (tok->indent+1 >= MAXINDENT) { 
-                    tok->done = E_TOODEEP; 
-                    tok->cur = tok->inp; 
-                    return ERRORTOKEN; 
-                } 
-                if (altcol <= tok->altindstack[tok->indent]) { 
-                    return indenterror(tok); 
-                } 
-                tok->pendin++; 
-                tok->indstack[++tok->indent] = col; 
-                tok->altindstack[tok->indent] = altcol; 
-            } 
-            else /* col < tok->indstack[tok->indent] */ { 
-                /* Dedent -- any number, must be consistent */ 
-                while (tok->indent > 0 && 
-                    col < tok->indstack[tok->indent]) { 
-                    tok->pendin--; 
-                    tok->indent--; 
-                } 
-                if (col != tok->indstack[tok->indent]) { 
-                    tok->done = E_DEDENT; 
-                    tok->cur = tok->inp; 
-                    return ERRORTOKEN; 
-                } 
-                if (altcol != tok->altindstack[tok->indent]) { 
-                    return indenterror(tok); 
-                } 
-            } 
-        } 
-    } 
- 
-    tok->start = tok->cur; 
- 
-    /* Return pending indents/dedents */ 
-    if (tok->pendin != 0) { 
-        if (tok->pendin < 0) { 
-            tok->pendin++; 
-            return DEDENT; 
-        } 
-        else { 
-            tok->pendin--; 
-            return INDENT; 
-        } 
-    } 
- 
+            else {
+                blankline = 1; /* Ignore completely */
+            }
+            /* We can't jump back right here since we still
+               may need to skip to the end of a comment */
+        }
+        if (!blankline && tok->level == 0) {
+            if (col == tok->indstack[tok->indent]) {
+                /* No change */
+                if (altcol != tok->altindstack[tok->indent]) {
+                    return indenterror(tok);
+                }
+            }
+            else if (col > tok->indstack[tok->indent]) {
+                /* Indent -- always one */
+                if (tok->indent+1 >= MAXINDENT) {
+                    tok->done = E_TOODEEP;
+                    tok->cur = tok->inp;
+                    return ERRORTOKEN;
+                }
+                if (altcol <= tok->altindstack[tok->indent]) {
+                    return indenterror(tok);
+                }
+                tok->pendin++;
+                tok->indstack[++tok->indent] = col;
+                tok->altindstack[tok->indent] = altcol;
+            }
+            else /* col < tok->indstack[tok->indent] */ {
+                /* Dedent -- any number, must be consistent */
+                while (tok->indent > 0 &&
+                    col < tok->indstack[tok->indent]) {
+                    tok->pendin--;
+                    tok->indent--;
+                }
+                if (col != tok->indstack[tok->indent]) {
+                    tok->done = E_DEDENT;
+                    tok->cur = tok->inp;
+                    return ERRORTOKEN;
+                }
+                if (altcol != tok->altindstack[tok->indent]) {
+                    return indenterror(tok);
+                }
+            }
+        }
+    }
+
+    tok->start = tok->cur;
+
+    /* Return pending indents/dedents */
+    if (tok->pendin != 0) {
+        if (tok->pendin < 0) {
+            tok->pendin++;
+            return DEDENT;
+        }
+        else {
+            tok->pendin--;
+            return INDENT;
+        }
+    }
+
     /* Peek ahead at the next character */
     c = tok_nextc(tok);
     tok_backup(tok, c);
@@ -1304,23 +1304,23 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         tok->async_def_nl = 0;
     }
 
- again: 
-    tok->start = NULL; 
-    /* Skip spaces */ 
-    do { 
-        c = tok_nextc(tok); 
-    } while (c == ' ' || c == '\t' || c == '\014'); 
- 
-    /* Set start of current token */ 
-    tok->start = tok->cur - 1; 
- 
+ again:
+    tok->start = NULL;
+    /* Skip spaces */
+    do {
+        c = tok_nextc(tok);
+    } while (c == ' ' || c == '\t' || c == '\014');
+
+    /* Set start of current token */
+    tok->start = tok->cur - 1;
+
     /* Skip comment, unless it's a type comment */
-    if (c == '#') { 
+    if (c == '#') {
         const char *prefix, *p, *type_start;
 
-        while (c != EOF && c != '\n') { 
-            c = tok_nextc(tok); 
-        } 
+        while (c != EOF && c != '\n') {
+            c = tok_nextc(tok);
+        }
 
         if (tok->type_comments) {
             p = tok->start;
@@ -1371,56 +1371,56 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
                 }
             }
         }
-    } 
- 
-    /* Check for EOF and errors now */ 
-    if (c == EOF) { 
-        return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; 
-    } 
- 
-    /* Identifier (most frequent token!) */ 
-    nonascii = 0; 
-    if (is_potential_identifier_start(c)) { 
-        /* Process the various legal combinations of b"", r"", u"", and f"". */ 
-        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0; 
-        while (1) { 
-            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B')) 
-                saw_b = 1; 
-            /* Since this is a backwards compatibility support literal we don't 
-               want to support it in arbitrary order like byte literals. */ 
-            else if (!(saw_b || saw_u || saw_r || saw_f) 
-                     && (c == 'u'|| c == 'U')) { 
-                saw_u = 1; 
-            } 
-            /* ur"" and ru"" are not supported */ 
-            else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) { 
-                saw_r = 1; 
-            } 
-            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) { 
-                saw_f = 1; 
-            } 
-            else { 
-                break; 
-            } 
-            c = tok_nextc(tok); 
-            if (c == '"' || c == '\'') { 
-                goto letter_quote; 
-            } 
-        } 
-        while (is_potential_identifier_char(c)) { 
-            if (c >= 128) { 
-                nonascii = 1; 
-            } 
-            c = tok_nextc(tok); 
-        } 
-        tok_backup(tok, c); 
-        if (nonascii && !verify_identifier(tok)) { 
-            return ERRORTOKEN; 
-        } 
+    }
+
+    /* Check for EOF and errors now */
+    if (c == EOF) {
+        return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
+    }
+
+    /* Identifier (most frequent token!) */
+    nonascii = 0;
+    if (is_potential_identifier_start(c)) {
+        /* Process the various legal combinations of b"", r"", u"", and f"". */
+        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
+        while (1) {
+            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
+                saw_b = 1;
+            /* Since this is a backwards compatibility support literal we don't
+               want to support it in arbitrary order like byte literals. */
+            else if (!(saw_b || saw_u || saw_r || saw_f)
+                     && (c == 'u'|| c == 'U')) {
+                saw_u = 1;
+            }
+            /* ur"" and ru"" are not supported */
+            else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) {
+                saw_r = 1;
+            }
+            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) {
+                saw_f = 1;
+            }
+            else {
+                break;
+            }
+            c = tok_nextc(tok);
+            if (c == '"' || c == '\'') {
+                goto letter_quote;
+            }
+        }
+        while (is_potential_identifier_char(c)) {
+            if (c >= 128) {
+                nonascii = 1;
+            }
+            c = tok_nextc(tok);
+        }
+        tok_backup(tok, c);
+        if (nonascii && !verify_identifier(tok)) {
+            return ERRORTOKEN;
+        }
+
+        *p_start = tok->start;
+        *p_end = tok->cur;
 
-        *p_start = tok->start; 
-        *p_end = tok->cur; 
- 
         /* async/await parsing block. */
         if (tok->cur - tok->start == 5 && tok->start[0] == 'a') {
             /* May be an 'async' or 'await' token.  For Python 3.7 or
@@ -1466,81 +1466,81 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
             }
         }
 
-        return NAME; 
-    } 
- 
-    /* Newline */ 
-    if (c == '\n') { 
-        tok->atbol = 1; 
-        if (blankline || tok->level > 0) { 
-            goto nextline; 
-        } 
-        *p_start = tok->start; 
-        *p_end = tok->cur - 1; /* Leave '\n' out of the string */ 
-        tok->cont_line = 0; 
+        return NAME;
+    }
+
+    /* Newline */
+    if (c == '\n') {
+        tok->atbol = 1;
+        if (blankline || tok->level > 0) {
+            goto nextline;
+        }
+        *p_start = tok->start;
+        *p_end = tok->cur - 1; /* Leave '\n' out of the string */
+        tok->cont_line = 0;
         if (tok->async_def) {
             /* We're somewhere inside an 'async def' function, and
                we've encountered a NEWLINE after its signature. */
             tok->async_def_nl = 1;
         }
-        return NEWLINE; 
-    } 
- 
-    /* Period or number starting with period? */ 
-    if (c == '.') { 
-        c = tok_nextc(tok); 
-        if (isdigit(c)) { 
-            goto fraction; 
-        } else if (c == '.') { 
-            c = tok_nextc(tok); 
-            if (c == '.') { 
-                *p_start = tok->start; 
-                *p_end = tok->cur; 
-                return ELLIPSIS; 
-            } 
-            else { 
-                tok_backup(tok, c); 
-            } 
-            tok_backup(tok, '.'); 
-        } 
-        else { 
-            tok_backup(tok, c); 
-        } 
-        *p_start = tok->start; 
-        *p_end = tok->cur; 
-        return DOT; 
-    } 
- 
-    /* Number */ 
-    if (isdigit(c)) { 
-        if (c == '0') { 
-            /* Hex, octal or binary -- maybe. */ 
-            c = tok_nextc(tok); 
-            if (c == 'x' || c == 'X') { 
-                /* Hex */ 
-                c = tok_nextc(tok); 
-                do { 
-                    if (c == '_') { 
-                        c = tok_nextc(tok); 
-                    } 
-                    if (!isxdigit(c)) { 
-                        tok_backup(tok, c); 
+        return NEWLINE;
+    }
+
+    /* Period or number starting with period? */
+    if (c == '.') {
+        c = tok_nextc(tok);
+        if (isdigit(c)) {
+            goto fraction;
+        } else if (c == '.') {
+            c = tok_nextc(tok);
+            if (c == '.') {
+                *p_start = tok->start;
+                *p_end = tok->cur;
+                return ELLIPSIS;
+            }
+            else {
+                tok_backup(tok, c);
+            }
+            tok_backup(tok, '.');
+        }
+        else {
+            tok_backup(tok, c);
+        }
+        *p_start = tok->start;
+        *p_end = tok->cur;
+        return DOT;
+    }
+
+    /* Number */
+    if (isdigit(c)) {
+        if (c == '0') {
+            /* Hex, octal or binary -- maybe. */
+            c = tok_nextc(tok);
+            if (c == 'x' || c == 'X') {
+                /* Hex */
+                c = tok_nextc(tok);
+                do {
+                    if (c == '_') {
+                        c = tok_nextc(tok);
+                    }
+                    if (!isxdigit(c)) {
+                        tok_backup(tok, c);
                         return syntaxerror(tok, "invalid hexadecimal literal");
-                    } 
-                    do { 
-                        c = tok_nextc(tok); 
-                    } while (isxdigit(c)); 
-                } while (c == '_'); 
-            } 
-            else if (c == 'o' || c == 'O') { 
-                /* Octal */ 
-                c = tok_nextc(tok); 
-                do { 
-                    if (c == '_') { 
-                        c = tok_nextc(tok); 
-                    } 
-                    if (c < '0' || c >= '8') { 
-                        tok_backup(tok, c); 
+                    }
+                    do {
+                        c = tok_nextc(tok);
+                    } while (isxdigit(c));
+                } while (c == '_');
+            }
+            else if (c == 'o' || c == 'O') {
+                /* Octal */
+                c = tok_nextc(tok);
+                do {
+                    if (c == '_') {
+                        c = tok_nextc(tok);
+                    }
+                    if (c < '0' || c >= '8') {
+                        tok_backup(tok, c);
                         if (isdigit(c)) {
                             return syntaxerror(tok,
                                     "invalid digit '%c' in octal literal", c);
@@ -1548,25 +1548,25 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
                         else {
                             return syntaxerror(tok, "invalid octal literal");
                         }
-                    } 
-                    do { 
-                        c = tok_nextc(tok); 
-                    } while ('0' <= c && c < '8'); 
-                } while (c == '_'); 
+                    }
+                    do {
+                        c = tok_nextc(tok);
+                    } while ('0' <= c && c < '8');
+                } while (c == '_');
                 if (isdigit(c)) {
                     return syntaxerror(tok,
                             "invalid digit '%c' in octal literal", c);
                 }
-            } 
-            else if (c == 'b' || c == 'B') { 
-                /* Binary */ 
-                c = tok_nextc(tok); 
-                do { 
-                    if (c == '_') { 
-                        c = tok_nextc(tok); 
-                    } 
-                    if (c != '0' && c != '1') { 
-                        tok_backup(tok, c); 
+            }
+            else if (c == 'b' || c == 'B') {
+                /* Binary */
+                c = tok_nextc(tok);
+                do {
+                    if (c == '_') {
+                        c = tok_nextc(tok);
+                    }
+                    if (c != '0' && c != '1') {
+                        tok_backup(tok, c);
                         if (isdigit(c)) {
                             return syntaxerror(tok,
                                     "invalid digit '%c' in binary literal", c);
@@ -1574,123 +1574,123 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
                         else {
                             return syntaxerror(tok, "invalid binary literal");
                         }
-                    } 
-                    do { 
-                        c = tok_nextc(tok); 
-                    } while (c == '0' || c == '1'); 
-                } while (c == '_'); 
+                    }
+                    do {
+                        c = tok_nextc(tok);
+                    } while (c == '0' || c == '1');
+                } while (c == '_');
                 if (isdigit(c)) {
                     return syntaxerror(tok,
                             "invalid digit '%c' in binary literal", c);
                 }
-            } 
-            else { 
-                int nonzero = 0; 
-                /* maybe old-style octal; c is first char of it */ 
-                /* in any case, allow '0' as a literal */ 
-                while (1) { 
-                    if (c == '_') { 
-                        c = tok_nextc(tok); 
-                        if (!isdigit(c)) { 
-                            tok_backup(tok, c); 
+            }
+            else {
+                int nonzero = 0;
+                /* maybe old-style octal; c is first char of it */
+                /* in any case, allow '0' as a literal */
+                while (1) {
+                    if (c == '_') {
+                        c = tok_nextc(tok);
+                        if (!isdigit(c)) {
+                            tok_backup(tok, c);
                             return syntaxerror(tok, "invalid decimal literal");
-                        } 
-                    } 
-                    if (c != '0') { 
-                        break; 
-                    } 
-                    c = tok_nextc(tok); 
-                } 
-                if (isdigit(c)) { 
-                    nonzero = 1; 
-                    c = tok_decimal_tail(tok); 
-                    if (c == 0) { 
-                        return ERRORTOKEN; 
-                    } 
-                } 
-                if (c == '.') { 
-                    c = tok_nextc(tok); 
-                    goto fraction; 
-                } 
-                else if (c == 'e' || c == 'E') { 
-                    goto exponent; 
-                } 
-                else if (c == 'j' || c == 'J') { 
-                    goto imaginary; 
-                } 
-                else if (nonzero) { 
-                    /* Old-style octal: now disallowed. */ 
-                    tok_backup(tok, c); 
+                        }
+                    }
+                    if (c != '0') {
+                        break;
+                    }
+                    c = tok_nextc(tok);
+                }
+                if (isdigit(c)) {
+                    nonzero = 1;
+                    c = tok_decimal_tail(tok);
+                    if (c == 0) {
+                        return ERRORTOKEN;
+                    }
+                }
+                if (c == '.') {
+                    c = tok_nextc(tok);
+                    goto fraction;
+                }
+                else if (c == 'e' || c == 'E') {
+                    goto exponent;
+                }
+                else if (c == 'j' || c == 'J') {
+                    goto imaginary;
+                }
+                else if (nonzero) {
+                    /* Old-style octal: now disallowed. */
+                    tok_backup(tok, c);
                     return syntaxerror(tok,
                                        "leading zeros in decimal integer "
                                        "literals are not permitted; "
                                        "use an 0o prefix for octal integers");
-                } 
-            } 
-        } 
-        else { 
-            /* Decimal */ 
-            c = tok_decimal_tail(tok); 
-            if (c == 0) { 
-                return ERRORTOKEN; 
-            } 
-            { 
-                /* Accept floating point numbers. */ 
-                if (c == '.') { 
-                    c = tok_nextc(tok); 
-        fraction: 
-                    /* Fraction */ 
-                    if (isdigit(c)) { 
-                        c = tok_decimal_tail(tok); 
-                        if (c == 0) { 
-                            return ERRORTOKEN; 
-                        } 
-                    } 
-                } 
-                if (c == 'e' || c == 'E') { 
-                    int e; 
-                  exponent: 
-                    e = c; 
-                    /* Exponent part */ 
-                    c = tok_nextc(tok); 
-                    if (c == '+' || c == '-') { 
-                        c = tok_nextc(tok); 
-                        if (!isdigit(c)) { 
-                            tok_backup(tok, c); 
+                }
+            }
+        }
+        else {
+            /* Decimal */
+            c = tok_decimal_tail(tok);
+            if (c == 0) {
+                return ERRORTOKEN;
+            }
+            {
+                /* Accept floating point numbers. */
+                if (c == '.') {
+                    c = tok_nextc(tok);
+        fraction:
+                    /* Fraction */
+                    if (isdigit(c)) {
+                        c = tok_decimal_tail(tok);
+                        if (c == 0) {
+                            return ERRORTOKEN;
+                        }
+                    }
+                }
+                if (c == 'e' || c == 'E') {
+                    int e;
+                  exponent:
+                    e = c;
+                    /* Exponent part */
+                    c = tok_nextc(tok);
+                    if (c == '+' || c == '-') {
+                        c = tok_nextc(tok);
+                        if (!isdigit(c)) {
+                            tok_backup(tok, c);
                             return syntaxerror(tok, "invalid decimal literal");
-                        } 
-                    } else if (!isdigit(c)) { 
-                        tok_backup(tok, c); 
-                        tok_backup(tok, e); 
-                        *p_start = tok->start; 
-                        *p_end = tok->cur; 
-                        return NUMBER; 
-                    } 
-                    c = tok_decimal_tail(tok); 
-                    if (c == 0) { 
-                        return ERRORTOKEN; 
-                    } 
-                } 
-                if (c == 'j' || c == 'J') { 
-                    /* Imaginary part */ 
-        imaginary: 
-                    c = tok_nextc(tok); 
-                } 
-            } 
-        } 
-        tok_backup(tok, c); 
-        *p_start = tok->start; 
-        *p_end = tok->cur; 
-        return NUMBER; 
-    } 
- 
-  letter_quote: 
-    /* String */ 
-    if (c == '\'' || c == '"') { 
-        int quote = c; 
-        int quote_size = 1;             /* 1 or 3 */ 
-        int end_quote_size = 0; 
- 
+                        }
+                    } else if (!isdigit(c)) {
+                        tok_backup(tok, c);
+                        tok_backup(tok, e);
+                        *p_start = tok->start;
+                        *p_end = tok->cur;
+                        return NUMBER;
+                    }
+                    c = tok_decimal_tail(tok);
+                    if (c == 0) {
+                        return ERRORTOKEN;
+                    }
+                }
+                if (c == 'j' || c == 'J') {
+                    /* Imaginary part */
+        imaginary:
+                    c = tok_nextc(tok);
+                }
+            }
+        }
+        tok_backup(tok, c);
+        *p_start = tok->start;
+        *p_end = tok->cur;
+        return NUMBER;
+    }
+
+  letter_quote:
+    /* String */
+    if (c == '\'' || c == '"') {
+        int quote = c;
+        int quote_size = 1;             /* 1 or 3 */
+        int end_quote_size = 0;
+
         /* Nodes of type STRING, especially multi line strings
            must be handled differently in order to get both
            the starting line number and the column offset right.
@@ -1698,62 +1698,62 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         tok->first_lineno = tok->lineno;
         tok->multi_line_start = tok->line_start;
 
-        /* Find the quote size and start of string */ 
-        c = tok_nextc(tok); 
-        if (c == quote) { 
-            c = tok_nextc(tok); 
-            if (c == quote) { 
-                quote_size = 3; 
-            } 
-            else { 
-                end_quote_size = 1;     /* empty string found */ 
-            } 
-        } 
-        if (c != quote) { 
-            tok_backup(tok, c); 
-        } 
- 
-        /* Get rest of string */ 
-        while (end_quote_size != quote_size) { 
-            c = tok_nextc(tok); 
-            if (c == EOF) { 
-                if (quote_size == 3) { 
-                    tok->done = E_EOFS; 
-                } 
-                else { 
-                    tok->done = E_EOLS; 
-                } 
-                tok->cur = tok->inp; 
-                return ERRORTOKEN; 
-            } 
-            if (quote_size == 1 && c == '\n') { 
-                tok->done = E_EOLS; 
-                tok->cur = tok->inp; 
-                return ERRORTOKEN; 
-            } 
-            if (c == quote) { 
-                end_quote_size += 1; 
-            } 
-            else { 
-                end_quote_size = 0; 
-                if (c == '\\') { 
-                    tok_nextc(tok);  /* skip escaped char */ 
-                } 
-            } 
-        } 
- 
-        *p_start = tok->start; 
-        *p_end = tok->cur; 
-        return STRING; 
-    } 
- 
-    /* Line continuation */ 
-    if (c == '\\') { 
-        c = tok_nextc(tok); 
-        if (c != '\n') { 
-            tok->done = E_LINECONT; 
-            return ERRORTOKEN; 
-        } 
+        /* Find the quote size and start of string */
+        c = tok_nextc(tok);
+        if (c == quote) {
+            c = tok_nextc(tok);
+            if (c == quote) {
+                quote_size = 3;
+            }
+            else {
+                end_quote_size = 1;     /* empty string found */
+            }
+        }
+        if (c != quote) {
+            tok_backup(tok, c);
+        }
+
+        /* Get rest of string */
+        while (end_quote_size != quote_size) {
+            c = tok_nextc(tok);
+            if (c == EOF) {
+                if (quote_size == 3) {
+                    tok->done = E_EOFS;
+                }
+                else {
+                    tok->done = E_EOLS;
+                }
+                tok->cur = tok->inp;
+                return ERRORTOKEN;
+            }
+            if (quote_size == 1 && c == '\n') {
+                tok->done = E_EOLS;
+                tok->cur = tok->inp;
+                return ERRORTOKEN;
+            }
+            if (c == quote) {
+                end_quote_size += 1;
+            }
+            else {
+                end_quote_size = 0;
+                if (c == '\\') {
+                    tok_nextc(tok);  /* skip escaped char */
+                }
+            }
+        }
+
+        *p_start = tok->start;
+        *p_end = tok->cur;
+        return STRING;
+    }
+
+    /* Line continuation */
+    if (c == '\\') {
+        c = tok_nextc(tok);
+        if (c != '\n') {
+            tok->done = E_LINECONT;
+            return ERRORTOKEN;
+        }
         c = tok_nextc(tok);
         if (c == EOF) {
             tok->done = E_EOF;
@@ -1762,49 +1762,49 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         } else {
             tok_backup(tok, c);
         }
-        tok->cont_line = 1; 
-        goto again; /* Read next line */ 
-    } 
- 
-    /* Check for two-character token */ 
-    { 
-        int c2 = tok_nextc(tok); 
-        int token = PyToken_TwoChars(c, c2); 
-        if (token != OP) { 
-            int c3 = tok_nextc(tok); 
-            int token3 = PyToken_ThreeChars(c, c2, c3); 
-            if (token3 != OP) { 
-                token = token3; 
-            } 
-            else { 
-                tok_backup(tok, c3); 
-            } 
-            *p_start = tok->start; 
-            *p_end = tok->cur; 
-            return token; 
-        } 
-        tok_backup(tok, c2); 
-    } 
- 
-    /* Keep track of parentheses nesting level */ 
-    switch (c) { 
-    case '(': 
-    case '[': 
-    case '{': 
+        tok->cont_line = 1;
+        goto again; /* Read next line */
+    }
+
+    /* Check for two-character token */
+    {
+        int c2 = tok_nextc(tok);
+        int token = PyToken_TwoChars(c, c2);
+        if (token != OP) {
+            int c3 = tok_nextc(tok);
+            int token3 = PyToken_ThreeChars(c, c2, c3);
+            if (token3 != OP) {
+                token = token3;
+            }
+            else {
+                tok_backup(tok, c3);
+            }
+            *p_start = tok->start;
+            *p_end = tok->cur;
+            return token;
+        }
+        tok_backup(tok, c2);
+    }
+
+    /* Keep track of parentheses nesting level */
+    switch (c) {
+    case '(':
+    case '[':
+    case '{':
         if (tok->level >= MAXLEVEL) {
             return syntaxerror(tok, "too many nested parentheses");
         }
         tok->parenstack[tok->level] = c;
         tok->parenlinenostack[tok->level] = tok->lineno;
-        tok->level++; 
-        break; 
-    case ')': 
-    case ']': 
-    case '}': 
+        tok->level++;
+        break;
+    case ')':
+    case ']':
+    case '}':
         if (!tok->level) {
             return syntaxerror(tok, "unmatched '%c'", c);
         }
-        tok->level--; 
+        tok->level--;
         int opening = tok->parenstack[tok->level];
         if (!((opening == '(' && c == ')') ||
               (opening == '[' && c == ']') ||
@@ -1823,98 +1823,98 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
                         c, opening);
             }
         }
-        break; 
-    } 
- 
-    /* Punctuation character */ 
-    *p_start = tok->start; 
-    *p_end = tok->cur; 
-    return PyToken_OneChar(c); 
-} 
- 
-int 
+        break;
+    }
+
+    /* Punctuation character */
+    *p_start = tok->start;
+    *p_end = tok->cur;
+    return PyToken_OneChar(c);
+}
+
+int
 PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
-{ 
-    int result = tok_get(tok, p_start, p_end); 
-    if (tok->decoding_erred) { 
-        result = ERRORTOKEN; 
-        tok->done = E_DECODE; 
-    } 
-    return result; 
-} 
- 
-/* Get the encoding of a Python file. Check for the coding cookie and check if 
-   the file starts with a BOM. 
- 
-   PyTokenizer_FindEncodingFilename() returns NULL when it can't find the 
-   encoding in the first or second line of the file (in which case the encoding 
-   should be assumed to be UTF-8). 
- 
-   The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed 
-   by the caller. */ 
- 
-char * 
-PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) 
-{ 
-    struct tok_state *tok; 
-    FILE *fp; 
+{
+    int result = tok_get(tok, p_start, p_end);
+    if (tok->decoding_erred) {
+        result = ERRORTOKEN;
+        tok->done = E_DECODE;
+    }
+    return result;
+}
+
+/* Get the encoding of a Python file. Check for the coding cookie and check if
+   the file starts with a BOM.
+
+   PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
+   encoding in the first or second line of the file (in which case the encoding
+   should be assumed to be UTF-8).
+
+   The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
+   by the caller. */
+
+char *
+PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
+{
+    struct tok_state *tok;
+    FILE *fp;
     const char *p_start = NULL;
     const char *p_end = NULL;
     char *encoding = NULL;
- 
-    fd = _Py_dup(fd); 
-    if (fd < 0) { 
-        return NULL; 
-    } 
- 
-    fp = fdopen(fd, "r"); 
-    if (fp == NULL) { 
-        return NULL; 
-    } 
-    tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL); 
-    if (tok == NULL) { 
-        fclose(fp); 
-        return NULL; 
-    } 
-    if (filename != NULL) { 
-        Py_INCREF(filename); 
-        tok->filename = filename; 
-    } 
-    else { 
-        tok->filename = PyUnicode_FromString("<string>"); 
-        if (tok->filename == NULL) { 
-            fclose(fp); 
-            PyTokenizer_Free(tok); 
-            return encoding; 
-        } 
-    } 
-    while (tok->lineno < 2 && tok->done == E_OK) { 
-        PyTokenizer_Get(tok, &p_start, &p_end); 
-    } 
-    fclose(fp); 
-    if (tok->encoding) { 
-        encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1); 
-        if (encoding) 
+
+    fd = _Py_dup(fd);
+    if (fd < 0) {
+        return NULL;
+    }
+
+    fp = fdopen(fd, "r");
+    if (fp == NULL) {
+        return NULL;
+    }
+    tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
+    if (tok == NULL) {
+        fclose(fp);
+        return NULL;
+    }
+    if (filename != NULL) {
+        Py_INCREF(filename);
+        tok->filename = filename;
+    }
+    else {
+        tok->filename = PyUnicode_FromString("<string>");
+        if (tok->filename == NULL) {
+            fclose(fp);
+            PyTokenizer_Free(tok);
+            return encoding;
+        }
+    }
+    while (tok->lineno < 2 && tok->done == E_OK) {
+        PyTokenizer_Get(tok, &p_start, &p_end);
+    }
+    fclose(fp);
+    if (tok->encoding) {
+        encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
+        if (encoding)
             strcpy(encoding, tok->encoding);
-    } 
-    PyTokenizer_Free(tok); 
-    return encoding; 
-} 
- 
-char * 
-PyTokenizer_FindEncoding(int fd) 
-{ 
-    return PyTokenizer_FindEncodingFilename(fd, NULL); 
-} 
- 
-#ifdef Py_DEBUG 
- 
-void 
-tok_dump(int type, char *start, char *end) 
-{ 
-    printf("%s", _PyParser_TokenNames[type]); 
-    if (type == NAME || type == NUMBER || type == STRING || type == OP) 
-        printf("(%.*s)", (int)(end - start), start); 
-} 
- 
-#endif 
+    }
+    PyTokenizer_Free(tok);
+    return encoding;
+}
+
+char *
+PyTokenizer_FindEncoding(int fd)
+{
+    return PyTokenizer_FindEncodingFilename(fd, NULL);
+}
+
+#ifdef Py_DEBUG
+
+void
+tok_dump(int type, char *start, char *end)
+{
+    printf("%s", _PyParser_TokenNames[type]);
+    if (type == NAME || type == NUMBER || type == STRING || type == OP)
+        printf("(%.*s)", (int)(end - start), start);
+}
+
+#endif
diff --git a/contrib/tools/python3/src/Parser/tokenizer.h b/contrib/tools/python3/src/Parser/tokenizer.h
index 06dc1106d9c..5660ea38e94 100644
--- a/contrib/tools/python3/src/Parser/tokenizer.h
+++ b/contrib/tools/python3/src/Parser/tokenizer.h
@@ -1,65 +1,65 @@
-#ifndef Py_TOKENIZER_H 
-#define Py_TOKENIZER_H 
-#ifdef __cplusplus 
-extern "C" { 
-#endif 
- 
-#include "object.h" 
- 
-/* Tokenizer interface */ 
- 
-#include "token.h"      /* For token types */ 
- 
-#define MAXINDENT 100   /* Max indentation level */ 
+#ifndef Py_TOKENIZER_H
+#define Py_TOKENIZER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "object.h"
+
+/* Tokenizer interface */
+
+#include "token.h"      /* For token types */
+
+#define MAXINDENT 100   /* Max indentation level */
 #define MAXLEVEL 200    /* Max parentheses level */
- 
-enum decoding_state { 
-    STATE_INIT, 
-    STATE_RAW, 
-    STATE_NORMAL        /* have a codec associated with input */ 
-}; 
- 
-/* Tokenizer state */ 
-struct tok_state { 
-    /* Input state; buf <= cur <= inp <= end */ 
-    /* NB an entire line is held in the buffer */ 
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL */ 
-    char *cur;          /* Next character in buffer */ 
-    char *inp;          /* End of data in buffer */ 
+
+enum decoding_state {
+    STATE_INIT,
+    STATE_RAW,
+    STATE_NORMAL        /* have a codec associated with input */
+};
+
+/* Tokenizer state */
+struct tok_state {
+    /* Input state; buf <= cur <= inp <= end */
+    /* NB an entire line is held in the buffer */
+    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL */
+    char *cur;          /* Next character in buffer */
+    char *inp;          /* End of data in buffer */
     const char *end;    /* End of input buffer if buf != NULL */
     const char *start;  /* Start of current token if not NULL */
-    int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */ 
-    /* NB If done != E_OK, cur must be == inp!!! */ 
-    FILE *fp;           /* Rest of input; NULL if tokenizing a string */ 
-    int tabsize;        /* Tab spacing */ 
-    int indent;         /* Current indentation index */ 
-    int indstack[MAXINDENT];            /* Stack of indents */ 
-    int atbol;          /* Nonzero if at begin of new line */ 
-    int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */ 
-    const char *prompt, *nextprompt;          /* For interactive prompting */ 
-    int lineno;         /* Current line number */ 
+    int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
+    /* NB If done != E_OK, cur must be == inp!!! */
+    FILE *fp;           /* Rest of input; NULL if tokenizing a string */
+    int tabsize;        /* Tab spacing */
+    int indent;         /* Current indentation index */
+    int indstack[MAXINDENT];            /* Stack of indents */
+    int atbol;          /* Nonzero if at begin of new line */
+    int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
+    const char *prompt, *nextprompt;          /* For interactive prompting */
+    int lineno;         /* Current line number */
     int first_lineno;   /* First line of a single line or multi line string
                            expression (cf. issue 16806) */
-    int level;          /* () [] {} Parentheses nesting level */ 
-            /* Used to allow free continuations inside them */ 
+    int level;          /* () [] {} Parentheses nesting level */
+            /* Used to allow free continuations inside them */
     char parenstack[MAXLEVEL];
     int parenlinenostack[MAXLEVEL];
     PyObject *filename;
-    /* Stuff for checking on different tab sizes */ 
-    int altindstack[MAXINDENT];         /* Stack of alternate indents */ 
-    /* Stuff for PEP 0263 */ 
-    enum decoding_state decoding_state; 
-    int decoding_erred;         /* whether erred in decoding  */ 
-    int read_coding_spec;       /* whether 'coding:...' has been read  */ 
-    char *encoding;         /* Source encoding. */ 
-    int cont_line;          /* whether we are in a continuation line. */ 
-    const char* line_start;     /* pointer to start of current line */ 
+    /* Stuff for checking on different tab sizes */
+    int altindstack[MAXINDENT];         /* Stack of alternate indents */
+    /* Stuff for PEP 0263 */
+    enum decoding_state decoding_state;
+    int decoding_erred;         /* whether erred in decoding  */
+    int read_coding_spec;       /* whether 'coding:...' has been read  */
+    char *encoding;         /* Source encoding. */
+    int cont_line;          /* whether we are in a continuation line. */
+    const char* line_start;     /* pointer to start of current line */
     const char* multi_line_start; /* pointer to start of first line of
                                      a single line or multi line string
                                      expression (cf. issue 16806) */
-    PyObject *decoding_readline; /* open(...).readline */ 
-    PyObject *decoding_buffer; 
-    const char* enc;        /* Encoding for the current str. */ 
+    PyObject *decoding_readline; /* open(...).readline */
+    PyObject *decoding_buffer;
+    const char* enc;        /* Encoding for the current str. */
     char* str;
     char* input;       /* Tokenizer's newline translated copy of the string. */
 
@@ -71,18 +71,18 @@ struct tok_state {
     int async_def_indent; /* Indentation level of the outermost 'async def'. */
     int async_def_nl;     /* =1 if the outermost 'async def' had at least one
                              NEWLINE token after it. */
-}; 
- 
-extern struct tok_state *PyTokenizer_FromString(const char *, int); 
-extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); 
-extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*, 
-                                              const char *, const char *); 
-extern void PyTokenizer_Free(struct tok_state *); 
+};
+
+extern struct tok_state *PyTokenizer_FromString(const char *, int);
+extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
+extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
+                                              const char *, const char *);
+extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, const char **, const char **);
- 
+
 #define tok_dump _Py_tok_dump
 
-#ifdef __cplusplus 
-} 
-#endif 
-#endif /* !Py_TOKENIZER_H */ 
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_TOKENIZER_H */