From 31f2a419764a8ba77c2a970cfc80056c6cd06756 Mon Sep 17 00:00:00 2001
From: shadchin <shadchin@yandex-team.com>
Date: Mon, 12 Feb 2024 07:53:52 +0300
Subject: Update Python from 3.11.8 to 3.12.2
---
contrib/tools/python3/src/Python/fileutils.c | 349 +++++++++++++++++++++++----
1 file changed, 305 insertions(+), 44 deletions(-)
(limited to 'contrib/tools/python3/src/Python/fileutils.c')
diff --git a/contrib/tools/python3/src/Python/fileutils.c b/contrib/tools/python3/src/Python/fileutils.c
index 79ce20bd72..45de2b891d 100644
--- a/contrib/tools/python3/src/Python/fileutils.c
+++ b/contrib/tools/python3/src/Python/fileutils.c
@@ -8,7 +8,13 @@
#ifdef MS_WINDOWS
# include <malloc.h>
# include <windows.h>
-# include <pathcch.h> // PathCchCombineEx
+# include <winioctl.h> // FILE_DEVICE_* constants
+# include "pycore_fileutils_windows.h" // FILE_STAT_BASIC_INFORMATION
+# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP)
+# define PATHCCH_ALLOW_LONG_PATHS 0x01
+# else
+# include <pathcch.h> // PathCchCombineEx
+# endif
extern int winerror_to_errno(int);
#endif
@@ -77,7 +83,8 @@ _Py_device_encoding(int fd)
if (!valid)
Py_RETURN_NONE;
-#if defined(MS_WINDOWS)
+#ifdef MS_WINDOWS
+#ifdef HAVE_WINDOWS_CONSOLE_IO
UINT cp;
if (fd == 0)
cp = GetConsoleCP();
@@ -92,6 +99,9 @@ _Py_device_encoding(int fd)
}
return PyUnicode_FromFormat("cp%u", (unsigned int)cp);
+#else
+ Py_RETURN_NONE;
+#endif /* HAVE_WINDOWS_CONSOLE_IO */
#else
if (_PyRuntime.preconfig.utf8_mode) {
_Py_DECLARE_STR(utf_8, "utf-8");
@@ -191,7 +201,7 @@ extern int _Py_normalize_encoding(const char *, char *, size_t);
Py_DecodeLocale() uses mbstowcs()
-1: unknown, need to call check_force_ascii() to get the value
*/
-static int force_ascii = -1;
+#define force_ascii (_PyRuntime.fileutils.force_ascii)
static int
check_force_ascii(void)
@@ -603,9 +613,9 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen,
return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
errors);
#else
- int use_utf8 = (Py_UTF8Mode == 1);
+ int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1);
#ifdef MS_WINDOWS
- use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
+ use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0);
#endif
if (use_utf8) {
return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
@@ -795,9 +805,9 @@ encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos,
return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
raw_malloc, errors);
#else
- int use_utf8 = (Py_UTF8Mode == 1);
+ int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1);
#ifdef MS_WINDOWS
- use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
+ use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0);
#endif
if (use_utf8) {
return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
@@ -1048,6 +1058,13 @@ FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out)
*time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t);
}
+static void
+LARGE_INTEGER_to_time_t_nsec(LARGE_INTEGER *in_ptr, time_t *time_out, int* nsec_out)
+{
+ *nsec_out = (int)(in_ptr->QuadPart % 10000000) * 100; /* FILETIME is in units of 100 nsec. */
+ *time_out = Py_SAFE_DOWNCAST((in_ptr->QuadPart / 10000000) - secs_between_epochs, __int64, time_t);
+}
+
void
_Py_time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr)
{
@@ -1077,33 +1094,127 @@ attributes_to_mode(DWORD attr)
return m;
}
+
+typedef union {
+ FILE_ID_128 id;
+ struct {
+ uint64_t st_ino;
+ uint64_t st_ino_high;
+ };
+} id_128_to_ino;
+
+
void
_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag,
+ FILE_BASIC_INFO *basic_info, FILE_ID_INFO *id_info,
struct _Py_stat_struct *result)
{
memset(result, 0, sizeof(*result));
result->st_mode = attributes_to_mode(info->dwFileAttributes);
result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow;
- result->st_dev = info->dwVolumeSerialNumber;
- result->st_rdev = result->st_dev;
- FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec);
- FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
- FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec);
+ result->st_dev = id_info ? id_info->VolumeSerialNumber : info->dwVolumeSerialNumber;
+ result->st_rdev = 0;
+ /* st_ctime is deprecated, but we preserve the legacy value in our caller, not here */
+ if (basic_info) {
+ LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&basic_info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec);
+ } else {
+ FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec);
+ FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
+ FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec);
+ }
result->st_nlink = info->nNumberOfLinks;
- result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow;
+
+ if (id_info) {
+ id_128_to_ino file_id;
+ file_id.id = id_info->FileId;
+ result->st_ino = file_id.st_ino;
+ result->st_ino_high = file_id.st_ino_high;
+ }
+ if (!result->st_ino && !result->st_ino_high) {
+ /* should only occur for DirEntry_from_find_data, in which case the
+ index is likely to be zero anyway. */
+ result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow;
+ }
+
/* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will
open other name surrogate reparse points without traversing them. To
detect/handle these, check st_file_attributes and st_reparse_tag. */
result->st_reparse_tag = reparse_tag;
if (info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT &&
reparse_tag == IO_REPARSE_TAG_SYMLINK) {
- /* first clear the S_IFMT bits */
- result->st_mode ^= (result->st_mode & S_IFMT);
- /* now set the bits that make this a symlink */
- result->st_mode |= S_IFLNK;
+ /* set the bits that make this a symlink */
+ result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK;
}
result->st_file_attributes = info->dwFileAttributes;
}
+
+void
+_Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *info,
+ struct _Py_stat_struct *result)
+{
+ memset(result, 0, sizeof(*result));
+ result->st_mode = attributes_to_mode(info->FileAttributes);
+ result->st_size = info->EndOfFile.QuadPart;
+ LARGE_INTEGER_to_time_t_nsec(&info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
+ LARGE_INTEGER_to_time_t_nsec(&info->LastAccessTime, &result->st_atime, &result->st_atime_nsec);
+ result->st_nlink = info->NumberOfLinks;
+ result->st_dev = info->VolumeSerialNumber.QuadPart;
+ /* File systems with less than 128-bits zero pad into this field */
+ id_128_to_ino file_id;
+ file_id.id = info->FileId128;
+ result->st_ino = file_id.st_ino;
+ result->st_ino_high = file_id.st_ino_high;
+ /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will
+ open other name surrogate reparse points without traversing them. To
+ detect/handle these, check st_file_attributes and st_reparse_tag. */
+ result->st_reparse_tag = info->ReparseTag;
+ if (info->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT &&
+ info->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
+ /* set the bits that make this a symlink */
+ result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK;
+ }
+ result->st_file_attributes = info->FileAttributes;
+ switch (info->DeviceType) {
+ case FILE_DEVICE_DISK:
+ case FILE_DEVICE_VIRTUAL_DISK:
+ case FILE_DEVICE_DFS:
+ case FILE_DEVICE_CD_ROM:
+ case FILE_DEVICE_CONTROLLER:
+ case FILE_DEVICE_DATALINK:
+ break;
+ case FILE_DEVICE_DISK_FILE_SYSTEM:
+ case FILE_DEVICE_CD_ROM_FILE_SYSTEM:
+ case FILE_DEVICE_NETWORK_FILE_SYSTEM:
+ result->st_mode = (result->st_mode & ~S_IFMT) | 0x6000; /* _S_IFBLK */
+ break;
+ case FILE_DEVICE_CONSOLE:
+ case FILE_DEVICE_NULL:
+ case FILE_DEVICE_KEYBOARD:
+ case FILE_DEVICE_MODEM:
+ case FILE_DEVICE_MOUSE:
+ case FILE_DEVICE_PARALLEL_PORT:
+ case FILE_DEVICE_PRINTER:
+ case FILE_DEVICE_SCREEN:
+ case FILE_DEVICE_SERIAL_PORT:
+ case FILE_DEVICE_SOUND:
+ result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFCHR;
+ break;
+ case FILE_DEVICE_NAMED_PIPE:
+ result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFIFO;
+ break;
+ default:
+ if (info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+ result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFDIR;
+ }
+ break;
+ }
+}
+
#endif
/* Return information about a file.
@@ -1123,6 +1234,9 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status)
{
#ifdef MS_WINDOWS
BY_HANDLE_FILE_INFORMATION info;
+ FILE_BASIC_INFO basicInfo;
+ FILE_ID_INFO idInfo;
+ FILE_ID_INFO *pIdInfo = &idInfo;
HANDLE h;
int type;
@@ -1154,16 +1268,20 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status)
return 0;
}
- if (!GetFileInformationByHandle(h, &info)) {
+ if (!GetFileInformationByHandle(h, &info) ||
+ !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo))) {
/* The Win32 error is already set, but we also set errno for
callers who expect it */
errno = winerror_to_errno(GetLastError());
return -1;
}
- _Py_attribute_data_to_stat(&info, 0, status);
- /* specific to fstat() */
- status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow;
+ if (!GetFileInformationByHandleEx(h, FileIdInfo, &idInfo, sizeof(idInfo))) {
+ /* Failed to get FileIdInfo, so do not pass it along */
+ pIdInfo = NULL;
+ }
+
+ _Py_attribute_data_to_stat(&info, 0, &basicInfo, pIdInfo, status);
return 0;
#else
return fstat(fd, status);
@@ -1244,18 +1362,12 @@ _Py_stat(PyObject *path, struct stat *statbuf)
#ifdef MS_WINDOWS
int err;
-#if USE_UNICODE_WCHAR_CACHE
- const wchar_t *wpath = _PyUnicode_AsUnicode(path);
-#else /* USE_UNICODE_WCHAR_CACHE */
wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
-#endif /* USE_UNICODE_WCHAR_CACHE */
if (wpath == NULL)
return -2;
err = _Py_wstat(wpath, statbuf);
-#if !USE_UNICODE_WCHAR_CACHE
PyMem_Free(wpath);
-#endif /* USE_UNICODE_WCHAR_CACHE */
return err;
#else
int ret;
@@ -1278,6 +1390,13 @@ _Py_stat(PyObject *path, struct stat *statbuf)
#endif
}
+#ifdef MS_WINDOWS
+// For some Windows API partitions, SetHandleInformation() is declared
+// but none of the handle flags are defined.
+#ifndef HANDLE_FLAG_INHERIT
+#define HANDLE_FLAG_INHERIT 0x00000001
+#endif
+#endif
/* This function MUST be kept async-signal-safe on POSIX when raise=0. */
static int
@@ -1370,17 +1489,11 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
else
flags = 0;
- /* This check can be removed once support for Windows 7 ends. */
-#define CONSOLE_PSEUDOHANDLE(handle) (((ULONG_PTR)(handle) & 0x3) == 0x3 && \
- GetFileType(handle) == FILE_TYPE_CHAR)
-
- if (!CONSOLE_PSEUDOHANDLE(handle) &&
- !SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags)) {
+ if (!SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags)) {
if (raise)
PyErr_SetFromWindowsErr(0);
return -1;
}
-#undef CONSOLE_PSEUDOHANDLE
return 0;
#else
@@ -1663,11 +1776,8 @@ _Py_fopen_obj(PyObject *path, const char *mode)
Py_TYPE(path));
return NULL;
}
-#if USE_UNICODE_WCHAR_CACHE
- const wchar_t *wpath = _PyUnicode_AsUnicode(path);
-#else /* USE_UNICODE_WCHAR_CACHE */
+
wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
-#endif /* USE_UNICODE_WCHAR_CACHE */
if (wpath == NULL)
return NULL;
@@ -1675,9 +1785,7 @@ _Py_fopen_obj(PyObject *path, const char *mode)
wmode, Py_ARRAY_LENGTH(wmode));
if (usize == 0) {
PyErr_SetFromWindowsErr(0);
-#if !USE_UNICODE_WCHAR_CACHE
PyMem_Free(wpath);
-#endif /* USE_UNICODE_WCHAR_CACHE */
return NULL;
}
@@ -1688,9 +1796,7 @@ _Py_fopen_obj(PyObject *path, const char *mode)
} while (f == NULL
&& errno == EINTR && !(async_err = PyErr_CheckSignals()));
int saved_errno = errno;
-#if !USE_UNICODE_WCHAR_CACHE
PyMem_Free(wpath);
-#endif /* USE_UNICODE_WCHAR_CACHE */
#else
PyObject *bytes;
const char *path_bytes;
@@ -1767,7 +1873,15 @@ _Py_read(int fd, void *buf, size_t count)
Py_BEGIN_ALLOW_THREADS
errno = 0;
#ifdef MS_WINDOWS
+ _doserrno = 0;
n = read(fd, buf, (int)count);
+ // read() on a non-blocking empty pipe fails with EINVAL, which is
+ // mapped from the Windows error code ERROR_NO_DATA.
+ if (n < 0 && errno == EINVAL) {
+ if (_doserrno == ERROR_NO_DATA) {
+ errno = EAGAIN;
+ }
+ }
#else
n = read(fd, buf, count);
#endif
@@ -1821,6 +1935,7 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
}
}
}
+
#endif
if (count > _PY_WRITE_MAX) {
count = _PY_WRITE_MAX;
@@ -1831,7 +1946,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
Py_BEGIN_ALLOW_THREADS
errno = 0;
#ifdef MS_WINDOWS
- n = write(fd, buf, (int)count);
+ // write() on a non-blocking pipe fails with ENOSPC on Windows if
+ // the pipe lacks available space for the entire buffer.
+ int c = (int)count;
+ do {
+ _doserrno = 0;
+ n = write(fd, buf, c);
+ if (n >= 0 || errno != ENOSPC || _doserrno != 0) {
+ break;
+ }
+ errno = EAGAIN;
+ c /= 2;
+ } while (c > 0);
#else
n = write(fd, buf, count);
#endif
@@ -1846,7 +1972,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
do {
errno = 0;
#ifdef MS_WINDOWS
- n = write(fd, buf, (int)count);
+ // write() on a non-blocking pipe fails with ENOSPC on Windows if
+ // the pipe lacks available space for the entire buffer.
+ int c = (int)count;
+ do {
+ _doserrno = 0;
+ n = write(fd, buf, c);
+ if (n >= 0 || errno != ENOSPC || _doserrno != 0) {
+ break;
+ }
+ errno = EAGAIN;
+ c /= 2;
+ } while (c > 0);
#else
n = write(fd, buf, count);
#endif
@@ -2088,6 +2225,72 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
#endif
}
+// The Windows Games API family implements the PathCch* APIs in the Xbox OS,
+// but does not expose them yet. Load them dynamically until
+// 1) they are officially exposed
+// 2) we stop supporting older versions of the GDK which do not expose them
+#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP)
+HRESULT
+PathCchSkipRoot(const wchar_t *path, const wchar_t **rootEnd)
+{
+ static int initialized = 0;
+ typedef HRESULT(__stdcall *PPathCchSkipRoot) (PCWSTR pszPath,
+ PCWSTR *ppszRootEnd);
+ static PPathCchSkipRoot _PathCchSkipRoot;
+
+ if (initialized == 0) {
+ HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL,
+ LOAD_LIBRARY_SEARCH_SYSTEM32);
+ if (pathapi) {
+ _PathCchSkipRoot = (PPathCchSkipRoot)GetProcAddress(
+ pathapi, "PathCchSkipRoot");
+ }
+ else {
+ _PathCchSkipRoot = NULL;
+ }
+ initialized = 1;
+ }
+
+ if (!_PathCchSkipRoot) {
+ return E_NOINTERFACE;
+ }
+
+ return _PathCchSkipRoot(path, rootEnd);
+}
+
+static HRESULT
+PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname,
+ const wchar_t *relfile, unsigned long flags)
+{
+ static int initialized = 0;
+ typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut,
+ size_t cchPathOut,
+ PCWSTR pszPathIn,
+ PCWSTR pszMore,
+ unsigned long dwFlags);
+ static PPathCchCombineEx _PathCchCombineEx;
+
+ if (initialized == 0) {
+ HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL,
+ LOAD_LIBRARY_SEARCH_SYSTEM32);
+ if (pathapi) {
+ _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress(
+ pathapi, "PathCchCombineEx");
+ }
+ else {
+ _PathCchCombineEx = NULL;
+ }
+ initialized = 1;
+ }
+
+ if (!_PathCchCombineEx) {
+ return E_NOINTERFACE;
+ }
+
+ return _PathCchCombineEx(buffer, bufsize, dirname, relfile, flags);
+}
+
+#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */
// The caller must ensure "buffer" is big enough.
static int
@@ -2484,6 +2687,64 @@ error:
return -1;
}
#else /* MS_WINDOWS */
+int
+_Py_get_blocking(int fd)
+{
+ HANDLE handle;
+ DWORD mode;
+ BOOL success;
+
+ handle = _Py_get_osfhandle(fd);
+ if (handle == INVALID_HANDLE_VALUE) {
+ return -1;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ success = GetNamedPipeHandleStateW(handle, &mode,
+ NULL, NULL, NULL, NULL, 0);
+ Py_END_ALLOW_THREADS
+
+ if (!success) {
+ PyErr_SetFromWindowsErr(0);
+ return -1;
+ }
+
+ return !(mode & PIPE_NOWAIT);
+}
+
+int
+_Py_set_blocking(int fd, int blocking)
+{
+ HANDLE handle;
+ DWORD mode;
+ BOOL success;
+
+ handle = _Py_get_osfhandle(fd);
+ if (handle == INVALID_HANDLE_VALUE) {
+ return -1;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ success = GetNamedPipeHandleStateW(handle, &mode,
+ NULL, NULL, NULL, NULL, 0);
+ if (success) {
+ if (blocking) {
+ mode &= ~PIPE_NOWAIT;
+ }
+ else {
+ mode |= PIPE_NOWAIT;
+ }
+ success = SetNamedPipeHandleState(handle, &mode, NULL, NULL);
+ }
+ Py_END_ALLOW_THREADS
+
+ if (!success) {
+ PyErr_SetFromWindowsErr(0);
+ return -1;
+ }
+ return 0;
+}
+
void*
_Py_get_osfhandle_noraise(int fd)
{
--
cgit v1.2.3