diff options
| author | Alexander Smirnov <[email protected]> | 2025-02-06 00:51:36 +0000 |
|---|---|---|
| committer | Alexander Smirnov <[email protected]> | 2025-02-06 00:51:36 +0000 |
| commit | 56094433309e9ee14b6be144db40dd8c200df4c6 (patch) | |
| tree | e331c04c654370c8d6d5578b8c81a7828c254eaf | |
| parent | 95fac681074068aad4d6bc5a02b1d7a153de5a09 (diff) | |
| parent | 37bab6825374df518e0b1cb748ce7bd2cef55688 (diff) | |
Merge branch 'rightlib' into merge-libs-250206-0050
347 files changed, 4614 insertions, 39940 deletions
diff --git a/build/conf/java.conf b/build/conf/java.conf index 38c1ead5e3b..913a5f90224 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -357,6 +357,7 @@ module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { PEERDIR+=$JDK_RESOURCE_PEERDIR $EXTERNAL_JAVA_EXTRA_PEERDIR when ($KOTLIN_PROTO == "yes") { PEERDIR+=build/platform/java/kotlin contrib/java/org/jetbrains/kotlin/kotlin-stdlib-jdk8/${_KOTLIN_VERSION} + _WITH_KOTLIN_PROTO_SEM=$_DO_WITH_KOTLIN_PROTO_SEM } when ($MAVEN_EXPORT == "yes") { @@ -2017,6 +2018,7 @@ DETEKT_VERSION=1.23.7 _KOTLIN_SEM= \ ${_WITH_KOTLIN_SEM} \ ${_KOTLIN_VERSION_SEM} \ + ${_WITH_KOTLIN_PROTO_SEM} \ ${_WITH_KOTLIN_GRPC_SEM} \ ${_WITH_KOTLINC_PLUGIN_ALLOPEN_SEM} \ ${_WITH_KOTLINC_PLUGIN_LOMBOK_SEM} \ @@ -2092,18 +2094,9 @@ macro JAVA_DEPENDENCIES_CONFIGURATION(Args...) { } # tag:java-specific -JAR_INCLUDE_FILTER_VALUE= JAR_EXCLUDE_FILTER_VALUE= # tag:java-specific -### @usage: JAR_INCLUDE(Filters...) -### Filter .jar file content: keep only matched files -### * and ** patterns are supported (like JAVA_SRCS) -macro JAR_INCLUDE(Filters...) { - SET_APPEND(JAR_INCLUDE_FILTER_VALUE $Filters) -} - -# tag:java-specific ### @usage: JAR_EXCLUDE(Filters...) ### Filter .jar file content: remove matched files ### * and ** patterns are supported (like JAVA_SRCS) diff --git a/build/conf/proto.conf b/build/conf/proto.conf index 2ac19f9446b..356293d9d96 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -674,6 +674,8 @@ module _CPP_PROTO : LIBRARY { } } +_WITH_KOTLIN_PROTO_SEM= +_DO_WITH_KOTLIN_PROTO_SEM=&& with_kotlin && kotlin_version ${_KOTLIN_VERSION} && proto_kotlin module _JAVA_PROTO: EXTERNAL_JAVA_LIBRARY { .EXTS=.jsrc .ALLOWED=GRPC @@ -685,6 +687,7 @@ module _JAVA_PROTO: EXTERNAL_JAVA_LIBRARY { when ($KOTLIN_PROTO == "yes") { KOTLIN_PROTO_PEERS=contrib/java/com/google/protobuf/protobuf-kotlin/${JAVA_PROTO_RUNTIME_VERSION} KOTLIN_PROTO_FLAGS=--kotlin_out=$ARCADIA_BUILD_ROOT/java_out + _WITH_KOTLIN_PROTO_SEM=$_DO_WITH_KOTLIN_PROTO_SEM } when ($GRADLE_EXPORT_PUBLISHING == "yes") { _GRADLE_EXPORT_PUBLISHING_SEM=$_DO_GRADLE_EXPORT_PUBLISHING_SEM diff --git a/build/conf/settings.conf b/build/conf/settings.conf index a9be913dd36..23186f34778 100644 --- a/build/conf/settings.conf +++ b/build/conf/settings.conf @@ -125,8 +125,6 @@ _FOLDABLE_VARS=\ # end of _FOLDABLE_VARS list -_MODULE_SCOPE_ONLY_VARS=PIC - ARCADIA_TEST_ROOT=../arcadia_tests_data/ DEFAULT_REQUIREMENTS=network:restricted cpu:1 ram:32 diff --git a/build/export_generators/gradle/generator.toml b/build/export_generators/gradle/generator.toml index f447c1ed6d5..4c2b464c189 100644 --- a/build/export_generators/gradle/generator.toml +++ b/build/export_generators/gradle/generator.toml @@ -25,6 +25,7 @@ add_vcs_info_to_mf="bool" junit4_test="flag" junit5_test="flag" app_main_class="str" +jar_source_set="list" enable_preview="flag" use_annotation_processor="list" use_errorprone="flag" @@ -43,6 +44,9 @@ proto_common_version="str" proto_grpc="flag" proto_grpc_version="str" +proto_kotlin="flag" +proto_kotlin_grpc="flag" +proto_kotlin_grpc_version="str" kotlin_version="str" with_kotlin="flag" @@ -51,6 +55,27 @@ with_kotlinc_plugin_lombok="set" with_kotlinc_plugin_noarg="set" with_kotlinc_plugin_serialization="set" +runs="list" +runs-args="list" +runs-classpath="list" +runs-cwd="str" +runs-in="list" +runs-in_dir="list" +runs-in_dirs_inputs="list" +runs-in_noparse="list" +runs-out="list" +runs-out_dir="list" +runs-tool="list" + +custom_runs="list" +custom_runs-depends="list" +custom_runs-env="list" +custom_runs-command="list" +custom_runs-outputs="list" +custom_runs-cwd="str" + +javac-flags="list" + [attrs.root] [attrs.dir] diff --git a/build/export_generators/ide-gradle/generator.toml b/build/export_generators/ide-gradle/generator.toml index cbb34c45e47..6e7d7476339 100644 --- a/build/export_generators/ide-gradle/generator.toml +++ b/build/export_generators/ide-gradle/generator.toml @@ -47,6 +47,7 @@ proto_common_version="str" proto_grpc="flag" proto_grpc_version="str" +proto_kotlin="flag" proto_kotlin_grpc="flag" proto_kotlin_grpc_version="str" diff --git a/build/export_generators/ide-gradle/proto_plugins.jinja b/build/export_generators/ide-gradle/proto_plugins.jinja index 655614672d0..28b9910492a 100644 --- a/build/export_generators/ide-gradle/proto_plugins.jinja +++ b/build/export_generators/ide-gradle/proto_plugins.jinja @@ -1,11 +1,14 @@ {#- empty string #} plugins { id("java-library") - id("com.google.protobuf") version "0.8.19" + id("com.google.protobuf") version "0.9.4" {%- if publish %} `maven-publish` `signing` {%- endif %} +{%- if with_kotlin and kotlin_version %} + kotlin("jvm") version "{{ kotlin_version }}" +{%- endif %} } {%- if jdk_version %} diff --git a/build/export_generators/ide-gradle/proto_prepare.jinja b/build/export_generators/ide-gradle/proto_prepare.jinja index 2ac84b5014b..cf6fc96adf8 100644 --- a/build/export_generators/ide-gradle/proto_prepare.jinja +++ b/build/export_generators/ide-gradle/proto_prepare.jinja @@ -1,4 +1,5 @@ {#- empty string #} +{%- if target.proto_files|length %} val prepareMainProtos = tasks.register<Copy>("prepareMainProtos") { from("$project_root") { {#- list of all current project proto files -#} @@ -8,6 +9,7 @@ val prepareMainProtos = tasks.register<Copy>("prepareMainProtos") { } into(mainProtosDir) } +{%- endif %} {% if libraries|length -%} val extractMainLibrariesProtos = tasks.register<Copy>("extractMainLibrariesProtos") { @@ -23,10 +25,15 @@ val extractMainLibrariesProtos = tasks.register<Copy>("extractMainLibrariesProto {% endif -%} afterEvaluate { +{%- if target.proto_files|length %} tasks.getByName("extractProto").dependsOn(prepareMainProtos) +{%- endif %} {%- if libraries|length %} tasks.getByName("extractProto").dependsOn(extractMainLibrariesProtos) {%- endif %} } tasks.getByName("sourcesJar").dependsOn("generateProto") +{%- if with_kotlin %} +tasks.getByName("compileKotlin").dependsOn("generateProto") +{%- endif %} diff --git a/build/export_generators/ide-gradle/proto_source_sets.jinja b/build/export_generators/ide-gradle/proto_source_sets.jinja index fffe7301a65..7c7814d940e 100644 --- a/build/export_generators/ide-gradle/proto_source_sets.jinja +++ b/build/export_generators/ide-gradle/proto_source_sets.jinja @@ -1,15 +1,15 @@ {#- empty string #} sourceSets { main { - java.srcDir("$buildDir/generated/sources/proto/main/java") + java.srcDir("$buildDir/generated/source/proto/main/java") {%- if target.proto_grpc %} - java.srcDir("$buildDir/generated/sources/proto/main/grpc") + java.srcDir("$buildDir/generated/source/proto/main/grpc") {%- endif %} } test { - java.srcDir("$buildDir/generated/sources/proto/test/java") + java.srcDir("$buildDir/generated/source/proto/test/java") {%- if target.proto_grpc %} - java.srcDir("$buildDir/generated/sources/proto/test/grpc") + java.srcDir("$buildDir/generated/source/proto/test/grpc") {%- endif %} } } @@ -20,3 +20,7 @@ sourceSets { sourceSets.main.java.srcDirs += "{{ srcdir_glob[0] }}" {% endfor -%} {%- endif %} + +tasks.withType<Jar>() { + duplicatesStrategy = DuplicatesStrategy.INCLUDE +} diff --git a/build/export_generators/ide-gradle/proto_vars.jinja b/build/export_generators/ide-gradle/proto_vars.jinja index 761c74b070a..3ef6275d879 100644 --- a/build/export_generators/ide-gradle/proto_vars.jinja +++ b/build/export_generators/ide-gradle/proto_vars.jinja @@ -1,4 +1,6 @@ {%- set publish = target.publish -%} {%- set libraries = target.consumer|selectattr('type', 'eq', 'library') -%} +{%- set with_kotlin = target.with_kotlin -%} +{%- set kotlin_version = target.kotlin_version -%} {%- include "[generator]/jdk.jinja" -%} diff --git a/build/export_generators/ide-gradle/protobuf.jinja b/build/export_generators/ide-gradle/protobuf.jinja index 68032069572..d2935b18b41 100644 --- a/build/export_generators/ide-gradle/protobuf.jinja +++ b/build/export_generators/ide-gradle/protobuf.jinja @@ -1,7 +1,5 @@ {#- empty string #} protobuf { - generatedFilesBaseDir = "$buildDir/generated/sources/proto" - protoc { // Download from repositories artifact = "com.google.protobuf:protoc:{%- if target.proto_compiler_version -%}{{ target.proto_compiler_version }}{%- else -%}3.22.5{%- endif -%}" @@ -18,16 +16,21 @@ protobuf { } {%- endif %} } +{%- endif %} + +{%- if target.proto_kotlin or target.proto_grpc %} generateProtoTasks { all().forEach { +{%- if target.proto_grpc %} it.plugins { id("grpc") -{%- if target.proto_kotlin_grpc %} +{%- if target.proto_kotlin_grpc %} id("grpckt") -{%- endif %} +{%- endif %} } -{%- if target.proto_kotlin_grpc %} +{%- endif %} +{%- if target.proto_kotlin or target.proto_kotlin_grpc %} it.builtins { create("kotlin") } diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index afe604cceba..cf64b9c310f 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7922609627" + "uri": "sbr:7949054326" }, "darwin-arm64": { - "uri": "sbr:7922609038" + "uri": "sbr:7949053519" }, "linux": { - "uri": "sbr:7922610908" + "uri": "sbr:7949055512" }, "linux-aarch64": { - "uri": "sbr:7922608408" + "uri": "sbr:7949052736" }, "win32-clang-cl": { - "uri": "sbr:7922610123" + "uri": "sbr:7949054892" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 1f38d150deb..dc8f04e8a70 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7922586572" + "uri": "sbr:7949059406" }, "darwin-arm64": { - "uri": "sbr:7922585908" + "uri": "sbr:7949058862" }, "linux": { - "uri": "sbr:7922588023" + "uri": "sbr:7949061040" }, "linux-aarch64": { - "uri": "sbr:7922584959" + "uri": "sbr:7949058277" }, "win32-clang-cl": { - "uri": "sbr:7922587360" + "uri": "sbr:7949060018" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 57ac821fa5d..7d6e42c543e 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -684,6 +684,7 @@ "7851725506": "https://devtools-registry.s3.yandex.net/7851725506", "7900623167": "https://devtools-registry.s3.yandex.net/7900623167", "7922609627": "https://devtools-registry.s3.yandex.net/7922609627", + "7949054326": "https://devtools-registry.s3.yandex.net/7949054326", "5766171800": "https://devtools-registry.s3.yandex.net/5766171800", "5805430761": "https://devtools-registry.s3.yandex.net/5805430761", "5829025456": "https://devtools-registry.s3.yandex.net/5829025456", @@ -747,6 +748,7 @@ "7851725041": "https://devtools-registry.s3.yandex.net/7851725041", "7900622686": "https://devtools-registry.s3.yandex.net/7900622686", "7922609038": "https://devtools-registry.s3.yandex.net/7922609038", + "7949053519": "https://devtools-registry.s3.yandex.net/7949053519", "5766173070": "https://devtools-registry.s3.yandex.net/5766173070", "5805432830": "https://devtools-registry.s3.yandex.net/5805432830", "5829031598": "https://devtools-registry.s3.yandex.net/5829031598", @@ -810,6 +812,7 @@ "7851726171": "https://devtools-registry.s3.yandex.net/7851726171", "7900624427": "https://devtools-registry.s3.yandex.net/7900624427", "7922610908": "https://devtools-registry.s3.yandex.net/7922610908", + "7949055512": "https://devtools-registry.s3.yandex.net/7949055512", "5766171341": "https://devtools-registry.s3.yandex.net/5766171341", "5805430188": "https://devtools-registry.s3.yandex.net/5805430188", "5829023352": "https://devtools-registry.s3.yandex.net/5829023352", @@ -873,6 +876,7 @@ "7851724808": "https://devtools-registry.s3.yandex.net/7851724808", "7900622150": "https://devtools-registry.s3.yandex.net/7900622150", "7922608408": "https://devtools-registry.s3.yandex.net/7922608408", + "7949052736": "https://devtools-registry.s3.yandex.net/7949052736", "5766172695": "https://devtools-registry.s3.yandex.net/5766172695", "5805432230": "https://devtools-registry.s3.yandex.net/5805432230", "5829029743": "https://devtools-registry.s3.yandex.net/5829029743", @@ -936,6 +940,7 @@ "7851725913": "https://devtools-registry.s3.yandex.net/7851725913", "7900623779": "https://devtools-registry.s3.yandex.net/7900623779", "7922610123": "https://devtools-registry.s3.yandex.net/7922610123", + "7949054892": "https://devtools-registry.s3.yandex.net/7949054892", "4307890075": "https://devtools-registry.s3.yandex.net/4307890075", "5517245192": "https://devtools-registry.s3.yandex.net/5517245192", "4307901240": "https://devtools-registry.s3.yandex.net/4307901240", @@ -1900,6 +1905,7 @@ "7851725506": "devtools/ymake/bin/ymake for darwin", "7900623167": "devtools/ymake/bin/ymake for darwin", "7922609627": "devtools/ymake/bin/ymake for darwin", + "7949054326": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -1963,6 +1969,7 @@ "7851725041": "devtools/ymake/bin/ymake for darwin-arm64", "7900622686": "devtools/ymake/bin/ymake for darwin-arm64", "7922609038": "devtools/ymake/bin/ymake for darwin-arm64", + "7949053519": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -2026,6 +2033,7 @@ "7851726171": "devtools/ymake/bin/ymake for linux", "7900624427": "devtools/ymake/bin/ymake for linux", "7922610908": "devtools/ymake/bin/ymake for linux", + "7949055512": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -2089,6 +2097,7 @@ "7851724808": "devtools/ymake/bin/ymake for linux-aarch64", "7900622150": "devtools/ymake/bin/ymake for linux-aarch64", "7922608408": "devtools/ymake/bin/ymake for linux-aarch64", + "7949052736": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -2152,6 +2161,7 @@ "7851725913": "devtools/ymake/bin/ymake for win32-clang-cl", "7900623779": "devtools/ymake/bin/ymake for win32-clang-cl", "7922610123": "devtools/ymake/bin/ymake for win32-clang-cl", + "7949054892": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", diff --git a/build/plugins/java.py b/build/plugins/java.py index 12789097c62..4b356e3bd79 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -118,26 +118,6 @@ def onjava_module(unit, *args): if valid: data['EXTERNAL_DEPENDENCIES'] = [valid] - if unit.get('MAKE_UBERJAR_VALUE') == 'yes': - if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM': - ymake.report_configure_error('{}: UBERJAR supported only for JAVA_PROGRAM module type'.format(unit.path())) - data['UBERJAR'] = extract_macro_calls(unit, 'MAKE_UBERJAR_VALUE', args_delim) - data['UBERJAR_PREFIX'] = extract_macro_calls(unit, 'UBERJAR_PREFIX_VALUE', args_delim) - data['UBERJAR_HIDE_EXCLUDE'] = extract_macro_calls(unit, 'UBERJAR_HIDE_EXCLUDE_VALUE', args_delim) - data['UBERJAR_PATH_EXCLUDE'] = extract_macro_calls(unit, 'UBERJAR_PATH_EXCLUDE_VALUE', args_delim) - data['UBERJAR_MANIFEST_TRANSFORMER_MAIN'] = extract_macro_calls( - unit, 'UBERJAR_MANIFEST_TRANSFORMER_MAIN_VALUE', args_delim - ) - data['UBERJAR_MANIFEST_TRANSFORMER_ATTRIBUTE'] = extract_macro_calls( - unit, 'UBERJAR_MANIFEST_TRANSFORMER_ATTRIBUTE_VALUE', args_delim - ) - data['UBERJAR_APPENDING_TRANSFORMER'] = extract_macro_calls( - unit, 'UBERJAR_APPENDING_TRANSFORMER_VALUE', args_delim - ) - data['UBERJAR_SERVICES_RESOURCE_TRANSFORMER'] = extract_macro_calls( - unit, 'UBERJAR_SERVICES_RESOURCE_TRANSFORMER_VALUE', args_delim - ) - if unit.get('WITH_JDK_VALUE') == 'yes': if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM': ymake.report_configure_error( diff --git a/build/sysincl/emscripten.yml b/build/sysincl/emscripten.yml index cf3eb5c5221..2ac68cc16c3 100644 --- a/build/sysincl/emscripten.yml +++ b/build/sysincl/emscripten.yml @@ -33,9 +33,6 @@ - string.h: - contrib/restricted/emscripten/system/lib/libc/musl/include/string.h - contrib/restricted/emscripten/system/lib/libc/musl/src/include/string.h - - sys/auxv.h: - - contrib/restricted/emscripten/system/lib/libc/musl/include/sys/auxv.h - - contrib/restricted/emscripten/system/lib/libc/musl/src/include/sys/auxv.h - sys/membarrier.h: - contrib/restricted/emscripten/system/lib/libc/musl/include/sys/membarrier.h - contrib/restricted/emscripten/system/lib/libc/musl/src/include/sys/membarrier.h @@ -49,6 +46,7 @@ - contrib/restricted/emscripten/system/lib/libc/musl/include/sys/time.h - contrib/restricted/emscripten/system/lib/libc/musl/src/include/sys/time.h - time.h: + - contrib/restricted/emscripten/system/include/compat/time.h - contrib/restricted/emscripten/system/lib/libc/musl/include/time.h - contrib/restricted/emscripten/system/lib/libc/musl/src/include/time.h @@ -214,12 +212,10 @@ - strings.h: contrib/restricted/emscripten/system/lib/libc/musl/include/strings.h - stropts.h: contrib/restricted/emscripten/system/lib/libc/musl/include/stropts.h - sys/acct.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/acct.h - - sys/auxv.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/auxv.h - sys/cachectl.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/cachectl.h - sys/dir.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/dir.h - sys/epoll.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/epoll.h - sys/errno.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/errno.h - - sys/eventfd.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/eventfd.h - sysexits.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sysexits.h - sys/fcntl.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/fcntl.h - sys/file.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/file.h @@ -237,7 +233,6 @@ - sys/param.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/param.h - sys/personality.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/personality.h - sys/poll.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/poll.h - - sys/prctl.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/prctl.h - sys/procfs.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/procfs.h - sys/ptrace.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/ptrace.h - sys/quota.h: contrib/restricted/emscripten/system/lib/libc/musl/include/sys/quota.h diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 41dcbab4f92..dccb35fe3de 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -435,6 +435,10 @@ when ($ARCH_XTENSA == "yes") { DEFAULT_ALLOCATOR=FAKE } +when ($OS_EMSCRIPTEN == "yes") { + DEFAULT_ALLOCATOR=FAKE +} + # tag:internal ### @usage: CHECK_CONFIG_H(<conf_header>) # internal ### @@ -1297,6 +1301,12 @@ module _BASE_UNITTEST: _BASE_PROGRAM { when ($UT_SKIP_EXCEPTIONS == "yes") { C_DEFINES+=-DUT_SKIP_EXCEPTIONS } + when ($OS_EMSCRIPTEN == "yes") { + # For WASM platforms at present we just check that code compiles without errors, + # these entries just suppresses linker errors. + LDFLAGS += -Wl,--no-entry + LDFLAGS += -Wl,--import-undefined + } SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS canondata/result.json) # Assume that no code may depend on unit test output and thus # do not mandate license markup in such modules. diff --git a/contrib/libs/libtiff/.yandex_meta/__init__.py b/contrib/libs/libtiff/.yandex_meta/__init__.py index 774c127d157..67f11e32e2c 100644 --- a/contrib/libs/libtiff/.yandex_meta/__init__.py +++ b/contrib/libs/libtiff/.yandex_meta/__init__.py @@ -16,7 +16,7 @@ def post_install(self): libtiff = CMakeNinjaNixProject( - owners=["g:images"], + owners=["g:cpp-contrib", "g:images"], arcdir="contrib/libs/libtiff", nixattr="libtiff", build_targets=["tiff", "tiffxx"], diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report index bfb68db69e1..918e51b5ea7 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report @@ -391,7 +391,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - tif_lzw.c [1439:1440] + tif_lzw.c [1437:1438] KEEP COPYRIGHT_SERVICE_LABEL ef9e24bc2ee738c92b1a0aae6c46127d BELONGS ya.make diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report index 38e76a4335e..3ac1388219b 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report @@ -52,7 +52,7 @@ BELONGS ya.make Match type : NOTICE Links : http://www.opensource.org/licenses/BSD-3-Clause, https://spdx.org/licenses/BSD-3-Clause Files with this license: - ChangeLog [12657:12659] + ChangeLog [14240:14242] KEEP MIT 85545b6051abd36fce0fc9cd8d2d53cc BELONGS ya.make @@ -75,7 +75,7 @@ BELONGS ya.make Match type : TEXT Links : https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/bsla.LICENSE Files with this license: - tif_lzw.c [1446:1456] + tif_lzw.c [1444:1454] KEEP libtiff a9ebcbf36a981b235e0c3e99fd1fad00 BELONGS ya.make diff --git a/contrib/libs/libtiff/.yandex_meta/override.nix b/contrib/libs/libtiff/.yandex_meta/override.nix index 6eb6872b313..f2bace9f33c 100644 --- a/contrib/libs/libtiff/.yandex_meta/override.nix +++ b/contrib/libs/libtiff/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; rec { - version = "4.6.0"; + version = "4.7.0"; src = fetchFromGitLab { owner = "libtiff"; repo = "libtiff"; rev = "v${version}"; - hash = "sha256-qCg5qjsPPynCHIg0JsPJldwVdcYkI68zYmyNAKUCoyw="; + hash = "sha256-SuK9/a6OUAumEe1kz1itFJGKxJzbmHkBVLMnyXhIwmQ="; }; patches = []; diff --git a/contrib/libs/libtiff/ChangeLog b/contrib/libs/libtiff/ChangeLog index 87b5f126bb4..a6ab18783c2 100644 --- a/contrib/libs/libtiff/ChangeLog +++ b/contrib/libs/libtiff/ChangeLog @@ -1,3 +1,1586 @@ +2024-09-11 Even Rouault <[email protected]> + + libtiff v4.7.0rc2 preparation + +2024-09-11 Even Rouault <[email protected]> + + configure.ac: make a provision to look for up to python 3.16 ... + hopefully we'll have ditched autoconf support at that point... + + Fix test dependency on tiff2rgba-32BPP that breaks 'make check -jX' + + test/tiff2ps*.sh scripts: try to use POSIX only diff flags. + +2024-09-11 Even Rouault <[email protected]> + + Merge branch 'automake_doc_fix' into 'master' + doc/Makefile.am: make sure that 'doc-html' and 'doc-man' targets have dependencies to avoid them to be rebuilt by successive make + + See merge request libtiff/libtiff!657 + +2024-09-10 Even Rouault <[email protected]> + + Merge branch '4_7_0_rst_rgb2ycbcr_thumbnail' into 'master' + v4.7.0.rst: clarify that rgb2ycbcr and thumbnail are not installed + + See merge request libtiff/libtiff!659 + +2024-09-10 Even Rouault <[email protected]> + + v4.7.0.rst: clarify that rgb2ycbcr and thumbnail are not installed. + +2024-09-08 Even Rouault <[email protected]> + + doc/Makefile.am: make sure that 'doc-html' and 'doc-man' targets have dependencies to avoid them to be rebuilt by successive make + + doc/Makefile.am: move rst_sources and EXTRA_DIST variables on top of file + +2024-09-07 Even Rouault <[email protected]> + + Merge branch 'index_rst_formatting_issue' into 'master' + index.rst: fix formatting issue + + See merge request libtiff/libtiff!656 + +2024-09-07 Even Rouault <[email protected]> + + index.rst: fix formatting issue. + +2024-09-07 Even Rouault <[email protected]> + + Merge branch 'prepare_release_4_7_0' into 'master' + Prepare release 4.7.0 + + See merge request libtiff/libtiff!654 + +2024-09-07 Even Rouault <[email protected]> + + Doc: remove 'Master' terminlogy. + + v4.7.0.rst: remove mention about autoconf-archive since that change has been reverted + + Revert "autotools: allow pulling in updated macros from autoconf-archive" + This reverts commit c820d16c30c27b3be6c3d10834b6d75607ae9d77. + + Revert "HOWTO-RELEASE: document the necessary dependencies for producing dist tarballs" + This reverts commit f1a91e42b3f23641cb5559fb60e1d73d201275eb. + + Revert "configure.ac: fix HAVE_OPENGL determination due to recent changes" + This reverts commit c370e67f7fd9320674544113218b7fbca4f65c2b. + + configure.ac: fix HAVE_OPENGL determination due to recent changes. + + Update version numbers for 4.7.0. + + Update ChangeLog and create release notes for v4.7.0. + +2024-09-06 Even Rouault <[email protected]> + + typo fixes. + +2024-09-06 Even Rouault <[email protected]> + + Merge branch 'autoconf-archive' into 'master' + autotools: allow pulling in updated macros from autoconf-archive + + See merge request libtiff/libtiff!645 + +2024-09-06 Even Rouault <[email protected]> + + Merge branch 'remove_last_usage_of_get_field_type' into 'master' + Change last usage of get_field_type at TIFFWriteDirectorySec() to set_field_type + + See merge request libtiff/libtiff!653 + +2024-09-06 Even Rouault <[email protected]> + + libtiff v4.7.0rc1 preparation + +2024-09-06 Even Rouault <[email protected]> + + Merge branch 'autoconf-archive' into 'master' + autotools: allow pulling in updated macros from autoconf-archive + + See merge request libtiff/libtiff!645 + +2024-09-06 Even Rouault <[email protected]> + + Merge branch 'remove_last_usage_of_get_field_type' into 'master' + Change last usage of get_field_type at TIFFWriteDirectorySec() to set_field_type + + See merge request libtiff/libtiff!653 + +2024-09-05 Su_Laus <[email protected]> + + Change last usage of get_field_type at TIFFWriteDirectorySec() for codec related tags to set_field_type. + +2024-09-03 Eli Schwartz <[email protected]> + + HOWTO-RELEASE: document the necessary dependencies for producing dist tarballs + + autogen.sh: actually return failure if any step failed. + Previously, it exited with failure if and only if the wget calls failed. + But it is possible for any command to fail, e.g. if the autotools are + broken, misconfigured, or missing dependencies. + +2024-09-03 Eli Schwartz <[email protected]> + + autotools: allow pulling in updated macros from autoconf-archive. + Instead of inlining a few macros in acinclude.m4 and forgetting about + them, teach aclocal to install the macros to m4/ when autoreconf is run. + They are no longer tracked in one big file, nor tracked at all in git, + but producing dist tarballs will bundle them up just like `configure` + and `Makefile.in`. + + Note: previous behavior of AX_CHECK_GL* was to simply ignore support and + proceed, if it was asked for and not found. The updated macros expect to + opt into this by specifying action-if-not-found. + + VL_* is obsolete. Upgrade to AX_CFLAGS_WARN_ALL. + +2024-09-03 Even Rouault <[email protected]> + + Merge branch 'XZY2RGB' into 'master' + Fixes #644 Index comparison as size_t to avoid overflow in TIFFXYZToRGB + + Closes #644 + + See merge request libtiff/libtiff!649 + +2024-09-03 Nicolas Badoux <[email protected]> + + Fixes #644 Index comparison as size_t to avoid overflow in TIFFXYZToRGB. + +2024-09-01 Even Rouault <[email protected]> + + Merge branch 'ci_werror' into 'master' + CI: build with -Wall -Wextra -Werror + + See merge request libtiff/libtiff!652 + +2024-08-30 Even Rouault <[email protected]> + + configure.ac: avoid -Werror passed to CFLAGS to interfere with feature detection + + CI: build with -Wall -Wextra -Werror. + +2024-08-30 Roger Leigh <[email protected]> + + Merge branch 'ci-old-update' into 'master' + ci: Update "old" build jobs to use Ubuntu 22.04 + + See merge request libtiff/libtiff!651 + +2024-08-30 Roger Leigh <[email protected]> + + ci: Update "old" build jobs to use Ubuntu 22.04. + +2024-08-30 Even Rouault <[email protected]> + + Merge branch 'ci-ubuntu-22.04' into 'master' + ci: Update to use Ubuntu 24.04 CI images + + See merge request libtiff/libtiff!650 + +2024-08-30 Roger Leigh <[email protected]> + + ci: Update to use Ubuntu 24.04 CI images. + +2024-08-28 Even Rouault <[email protected]> + + Merge branch 'uv_encode' into 'master' + Fix #645 by using unsigned int for variable indexing an array in uv_decode() and uv_encode() + + Closes #645 + + See merge request libtiff/libtiff!648 + +2024-08-28 Nicolas Badoux <[email protected]> + + Fix #645 by using unsigned int for variable indexing an array in uv_decode() and uv_encode() + Change `vi` to unsigned int in `uv_decode()` and `uv_encode()` to ensure that upper bound `UV_NVS` cannot be satisfied by a negative value which will overflow when cast to an unsigned value at array access time. This fixes issue #645 where a segmentation fault was caused. + +2024-08-27 Even Rouault <[email protected]> + + Merge branch 'fix_643_solitary_CustomDirectory_read' into 'master' + Fix #643 by initializing pointer to TIFFSetField() and TIFFGetField() before TIFFReadGPSDirectory + + Closes #643 + + See merge request libtiff/libtiff!647 + +2024-08-27 Su Laus <[email protected]> + + Fix #643 by initializing pointer to TIFFSetField() and TIFFGetField() before TIFFReadGPSDirectory + Initialize pointer to TIFFSetField() and TIFFGetField() (i.e. tif-\>tif_tagmethods.vsetfield and tif-\>tif_tagmethods.vgetfield) even if the file is opened with "h" option. This fixes issue #643 where a segmentation fault was caused. + +2024-08-25 Even Rouault <[email protected]> + + Merge branch 'tiff_strip' into 'master' + Add non-zero check before division in TIFFComputeStrip + + See merge request libtiff/libtiff!646 + +2024-08-23 Nicolas Badoux <[email protected]> + + Add non-zero check before division in TIFFComputeStrip. + +2024-08-15 Even Rouault <[email protected]> + + Merge branch 'fix_564' into 'master' + TIFFScanlineSize64(): revert merge request #564 + + See merge request libtiff/libtiff!644 + +2024-08-15 Even Rouault <[email protected]> + + TIFFScanlineSize64(): revert merge request #564. + https://gitlab.com/libtiff/libtiff/-/merge_requests/564 broke decoding of: "tools/tiffcp -c none in.tif out.tif" + with [in.tif](/uploads/0475a88cf6e66652b030a0a3a0c47313/in.tif) + leading to shift each line of the target image by one extra pixel at each line. + Unfortunately, this cancels what https://gitlab.com/libtiff/libtiff/-/merge_requests/564 tried to fix. I'll let @caolanm propose an alternative fix in a follow-up merge request + +2024-08-15 Even Rouault <[email protected]> + + Merge branch 'coverity_fix_from_MR_634' into 'master' + Fix some Coverity Scan issues introduced by MR 634. + + See merge request libtiff/libtiff!643 + +2024-08-14 Su_Laus <[email protected]> + + Fix some Coverity Scan issues introduced by MR 634. + +2024-08-11 Even Rouault <[email protected]> + + Merge branch 'lerc-pkgconfig' into 'master' + libtiff-4.pc: Fix `Requires.private` missing `Lerc`. + + See merge request libtiff/libtiff!633 + +2024-08-11 Even Rouault <[email protected]> + + Merge branch 'manpage_update_directory_incrementing' into 'master' + Amend manpages for changes in current directory index behaviour. + + See merge request libtiff/libtiff!639 + +2024-08-11 Su Laus <[email protected]> + + Amend manpages for changes in current directory index behaviour. + +2024-08-11 Even Rouault <[email protected]> + + Merge branch 'tiffcrop_coverity_20240723' into 'master' + Attempt to address tiffcrop Coverity scan issues 1605444, 1605445, and 1605449. + + See merge request libtiff/libtiff!642 + +2024-08-11 Lee Howard <[email protected]> + + Attempt to address tiffcrop Coverity scan issues 1605444, 1605445, and 1605449. + +2024-08-11 Even Rouault <[email protected]> + + Merge branch 'fix_CurDir_wrong_incrementing' into 'master' + Fix cases where tif_curdir is set incorrectly + + See merge request libtiff/libtiff!634 + +2024-08-11 Su Laus <[email protected]> + + Fix cases where tif_curdir is set incorrectly. + Fix cases where the current directory number (tif_curdir) is set inconsistently or incorrectly, depending on the previous history. + + See additional checks and tests in test/test_directory.c of this MR for intended setting of tif_curdir (i.e. TIFFCurrentDirectory(tif)). + +2024-07-12 Even Rouault <[email protected]> + + Merge branch 'coverity_fixes_tiff2pdf_#2' into 'master' + Try to fix additional Coverity issues in tiff2pdf + + See merge request libtiff/libtiff!641 + +2024-07-12 Su Laus <[email protected]> + + Try to fix additional Coverity issues in tiff2pdf. + +2024-07-10 Even Rouault <[email protected]> + + Merge branch 'fix_autoconf_diff_parameter_crlf' into 'master' + Add parameter for "diff" used in autoconf test scripts to ignore LF and CRLF differences. + + See merge request libtiff/libtiff!640 + +2024-07-09 Even Rouault <[email protected]> + + Merge branch 'coverity_fixes_tiff2pdf' into 'master' + Fix Coverity issues in tiff2pdf. + + See merge request libtiff/libtiff!638 + +2024-07-09 Su Laus <[email protected]> + + Fix Coverity issues in tiff2pdf. + +2024-06-29 Su_Laus <[email protected]> + + Add parameter for "diff" used in autoconf test scripts to ignore LF and CRLF differences. + +2024-06-28 Even Rouault <[email protected]> + + Merge branch 'coverity_fixes_test_thumbnail' into 'master' + Coverity Scan fixes in thumbnail.c and custom_dir_EXIF_231.c + + See merge request libtiff/libtiff!637 + +2024-06-28 Su Laus <[email protected]> + + Coverity Scan fixes in thumbnail.c and custom_dir_EXIF_231.c. + +2024-06-25 Even Rouault <[email protected]> + + Merge branch 'coverity_fixes' into 'master' + Try to silence new Coverity Scan false positives + + See merge request libtiff/libtiff!636 + +2024-06-24 Even Rouault <[email protected]> + + Try to silence new Coverity Scan false positives. + There are other warnings in tools/ that I'll let others deal with. + +2024-06-24 Even Rouault <[email protected]> + + Merge branch 'fix/make' into 'master' + fix: error when running make clean + + Closes #630 + + See merge request libtiff/libtiff!635 + +2024-06-24 Kévin Dunglas <[email protected]> + + fix: error when running make clean. + +2024-06-15 Niklas Hambüchen <[email protected]> + + libtiff-4.pc: Fix `Requires.private` missing `Lerc`. + It provides a `.pc` file starting from version 4 in: + + https://github.com/Esri/lerc/blob/8d6e8251544bbe1379feb0fe0a0934e43ca1a6cd/Lerc.pc.in + + libtiff's CMake build system already has support for this, + adding `Lerc` to `Requires.private` if Lerc is >= 4. + + In contrast to the CMake build system, autoconf has no good way + to check for checking the Lerc version, we do it unconditionally + there. + +2024-06-13 Even Rouault <[email protected]> + + Merge branch 'rfc02_text__restore_tools' into 'master' + Text for RFC 2: Restoring needed libtiff tools + + See merge request libtiff/libtiff!581 + +2024-06-13 Even Rouault <[email protected]> + + Merge branch 'add_some_conversion_test_to_cmake' into 'master' + Port some basic sanity checks from autoconf to cmake. + + See merge request libtiff/libtiff!630 + +2024-06-10 Even Rouault <[email protected]> + + Merge branch 'tiffcp_coverity_check_malloc_return' into 'master' + tiffcp: Add check for limitMalloc return to fix Coverity CID 1603334 + + See merge request libtiff/libtiff!632 + +2024-06-10 Su_Laus <[email protected]> + + tiffcp add check for limitMalloc return to fix Coverity CID 1603334. + +2024-06-07 Even Rouault <[email protected]> + + Merge branch 'aim-nara-master-patch-67955' into 'master' + Remove unnecessary `2` in tiffcmp name + + See merge request libtiff/libtiff!631 + +2024-06-07 AIM | Nara <[email protected]> + + Remove unnecessary `2` in tiffcmp name. + +2024-05-30 Su_Laus <[email protected]> + + Port some basic sanity checks from autoconf to cmake. Checks are for tiffcp and tiffcrop RGB->YCbCr JPEG conversions (see MR !611 / 'tiffcp_tiffcrop_RGB_YCbCr_tests') + +2024-05-29 Even Rouault <[email protected]> + + Merge branch 'ofz65182' into 'master' + an issue seen in putcontig8bitYCbCr22tile + + See merge request libtiff/libtiff!564 + +2024-05-29 Caolán McNamara <[email protected]> + + TIFFScanlineSize64(): fix computation in non-JPEG YCbCr case (#564) + Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=65182 + +2024-05-29 Even Rouault <[email protected]> + + Merge branch 'fix_375' into 'master' + TIFFRead[Scanline/EncodedStrip/EncodeTile]: 0-initialize output buffer if setupdecode fails ; most codecs: zero-initialize (not-yet-written parts of) output buffer if failure + + Closes #375 + + See merge request libtiff/libtiff!628 + +2024-05-29 Even Rouault <[email protected]> + + Merge branch 'tiff2pdf-planar' into 'master' + Inconsistent PLANARCONFIG value for the input and output TIFF in tiff2pdf.c + + See merge request libtiff/libtiff!629 + +2024-05-29 Timothy Lyanguzov <[email protected]> + + Inconsistent PLANARCONFIG value for the input and output TIFF in tiff2pdf.c + Fixed a bug in tiff2pdf.c whereas the PLANARCONFIG for the output TIFF is set to a fixed value of PLANARCONFIG_CONTIG as opposed to match the value of the input TIFF. + This mainly manifest in the heap-buffer-overread due to the difference of sizes in the buffers (between input and output) allocated and read to the size of output buffer. + + Reimplementation of !92 + +2024-05-27 Even Rouault <[email protected]> + + tif_lzma/tif_lzw: add code comments. + +2024-05-26 Even Rouault <[email protected]> + + ZSTDDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + PixarLogDecode: zero-initialize output buffer if failure. + + PackBitsDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + JPEGDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + LERCDecode: zero-initialize output buffer if failure. + + LZMADecode: zero-initialize (not-yet-written parts of) output buffer if failure, and handle repeated calls to decoding same tile/strip if previous failure occured + + TWebPDecode: zero-initialize output buffer if failure, and handle repeated calls to decoding same tile/strip if previous failure occured + + ThunderDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + OJPEGDecode: zero-initialize output buffer if failure. + + JBIGDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + ZIPDecode: zero-initialize (not-yet-written parts of) output buffer if failure, and handle repeated calls to decoding same tile/strip if previous failure occured + + LZWDecode: zero-initialize (not-yet-written parts of) output buffer if failure + + TIFFRead[Scanline/EncodedStrip/EncodeTile]: 0-initialize output buffer if setupdecode fails + Fixes #375 + +2024-05-25 Su_Laus <[email protected]> + + Text for RFC 2: Restoring needed libtiff tools - file added to doc/Makefile.am and rebased. + + Text for RFC 2: Restoring needed libtiff tools - approved with voting history updated. + + Text for RFC 2: Restoring needed libtiff tools - further amended. + + Text for RFC 2: Restoring needed libtiff tools - amended. + + Text for RFC 2: Restoring needed libtiff tools. + +2024-05-23 Even Rouault <[email protected]> + + Merge branch 'tiff2pdf_539' into 'master' + fix tiff2pdf issue #539; however, there certainly remain many JPEG-compressed... + + See merge request libtiff/libtiff!597 + +2024-05-23 Lee Howard <[email protected]> + + tiff2pdf: fix issue with JPEG restart-interval marker when converting from JPEG-compressed files + Fixes #539 + + however, there certainly remain many JPEG-compressed TIFFs which tiff2pdf will fumble without disabling data passthrough (using the -n option) - I supect that it has to do with the JPEG tables being removed from the JPEG and placed into the TIFF headers - as tiff2pdf would need to put these back into the JPEG instead of just passing the data through to the PDF + +2024-05-22 Even Rouault <[email protected]> + + Merge branch 'test_fix' into 'master' + test/Makefile.am: make sure that all test images are bundled by make dist even... + + See merge request libtiff/libtiff!626 + +2024-05-22 Even Rouault <[email protected]> + + Merge branch 'add_missing_rst_files' into 'master' + doc/Makefile.am: add rfcs/ files in rst_sources variable + + See merge request libtiff/libtiff!624 + +2024-05-22 Even Rouault <[email protected]> + + test/Makefile.am: make sure that all test images are bundled by make dist even if JPEG or JBIG is not available + +2024-05-22 Even Rouault <[email protected]> + + Merge branch 'timothyl/32bpp-cmake-test' into 'master' + Add new test to CMake + + See merge request libtiff/libtiff!625 + +2024-05-22 Timothy Lyanguzov <[email protected]> + + Add new test to CMake. + See previous merge request libtiff/libtiff!611 + +2024-05-22 Even Rouault <[email protected]> + + doc/Makefile.am: add rfcs/ files in rst_sources variable. + +2024-05-22 Even Rouault <[email protected]> + + Merge branch 'tiffcp_tiffcrop_RGB_YCbCr_tests' into 'master' + Add some basic sanity checks for tiffcp and tiffcrop RGB->YCbCr JPEG conversions. + + See merge request libtiff/libtiff!611 + +2024-05-22 Lee Howard <[email protected]> + + Add some basic sanity checks for tiffcp and tiffcrop RGB->YCbCr JPEG conversions. + +2024-05-21 Even Rouault <[email protected]> + + Merge branch 'tiffcp_LZW_JBIG_test' into 'master' + Add basic sanity check for tiffcp LZW->JBIG conversion. + + See merge request libtiff/libtiff!612 + +2024-05-21 Even Rouault <[email protected]> + + Merge branch 'tiffcrop_masonwilde_fix' into 'master' + Apply "Fix heap-buffer-overflow in function extractImageSection" + + See merge request libtiff/libtiff!613 + +2024-05-21 Lee Howard <[email protected]> + + tiffcrop: Apply "Fix heap-buffer-overflow in function extractImageSection" + https://gitlab.com/masonwilde/libtiff/-/commit/848434a81c443f59ec90d41218eba6e48a450a11 + + authored by zhailiangliang commit 848434a81c443f59ec90d41218eba6e48a450a11 + + The overflow conditions were previously addressed, but this provides additional error messages. + +2024-05-21 Even Rouault <[email protected]> + + Merge branch 'fix_183_alternative' into 'master' + OJPEG: reset subsampling_convert_state =0 in OJPEGPreDecode (fixes tiff2pdf issue #183) + + See merge request libtiff/libtiff!621 + +2024-05-21 Even Rouault <[email protected]> + + Merge branch 'thunder-fix' into 'master' + A fix for Thunder RLE + + See merge request libtiff/libtiff!623 + +2024-05-21 Timothy Lyanguzov <[email protected]> + + A fix for Thunder RLE. + The commit https://gitlab.com/robinwatts/libtiff/-/commit/be0519ca3d222c19008fe160873f3b5c28e6b36b states: + + Thunder RLE can fail to decode last run. Bug seen with GhostPDL. Decode of test tif file gives valgrind errors due to the "thunder decode" failing to extract the final run of compressed pixels. + + The logic in the decoder, presumably intended to spot overruns of data, is incorrect in that runs that end at the end of a row (npixels == maxpixels) will not be decoded. + + Fix this by limiting 'n', the number of pixels to copy. Note that npixels is updated by the 'unlimited' value to ensure the error reporting at the end of the loop still works. + +2024-05-18 Even Rouault <[email protected]> + + Merge branch 'tiff2ps_fax2tiff_476' into 'master' + fix fax2ps and fax2tiff bugs #476 + + See merge request libtiff/libtiff!598 + +2024-05-18 Lee Howard <[email protected]> + + Fix fax2ps and fax2tiff memory leaks. + Fixes #476 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiff2rgba_469' into 'master' + fix tiff2rgba issue #469 + + See merge request libtiff/libtiff!600 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'formatting-fixes' into 'master' + Formatting fixes + + See merge request libtiff/libtiff!622 + +2024-05-17 Even Rouault <[email protected]> + + Formatting fixes. + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'coverity_fixes' into 'master' + 3 Coverity Scan fixes in tools & contrib + + See merge request libtiff/libtiff!592 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiff2pdf_596' into 'master' + tiff2pdf issue #596 + + See merge request libtiff/libtiff!594 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiffdither_473' into 'master' + fix tiffdither bug #473 + + See merge request libtiff/libtiff!599 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'fax2tiff_468' into 'master' + fix fax2tiff issue #468 + + See merge request libtiff/libtiff!601 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiff2rgba_background' into 'master' + Add background gradient option for tiff2rgba alpha compositing. + + See merge request libtiff/libtiff!610 + +2024-05-17 Lee Howard <[email protected]> + + Add background gradient option for tiff2rgba alpha compositing. + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiff2pdf_253' into 'master' + Fix tiff2pdf issue #253 - red and blue were being swapped for RGBA decoding + + See merge request libtiff/libtiff!603 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'fax2tiff_249' into 'master' + Fix fax2tiff issue #249, unreasonable width input + + See merge request libtiff/libtiff!604 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiffcp_tiffcrop_228' into 'master' + correct tiffcp and tiffcrop issue #228 + + See merge request libtiff/libtiff!605 + +2024-05-17 Even Rouault <[email protected]> + + OJPEG: reset subsampling_convert_state =0 in OJPEGPreDecode (fixes tiff2pdf issue #183) + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'fax2tiff_191' into 'master' + Fix fax2tiff issue #191 - EOFB interpretation + + See merge request libtiff/libtiff!606 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiff2pdf_98' into 'master' + Fix tiff2pdf documentation issue #98 + + See merge request libtiff/libtiff!608 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiffgt_warning' into 'master' + Avoids a warning regarding fallthrough. + + See merge request libtiff/libtiff!609 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiffcrop_542_550_552' into 'master' + tiffcrop: fixes #542, #550, #552 (buffer overflows, use after free) + + See merge request libtiff/libtiff!595 + +2024-05-17 Lee Howard <[email protected]> + + tiffcrop: fixes #542, #550, #552 (buffer overflows, use after free) + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'Fedora_am_version' into 'master' + libtiff-am-version.patch from Fedora stating: + + See merge request libtiff/libtiff!617 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'formatting-fixes' into 'master' + Code formatting fixes + + See merge request libtiff/libtiff!620 + +2024-05-17 Even Rouault <[email protected]> + + Code formatting fixes. + Result of running ``pre-commit run --all`` + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'sphinx_quotes_dashes' into 'master' + Don't let Sphinx inconsistently alter quotes and dashes in rst files. + + See merge request libtiff/libtiff!615 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'tiffmedian_599' into 'master' + tiffmedian issue #599 + + See merge request libtiff/libtiff!593 + +2024-05-17 Lee Howard <[email protected]> + + tiffmedian: fix memory leaks. + Closes #599 + +2024-05-17 Even Rouault <[email protected]> + + Merge branch 'more_spelling_corrections' into 'master' + Typo fixes following... + + See merge request libtiff/libtiff!618 + +2024-05-17 Lee Howard <[email protected]> + + Typo fixes following https://gitlab.com/libtiff/libtiff/-/commit/c95c77350820e19c8e273925bc97259727d19695 + +2024-05-16 Lee Howard <[email protected]> + + libtiff-am-version.patch from Fedora stating: Back off the minimum required automake version to 1.11. There isn't anything in libtiff currently that actually requires 1.12, and changing this allows the package to be built on pre-F18 machines for easier testing. + + Don't let Sphinx inconsistently alter quotes and dashes in rst files. Author: Laszlo Boszormenyi (GCS) <[email protected]> Bug-Debian: https://bugs.debian.org/1028456 + + Add basic sanity check for tiffcp LZW->JBIG conversion. + + Avoids a warning regarding fallthrough. + + Fix tiff2pdf documentation issue #98. + + Fix fax2tiff issue #191 - EOFB interpretation. + + Update tiffcp.c. + + Correct tiffcrop output of RGB JPEG and tiffcp/tiffcrop documentation - issue #228 + + Fix tiffcp and tiffcrop issue 228 - step 1. + + Fix fax2tiff issue #249, unreasonable width input. + + Fix tiff2pdf issue #253 - red and blue were being swapped for RGBA decoding + + fix fax2tiff issue #468. + + fix tiff2rgba issue #469. + + fix tiffdither bug #473. + + tiff2pdf issue #596. + +2024-05-16 Even Rouault <[email protected]> + + contrib/addtiffo: validate return of TIFFWriteEncodedXXXX() calls (CID 1024680) + + tiffdump.c: fix wrong printf formattre in error message (CID 1472932) + + tiffset.c: avoid false positive Coverity Scan warning on 64-bit builds (CID 1518997) + +2024-05-16 Even Rouault <[email protected]> + + Merge branch 'master' into 'master' + Fix some Coverity warnings. + + See merge request libtiff/libtiff!590 + +2024-05-16 Lee Howard <[email protected]> + + Fix some Coverity warnings. + +2024-05-13 Even Rouault <[email protected]> + + Merge branch 'revert_tools_removement' into 'master' + Restore tools from archive and unsupported and tiffcp -i option with revert. + + See merge request libtiff/libtiff!589 + +2024-05-11 Su_Laus <[email protected]> + + Apply some newer changes to the reverted code and prepare note for 4.7.0. + + Revert "Remove -i option (ignore errors) from tiffcp, because almost all fuzzer issues were consequential errors from ignored errors because of the "-i" option." + This reverts commit 280a568ae887c27a422a5da862398ffdbcd9b84d. + +2024-05-11 Su_Laus <[email protected]> + + Revert "Move most TIFF tools to archive and keep some as unsupported (see #580)." + This reverts commit eab89a627f0a65e9a1a47c4b30b4802c80b1ac45. + + # Conflicts: + # tools/unsupported/CMakeLists.txt + # tools/unsupported/tif_tools-unsupported_versioninfo.rc + +2024-05-10 Even Rouault <[email protected]> + + Merge branch 'calloc-transposed-args' into 'master' + Fix -Werror=calloc-transposed-args with gcc 14 + + See merge request libtiff/libtiff!588 + +2024-05-10 Even Rouault <[email protected]> + + Fix -Werror=calloc-transposed-args with gcc 14. + +2024-05-09 Even Rouault <[email protected]> + + Merge branch 'EvaluateIFDdatasizeReading_ossfuzz_68327' into 'master' + EvaluateIFDdatasizeReading(): avoid unsigned integer overflow (master only) + + See merge request libtiff/libtiff!586 + +2024-04-27 Even Rouault <[email protected]> + + EvaluateIFDdatasizeReading(): avoid unsigned integer overflow (master only) + Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=68327 + +2024-04-25 Even Rouault <[email protected]> + + Merge branch 'ci-appveyor-cygwin-upgrade' into 'master' + appveyor: Upgrade cygwin packages + + See merge request libtiff/libtiff!585 + +2024-04-25 Roger Leigh <[email protected]> + + appveyor: Upgrade cygwin packages. + +2024-04-25 Even Rouault <[email protected]> + + Merge branch 'libjpeg_turbo_3_0' into 'master' + Change messages/comments refering libjpeg-turbo 2.2 to 3.0 + + See merge request libtiff/libtiff!584 + +2024-04-24 Even Rouault <[email protected]> + + Change messages/comments refering libjpeg-turbo 2.2 to 3.0. + libjpeg-turbo 2.2 was actually released as 3.0. Reflect that + +2024-04-22 Even Rouault <[email protected]> + + Merge branch 'EvaluateIFDdatasizeReading_unsigned_int_overflow_fix' into 'master' + EvaluateIFDdatasizeReading(): avoid potential unsigned integer overflow on corrupted tag + + See merge request libtiff/libtiff!582 + +2024-04-22 Even Rouault <[email protected]> + + EvaluateIFDdatasizeReading(): avoid potential unsigned integer overflow on corrupted tag + +2024-04-22 Even Rouault <[email protected]> + + Merge branch 'tif_dir_logging-627' into 'master' + tif_dir.c: Log source file, line number, and input tif for directory count error. + + Closes #627 + + See merge request libtiff/libtiff!583 + +2024-04-22 Kurt Schwehr <[email protected]> + + tif_dir.c: Log source file, line number, and input tif for directory count error. + +2024-04-14 Even Rouault <[email protected]> + + Merge branch 'fix_readrgbastrip_fpe' into 'master' + Avoiding FPEs (division by zero) in tif_getimage.c + + See merge request libtiff/libtiff!580 + +2024-04-09 Zurab Tsinadze <[email protected]> + + Avoid FPEs (division by zero) in tif_getimage.c. + +2024-04-06 Even Rouault <[email protected]> + + Merge branch 'appveyor-vs2022' into 'master' + appveyor: Add VS2022 builds + + See merge request libtiff/libtiff!579 + +2024-04-06 Roger Leigh <[email protected]> + + appveyor: Add VS2022 builds. + +2024-04-04 Roger Leigh <[email protected]> + + Merge branch 'appveyor-mingw-fix' into 'master' + appveyor: Use MinGW-w64 + + See merge request libtiff/libtiff!578 + +2024-04-04 Roger Leigh <[email protected]> + + appveyor: Use MinGW-w64. + +2024-04-03 Even Rouault <[email protected]> + + Merge branch 'update_rfc1_status' into 'master' + RFC1: update status to adopted + + See merge request libtiff/libtiff!577 + +2024-04-03 Even Rouault <[email protected]> + + RFC1: update status to adopted. + +2024-04-03 Even Rouault <[email protected]> + + Merge branch 'rfc1_psc' into 'master' + Doc: add 'RFC 1: Project Steering Committee Guidelines' and a PSC page + + See merge request libtiff/libtiff!566 + +2024-04-03 Even Rouault <[email protected]> + + Doc: add 'RFC 1: Project Steering Committee Guidelines' and a PSC page. + +2024-04-02 Even Rouault <[email protected]> + + Merge branch 'fix_coverity_test_directory_expansion' into 'master' + Fix coverity scan issue in test_directory.c + + See merge request libtiff/libtiff!576 + +2024-04-02 Su_Laus <[email protected]> + + Fix coverity scan issue in test_directory.c. + +2024-04-02 Even Rouault <[email protected]> + + Merge branch 'typo_fixes' into 'master' + Various typo fixes + + See merge request libtiff/libtiff!575 + +2024-04-02 Even Rouault <[email protected]> + + Various typo fixes. + +2024-03-31 Even Rouault <[email protected]> + + Merge branch 'test_directory_expansion_endianness' into 'master' + Enhance test_directory.c with big-endian and little-endian test cases + + See merge request libtiff/libtiff!574 + +2024-03-30 Su_Laus <[email protected]> + + Enhance test/test_directory.c with big-endian and little-endian test cases. + Insert testcase into test_directory.c for #618 "Failure in TIFFSetSubDirectory() will break subsequent TIFFSetDirectory() calls". + +2024-03-29 Even Rouault <[email protected]> + + Merge branch 'correct_coverity_issues_from_avoid_IFD_overwriting' into 'master' + Correct some coverity issues from "avoid IFD overwriting" + + See merge request libtiff/libtiff!573 + +2024-03-29 Su_Laus <[email protected]> + + Correct some coverity issues from "avoid IFD overwriting". + +2024-03-28 Even Rouault <[email protected]> + + Merge branch 'fix_gdal_9548' into 'master' + LERC codec: deal with issues with multi-band PlanarConfig=Contig and NaN values + + See merge request libtiff/libtiff!572 + +2024-03-26 Even Rouault <[email protected]> + + Merge branch 'avoid_IFD_overwriting_other_data_TIFFWriteDirectory' into 'master' + TIFFWriteDirectory: Avoid overwriting following data if an IFD is enlarged. + + See merge request libtiff/libtiff!565 + +2024-03-26 Su Laus <[email protected]> + + TIFFWriteDirectory: Avoid overwriting following data if an IFD is enlarged. + Avoid overwriting other data if an IFD that has already been written is enlarged. If entries are to be changed in an IFD that has already been written to the file, the memory area can be overwritten as long as the IFD data volume has not increased. Otherwise, the IFD must be written to a different location in the file. Up to now, LibTIFF has simply overwritten the IFD and other data areas in the file. + + With this MR, the size of the IFD, and location of IFD data, is tracked and checked if the IFD can be safely overwritten or has to be re-written to another location. + +2024-03-24 Even Rouault <[email protected]> + + LERC codec: deal with issues with multi-band PlanarConfig=Contig and NaN values + Fixes https://github.com/OSGeo/gdal/issues/9530. + Cf https://github.com/OSGeo/gdal/pull/9548 for the GDAL specific fix. + + The LERC codec up-to-now didn't do any special processing when writing a strile + in a multi-band PlanarConfig=Contig file, resulting in corruptions. Now we + generate a mask buffer in that situation. Actually, if the NaN values are not + at the same location accross bands, we need to write one mask-per-band, which + requires liblerc >= 3.0, and in that situation change the LERC blob to + multi-band instead of multi-depth/dim, with liblerc >= 3.0. When all bands have + NaN values at the same location, we generate a single mask, which is compatible + of liblerc < 3.0, but requires at last this version of libtiff to be read. This + is the best solution we come with to fix the issue while having the maximum + backward compatibility. + +2024-03-24 Even Rouault <[email protected]> + + Merge branch 'harmonize_TIFFheader_swapping' into 'master' + Correct and harmonize swapping of internal 'tif_header' parameters. + + See merge request libtiff/libtiff!571 + +2024-03-24 Su Laus <[email protected]> + + Correct and harmonize swapping of internal 'tif_header' parameters. + +2024-03-19 Even Rouault <[email protected]> + + Merge branch 'chore/printf-format-specifiers' into 'master' + chore: use correct format specifiers + + See merge request libtiff/libtiff!570 + +2024-03-19 Tom <[email protected]> + + chore: use correct format specifiers. + When building on my Mac, I get the following warnings: + + libtiff/tools/tiffcp.c:154:24: warning: format specifies type 'unsigned short' but the argument has type 'tdir_t' (aka 'unsigned int') [-Wformat] + comma, nextImage); + ^~~~~~~~~ + libtiff/tools/tiffcp.c:1295:39: warning: format specifies type 'unsigned short' but the argument has type 'tdir_t' (aka 'unsigned int') [-Wformat] + TIFFFileName(bias), TIFFCurrentDirectory(bias), + ^~~~~~~~~~~~~~~~~~~~~~~~~~ + libtiff/tools/tiffcp.c:1296:37: warning: format specifies type 'unsigned short' but the argument has type 'tdir_t' (aka 'unsigned int') [-Wformat] + TIFFFileName(in), TIFFCurrentDirectory(in)); + ^~~~~~~~~~~~~~~~~~~~~~~~ + libtiff/tools/tiffcp.c:1303:37: warning: format specifies type 'unsigned short' but the argument has type 'tdir_t' (aka 'unsigned int') [-Wformat] + TIFFFileName(in), TIFFCurrentDirectory(in)); + ^~~~~~~~~~~~~~~~~~~~~~~~ + libtiff/tools/tiffsplit.c:174:21: warning: format specifies type 'long long' but the argument has type 'size_t' (aka 'unsigned long') [-Wformat] + path_len); + ^~~~~~~~ + + This updates the format specifiers from `PRIu16` to `PRIu32` for `tdir_t` + (which is a `typdef`ed `uint32_t`) and from `TIFF_SSIZE_FORMAT` to + `TIFF_SIZE_FORMAT` for `size_t`, which makes sense given that it's an unsigned + type. + +2024-03-16 Even Rouault <[email protected]> + + Merge branch 'fix_628_FPE_at_TIFFhowmany' into 'master' + Avoiding FPE (division by zero) for TIFFhowmany_32() and TIFFhowmany_64()... + + Closes #628 + + See merge request libtiff/libtiff!568 + +2024-03-15 Su_Laus <[email protected]> + + Avoiding FPE (division by zero) for TIFFhowmany_32() and TIFFhowmany_64() macros by checking for denominator not zero before macros are executed. Fixes #628. + +2024-03-14 Even Rouault <[email protected]> + + Merge branch 'fix_wrong_return_TIFFIsBigTIFF_if_swapping_case' into 'master' + Fix wrong return of TIFFIsBigTIFF() in case byte-swapping is acitve + + See merge request libtiff/libtiff!567 + +2024-03-13 Su_Laus <[email protected]> + + Fix wrong return of TIFFIsBigTIFF() in case byte-swapping is acitve (i.e. little endian image on a big endian machine). + +2024-01-30 Even Rouault <[email protected]> + + Merge branch 'master' into 'master' + Setting the TIFFFieldInfo field `set_field_type` should consider `field_writecount` not `field_readcount` + + See merge request libtiff/libtiff!544 + +2024-01-30 Timo Stüber <[email protected]> + + Setting the TIFFFieldInfo field `set_field_type` should consider `field_writecount` not `field_readcount` + In most of the cases the field_writecount and field_readcount of the TIFFFieldInfo are equal. But they do not have to be the same. + + I encountered this issues when using an older release of Pillow. Since they use custom fields for decoding only, they set the readcount value to 0. As the result _TIFFSetGetType can not identify the type correctly and set TIFF_SETGET_UNDEFINED. + + Error Message: + ``` + TIFFLib: _TIFFWriteDirectorySec(): Rational2Double: .set_field_type is not 4 but 1. + ``` + + During the execution of TIFFMergeFieldInfo the fields get_field_type and set_field_type set. I guess the set_field_type should dependent on the field_writecount. + +2024-01-24 Even Rouault <[email protected]> + + Merge branch 'avoid_memory_leaks_using_TIFFCreateDirectory' into 'master' + Avoiding memory leaks when using TIFFCreateDirectory() + + See merge request libtiff/libtiff!563 + +2024-01-24 Su_Laus <[email protected]> + + Avoiding memory leaks when using TIFFCreateDirectory() by releasing the allocated memory in the tif-structure. + +2024-01-10 Even Rouault <[email protected]> + + Merge branch 'WIN32' into 'master' + Use #ifdef _WIN32 to test for Windows, and tiffio.h: remove definition of __WIN32__ + + See merge request libtiff/libtiff!562 + +2024-01-08 Even Rouault <[email protected]> + + test/raw_decode.c: remove likely ifdef tricks related to old compilers or unusual setups + + tif_jpeg.c/tif_ojpeg.c: remove likely ifdef tricks related to old compilers or unusual setups + + Remove _TIFFUInt64ToFloat() and _TIFFUInt64ToDouble() + + Remove support for _MSC_VER < 1500. + +2024-01-07 Even Rouault <[email protected]> + + Use #ifdef _WIN32 to test for Windows, and tiffio.h: remove definition of __WIN32__ + +2023-12-20 Even Rouault <[email protected]> + + Merge branch 'fix_579_non_terminated_ASCII_deletes_one_byte' into 'master' + Fixed issue with non-terminated ASCII arrays where the last character is overwritten (fixes #579) + + Closes #579 + + See merge request libtiff/libtiff!561 + +2023-12-20 Su_Laus <[email protected]> + + For non-terminated ASCII arrays, the buffer is first enlarged before a NULL is set at the end to avoid deleting the last character. Fixes #579. Some editorials in tif_dir.c. + +2023-12-09 Even Rouault <[email protected]> + + Merge branch 'fix_624_check_TIFFCreateAnonField_return' into 'master' + Fix 624: Check return value of _TIFFCreateAnonField() + + Closes #624 + + See merge request libtiff/libtiff!560 + +2023-12-01 Su_Laus <[email protected]> + + Check return value of _TIFFCreateAnonField(). + Fixes #624 + +2023-11-27 Even Rouault <[email protected]> + + Merge branch 'TIFFOpenOptionsSetMaxCumulatedMemAlloc' into 'master' + Add TIFFOpenOptionsSetMaxCumulatedMemAlloc() + + See merge request libtiff/libtiff!556 + +2023-11-27 Even Rouault <[email protected]> + + Add TIFFOpenOptionsSetMaxCumulatedMemAlloc() + This function complements ``TIFFOpenOptionsSetMaxSingleMemAlloc()`` to define the maximum cumulated memory allocations in byte, for a given TIFF handle, that libtiff internal memory allocation functions are allowed. + +2023-11-23 Even Rouault <[email protected]> + + Merge branch 'unity_clang_fixes' into 'master' + Fixes to avoid CLang warnings on unity builds + + See merge request libtiff/libtiff!558 + +2023-11-23 Even Rouault <[email protected]> + + Fixes to avoid CLang warnings on unity builds. + +2023-11-22 Even Rouault <[email protected]> + + Merge branch 'cmake_UNITY_BUILD' into 'master' + Changes to allow building with cmake -DCMAKE_UNITY_BUILD=ON + + See merge request libtiff/libtiff!557 + +2023-11-22 Even Rouault <[email protected]> + + CI: test -DCMAKE_UNITY_BUILD=ON. + + Changes to allow building with cmake -DCMAKE_UNITY_BUILD=ON. + +2023-11-22 Even Rouault <[email protected]> + + Merge branch 'manpage_506_TIFFClose_check_with_TIFFFlush' into 'master' + manpage: Note on using TIFFFlush() before TIFFClose() + + Closes #506 + + See merge request libtiff/libtiff!555 + +2023-11-16 Su_Laus <[email protected]> + + manpage: Note on using TIFFFlush() before TIFFClose() to check that the data has been successfully written to the file. + Closes #506 + +2023-11-11 Even Rouault <[email protected]> + + Merge branch 'manpage_621_TIFFOpenOptionsSetMaxSingleMemAlloc' into 'master' + manpage: Update TIFF documentation about TIFFOpenOptions.rst and memory limitting. + + Closes #621 + + See merge request libtiff/libtiff!553 + +2023-11-11 Even Rouault <[email protected]> + + Merge branch 'rectify_some_shadowed_declarations' into 'master' + Rectify some shadowed declarations and warnings in tools and test modules. + + See merge request libtiff/libtiff!552 + +2023-11-11 Even Rouault <[email protected]> + + Merge branch 'fix_TIFFRealloc_usage_to_TIFFReallocExt' into 'master' + Change internally used _TIFFRealloc() to _TIFFReallocExt() + + See merge request libtiff/libtiff!554 + +2023-11-11 Su_Laus <[email protected]> + + Change internally used _TIFFRealloc() to _TIFFReallocExt(), which respects the TIFFOpenOptionsSetMaxSingleMemAlloc() limit. + +2023-11-11 Timothy Lyanguzov <[email protected]> + + Fix typo. + +2023-11-09 Su_Laus <[email protected]> + + manpage: Update TIFF documentation about TIFFOpenOptions.rst and TIFFOpenOptionsSetMaxSingleMemAlloc() usage and some other small fixes. + +2023-11-08 Su_Laus <[email protected]> + + Rectify some shadowed declarations and warnings in tools and test modules. + +2023-11-04 Even Rouault <[email protected]> + + Merge branch 'fix_583' into 'master' + tif_fax3.c: error out after a number of times end-of-file has been reached (fixes #583) + + Closes #583 + + See merge request libtiff/libtiff!547 + +2023-11-04 Even Rouault <[email protected]> + + Merge branch 'fix_622' into 'master' + TIFFReadRGBAStrip/TIFFReadRGBATile: add more validation of col/row (fixes #622) + + Closes #622 + + See merge request libtiff/libtiff!546 + +2023-10-31 Even Rouault <[email protected]> + + Merge branch 'extra_limit_getfilesize' into 'master' + tif_dirread.c: only issue TIFFGetFileSize() for large enough RAM requests + + See merge request libtiff/libtiff!551 + +2023-10-31 Even Rouault <[email protected]> + + tif_dirread.c: only issue TIFFGetFileSize() for large enough RAM requests + +2023-10-31 Even Rouault <[email protected]> + + Merge branch 'TIFFFetchDirectory_remove_useless_check' into 'master' + TIFFFetchDirectory(): remove useless allocsize vs filesize check + + See merge request libtiff/libtiff!550 + +2023-10-31 Even Rouault <[email protected]> + + TIFFFetchDirectory(): remove useless allocsize vs filesize check. + CoverityScan rightly points that the max value for dircount16 * dirsize + is 4096 * 20. That's small enough not to do any check + +2023-10-31 Even Rouault <[email protected]> + + Merge branch 'limit_getfilesize' into 'master' + tif_dirread.c: only issue TIFFGetFileSize() for large enough RAM requests + + See merge request libtiff/libtiff!549 + +2023-10-31 Even Rouault <[email protected]> + + tif_dirread.c: only issue TIFFGetFileSize() for large enough RAM requests + Ammends 5320c9d89c054fa805d037d84c57da874470b01a + + This fixes a performance regression caught by the GDAL regression test + suite. + +2023-10-31 Even Rouault <[email protected]> + + Merge branch 'tif_dirread-typo' into 'master' + tif_dirread.c: Fix typo: greather -> greater + + See merge request libtiff/libtiff!548 + +2023-10-31 Kurt Schwehr <[email protected]> + + tif_dirread.c: Fix typo: greather -> greater. + +2023-10-31 Even Rouault <[email protected]> + + Merge branch 'fix_614_check_mem_allocation_vs_filelength' into 'master' + Prevent some out-of-memory attacks + + See merge request libtiff/libtiff!545 + +2023-10-31 Su Laus <[email protected]> + + Prevent some out-of-memory attacks. + Some small fuzzer files fake large amounts of data and provoke out-of-memory situations. For non-compressed data content / tags, out-of-memory can be prevented by comparing with the file size. + + At image reading, data size of some tags / data structures (StripByteCounts, StripOffsets, StripArray, TIFF directory) is compared with file size to prevent provoked out-of-memory attacks. + + See issue https://gitlab.com/libtiff/libtiff/-/issues/614#note_1602683857 + +2023-10-31 Even Rouault <[email protected]> + + tif_fax3.c: error out after a number of times end-of-file has been reached (fixes #583) + + TIFFReadRGBAStrip/TIFFReadRGBATile: add more validation of col/row (fixes #622) + +2023-10-30 Even Rouault <[email protected]> + + Merge branch 'fix_failed_subdirectory_read' into 'master' + #618: ensure absolute seeking is forced independent + + See merge request libtiff/libtiff!543 + +2023-10-25 Manuel Massing <[email protected]> + + #618: ensure absolute seeking is forced independent of TIFFReadDirectory success. + This fixes TIFFSetDirectory calls for the case where + TIFFReadDirectory was unsuccessful. + +2023-10-17 Even Rouault <[email protected]> + + Merge branch 'cmake' into 'master' + CMake: Fix TIFF_INCLUDE_DIRS + + See merge request libtiff/libtiff!541 + +2023-10-12 Even Rouault <[email protected]> + + Merge branch 'fix_616' into 'master' + tif_lzw.c: avoid warning about misaligned address with UBSAN (fixes #616) + + Closes #616 + + See merge request libtiff/libtiff!542 + +2023-10-12 Even Rouault <[email protected]> + + Merge branch 'tiffcp_cancel_ycbcr' into 'master' + tiffcp: replace PHOTOMETRIC_YCBCR with PHOTOMETRIC_RGB when outputing to... + + See merge request libtiff/libtiff!540 + +2023-10-12 Even Rouault <[email protected]> + + tif_lzw.c: avoid warning about misaligned address with UBSAN (fixes #616) + The branch suppressed by this commit wasn't useful as I've verified that + with -O2 and -O3 and gcc or clang, doing a memcpy() + _byteswap_uint64() + generates exactly the same code as _byteswap_uint64(*ptr). + The warning with UBSAN thus only comes from UBSAN generating special code + to catch the misalignment, but is unlikely to cause any issue in + practice given the above observation. + +2023-10-12 Julien Schueller <[email protected]> + + CMake: Fix TIFF_INCLUDE_DIRS. + +2023-10-08 Even Rouault <[email protected]> + + tiffcp: preserve TIFFTAG_REFERENCEBLACKWHITE when doing YCbCr JPEG -> YCbCr JPEG + +2023-10-07 Even Rouault <[email protected]> + + tiffcp: replace PHOTOMETRIC_YCBCR with PHOTOMETRIC_RGB when outputing to compression != JPEG (refs #571) + +2023-10-05 Even Rouault <[email protected]> + + Merge branch 'kmilos-master-patch-42063' into 'master' + CMake: MinGW compilers don't need a .def file for shared library + + See merge request libtiff/libtiff!448 + +2023-10-05 Even Rouault <[email protected]> + + Merge branch 'kmilos-master-patch-64997' into 'master' + CMake: move libdeflate and Lerc to Requires.private + + See merge request libtiff/libtiff!536 + +2023-10-05 Even Rouault <[email protected]> + + Merge branch 'clarify_archive' into 'master' + Doc: clarify unsupported (still in tarball) and archived (only in git) utilities + + See merge request libtiff/libtiff!532 + +2023-10-05 Even Rouault <[email protected]> + + Merge branch 'fix_571_alternative' into 'master' + tiffcp: Setting the correct photometric interpretation for the output image... + + Closes #571 + + See merge request libtiff/libtiff!539 + +2023-10-04 Even Rouault <[email protected]> + + tiffcp: do not copy tags YCBCRCOEFFICIENTS, YCBCRSUBSAMPLING, YCBCRPOSITIONING, REFERENCEBLACKWHITE. Only set TIFFTAG_YCBCRSUBSAMPLING when generating YCbCr JPEG + + tiffcp: Setting the correct photometric interpretation for the output image when converting JPEG-YCBCR to JPEG-RGB images. + Closes #571 + +2023-09-28 Miloš Komarčević <[email protected]> + + CMake: move libdeflate and Lerc to Requires.private. + +2023-09-27 Even Rouault <[email protected]> + + Merge branch 'kmilos-windres' into 'master' + CMake: enable resource compilation on all Windows + + See merge request libtiff/libtiff!537 + +2023-09-27 Miloš Komarčević <[email protected]> + + CMake: enable resource compilation on all Windows. + +2023-09-15 Even Rouault <[email protected]> + + Merge branch 'fix_608' into 'master' + tif_dirinfo.c: re-enable TIFFTAG_EP_CFAREPEATPATTERNDIM and TIFFTAG_EP_CFAPATTERN tags (fixes #608) + + Closes #608 + + See merge request libtiff/libtiff!534 + +2023-09-15 Even Rouault <[email protected]> + + tif_dirinfo.c: re-enable TIFFTAG_EP_CFAREPEATPATTERNDIM and TIFFTAG_EP_CFAPATTERN tags (fixes #608) + +2023-09-09 Even Rouault <[email protected]> + + Merge branch 'fix_606_tiffcp_check_also_input_compression_codec' into 'master' + tiffcp: Fixes #606. Check also codec of input image, not only from output image. + + Closes #606 + + See merge request libtiff/libtiff!533 + +2023-09-09 Su_Laus <[email protected]> + + Check also if codec of input image is available, independently from codec check of output image and return with error if not. Fixes #606. + +2023-09-08 Even Rouault <[email protected]> + + Doc: clarify unsupported (still in tarball) and archived (only in git) utilities + +2023-09-08 Even Rouault <[email protected]> + + Merge branch 'update_release_date' into 'master' + Update RELEASE-DATE + + See merge request libtiff/libtiff!531 + +2023-09-08 Even Rouault <[email protected]> + + Update RELEASE-DATE. + +2023-09-08 Even Rouault <[email protected]> + + Merge branch 'fix_605' into 'master' + autoconf.ac: fix detection of windows.h for mingw (fixes #605) + + Closes #605 + + See merge request libtiff/libtiff!530 + +2023-09-07 Even Rouault <[email protected]> + + autoconf.ac: fix detection of windows.h for mingw (fixes #605) + +2023-09-06 Even Rouault <[email protected]> + + Merge branch 'prepare_v4_6_0' into 'master' + Prepare for libtiff 4.6.0 release + + See merge request libtiff/libtiff!528 + +2023-09-05 Even Rouault <[email protected]> + + doc/index.rst: add warning about tools removal. + +2023-09-05 Even Rouault <[email protected]> + + Merge branch 'fix_603_readme_doc_reference_update' into 'master' + Update documentation reference within README.md. Fixes #603. + + Closes #603 + + See merge request libtiff/libtiff!529 + +2023-09-05 Su_Laus <[email protected]> + + Update documentation reference within README.md. Fixes #603. + +2023-09-05 Even Rouault <[email protected]> + + Fix references to mailing list. + 2023-09-05 Even Rouault <[email protected]> libtiff v4.6.0 released @@ -299,7 +1882,7 @@ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59751 In 738e0409 (refs #575), we disabled DNG / EP tags, but there was a - special proessing for TIFFTAG_EP_BATTERYLEVEL that must be disabled + special processing for TIFFTAG_EP_BATTERYLEVEL that must be disabled since the tag is no longer defined. 2023-06-09 Su_Laus <[email protected]> @@ -395,8 +1978,8 @@ tif_dirinfo.c: disable DNG 1.2->1.6 tags. They were added per b90b20d36d7833f54a1f3014c324f6c21b988006 but it has been found in https://gitlab.com/libtiff/libtiff/-/issues/575 that it - could cause compatibity issues with libtiff users, so this addition - should be defered for a feature release (likely 4.6.0) and not a patch one. + could cause compatibility issues with libtiff users, so this addition + should be deferred for a feature release (likely 4.6.0) and not a patch one. Fixes #575 @@ -576,7 +2159,7 @@ 2023-05-06 Even Rouault <[email protected]> Merge branch 'fix_548' into 'master' - LZWDecode(): avoid crash when trying to read again from a strip whith a... + LZWDecode(): avoid crash when trying to read again from a strip with a... Closes #548 @@ -610,7 +2193,7 @@ 2023-04-29 Even Rouault <[email protected]> - LZWDecode(): avoid crash when trying to read again from a strip whith a missing end-of-information marker (fixes #548) + LZWDecode(): avoid crash when trying to read again from a strip with a missing end-of-information marker (fixes #548) 2023-04-25 Su_Laus <[email protected]> @@ -1339,7 +2922,7 @@ Use structures that can be extended as extra argument. - Leverages and ammends https://gitlab.com/libtiff/libtiff/-/merge_requests/409 + Leverages and amends https://gitlab.com/libtiff/libtiff/-/merge_requests/409 2022-11-21 Even Rouault <[email protected]> @@ -2077,7 +3660,7 @@ to { TIFFTAG_INTEROPERABILITYIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_IFD8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 0, 0, "InteroperabilityIFDOffset", NULL }, - However, if those tags schould not be handled by LibTiff because they are deamed as abandoned or unwanted tags, those tags need to be defined with FIELD_IGNORE instead of FIELD_CUSTOM and keeping set_field_type = TIFF_SETGET_UNDEFINED + However, if those tags should not be handled by LibTiff because they are deemed as abandoned or unwanted tags, those tags need to be defined with FIELD_IGNORE instead of FIELD_CUSTOM and keeping set_field_type = TIFF_SETGET_UNDEFINED 2022-07-01 Su_Laus <[email protected]> @@ -2185,7 +3768,7 @@ tif_predict.c: make horAcc8() work with icc (ICC) 2021.6.0 20220226 -O2. For a reason I don't understand, recent ICC generates wrong code in -O2 mode for the stride = 3 and 4 cases. The modified code is more - straightfoward, so go for it. + straightforward, so go for it. 2022-06-19 Roger Leigh <[email protected]> @@ -2442,7 +4025,7 @@ Include stdlib.h in tif_lzw.c. In `tif_lzw.c`, a call is made to `_byteswap_uint64`. This is declared in `stdlib.h`. `stdlib.h` is not included in `tib_lzw.c`, so a name error may occur. - This change adds `#include stdlib.h` to `tif_lzw.c`, to prevent a name error from occuring when `stdlib.h` is not included. + This change adds `#include stdlib.h` to `tif_lzw.c`, to prevent a name error from occurring when `stdlib.h` is not included. 2022-05-29 Roger Leigh <[email protected]> @@ -2662,7 +4245,7 @@ 2022-04-22 Even Rouault <[email protected]> - tif_lzw.c: fix potential out-of-bounds error when trying to read in the same tile/strip after an error has occured (fixes #410) + tif_lzw.c: fix potential out-of-bounds error when trying to read in the same tile/strip after an error has occurred (fixes #410) 2022-04-06 Su_Laus <[email protected]> @@ -3035,7 +4618,7 @@ 2022-02-11 Even Rouault <[email protected]> Merge branch 'predictor_2_64bit' into 'master' - Predictor 2 (horizontal differenciation): support 64-bit + Predictor 2 (horizontal differentiation): support 64-bit See merge request libtiff/libtiff!296 @@ -3054,7 +4637,7 @@ 2022-02-09 Even Rouault <[email protected]> - Predictor 2 (horizontal differenciation): support 64-bit. + Predictor 2 (horizontal differentiation): support 64-bit. There's no reason not to support 64-bit. The TIFF 6 specification doesn't say anything about that (and even mention 4-bit, which we don't support) @@ -3237,8 +4820,8 @@ 2021-12-13 Even Rouault <[email protected]> OJPEG: avoid assertion when using TIFFReadScanline() (fixes #337) - Note: my analyis of the issue would be that the use of the scanline API - is currently propably broken with OJPEG. + Note: my analysis of the issue would be that the use of the scanline API + is currently probably broken with OJPEG. 2021-12-10 Even Rouault <[email protected]> @@ -3492,7 +5075,7 @@ 2021-09-23 Even Rouault <[email protected]> TIFFAppendToStrip(): fix rewrite-in-place logic. - reproducable in particular with packbits compression. + reproducible in particular with packbits compression. Fixes https://github.com/OSGeo/gdal/issues/4538 @@ -3960,7 +5543,7 @@ 2021-03-07 Antonio Valentino <[email protected]> Add LERC plugin. - The lerc plugin code has been copyed form GDAL. + The lerc plugin code has been copied from GDAL. 2021-03-06 Roger Leigh <[email protected]> @@ -4160,7 +5743,7 @@ Correct include order. - Eliminate implict fallthrough usage. + Eliminate implicit fallthrough usage. Use simple loops in place of manual loop unrolling. Rely on the compiler optimiser to unroll loops when appropriate. @@ -4422,7 +6005,7 @@ 2021-01-18 Even Rouault <[email protected]> Merge branch 'tiffcmp' into 'master' - tiffcmp: fix comparaison with pixels that are fractional number of bytes + tiffcmp: fix comparison with pixels that are fractional number of bytes Closes #53 @@ -4438,8 +6021,8 @@ 2021-01-15 Thomas Bernard <[email protected]> - tiffcmp: fix comparaison with pixels that are fractional number of bytes - For exemple : 4bits per sample + 3 samples per pixel => 1.5 bytes per pixel + tiffcmp: fix comparison with pixels that are fractional number of bytes + For example : 4bits per sample + 3 samples per pixel => 1.5 bytes per pixel tiff2ps: exit the loop in case of error. fixes #232 @@ -4634,7 +6217,7 @@ 2020-12-31 Kurt Schwehr <[email protected]> custom_dir_EXIF_231.c: dos2unix and codespell. - additonal, Varable, greather, alwasy + additional, Variable, greater, always 2020-12-31 Kurt Schwehr <[email protected]> @@ -7740,7 +9323,7 @@ 2017-11-23 Roger Leigh <[email protected]> - appveyor: Correct path for git clone and skip artefact archival. + appveyor: Correct path for git clone and skip artifact archival. 2017-11-22 Roger Leigh <[email protected]> diff --git a/contrib/libs/libtiff/RELEASE-DATE b/contrib/libs/libtiff/RELEASE-DATE index 68d943433ec..2abf40291e6 100644 --- a/contrib/libs/libtiff/RELEASE-DATE +++ b/contrib/libs/libtiff/RELEASE-DATE @@ -1 +1 @@ -20230908 +20240911 diff --git a/contrib/libs/libtiff/libtiff.map b/contrib/libs/libtiff/libtiff.map index 05da6d7e99e..3d15d9c76d2 100644 --- a/contrib/libs/libtiff/libtiff.map +++ b/contrib/libs/libtiff/libtiff.map @@ -214,3 +214,7 @@ LIBTIFF_4.5 { TIFFOpenOptionsSetErrorHandlerExtR; TIFFOpenOptionsSetWarningHandlerExtR; } LIBTIFF_4.4; + +LIBTIFF_4.6.1 { + TIFFOpenOptionsSetMaxCumulatedMemAlloc; +} LIBTIFF_4.5; diff --git a/contrib/libs/libtiff/tif_aux.c b/contrib/libs/libtiff/tif_aux.c index 49855bb0b20..e9606a42e0f 100644 --- a/contrib/libs/libtiff/tif_aux.c +++ b/contrib/libs/libtiff/tif_aux.c @@ -385,53 +385,6 @@ int TIFFGetFieldDefaulted(TIFF *tif, uint32_t tag, ...) return (ok); } -struct _Int64Parts -{ - int32_t low, high; -}; - -typedef union -{ - struct _Int64Parts part; - int64_t value; -} _Int64; - -float _TIFFUInt64ToFloat(uint64_t ui64) -{ - _Int64 i; - - i.value = ui64; - if (i.part.high >= 0) - { - return (float)i.value; - } - else - { - long double df; - df = (long double)i.value; - df += 18446744073709551616.0; /* adding 2**64 */ - return (float)df; - } -} - -double _TIFFUInt64ToDouble(uint64_t ui64) -{ - _Int64 i; - - i.value = ui64; - if (i.part.high >= 0) - { - return (double)i.value; - } - else - { - long double df; - df = (long double)i.value; - df += 18446744073709551616.0; /* adding 2**64 */ - return (double)df; - } -} - float _TIFFClampDoubleToFloat(double val) { if (val > FLT_MAX) diff --git a/contrib/libs/libtiff/tif_close.c b/contrib/libs/libtiff/tif_close.c index 907d7f139b7..d6bb7f1d7cc 100644 --- a/contrib/libs/libtiff/tif_close.c +++ b/contrib/libs/libtiff/tif_close.c @@ -110,6 +110,14 @@ void TIFFCleanup(TIFF *tif) _TIFFfreeExt(tif, tif->tif_fieldscompat); } + if (tif->tif_cur_cumulated_mem_alloc != 0) + { + TIFFErrorExtR(tif, "TIFFCleanup", + "tif_cur_cumulated_mem_alloc = %" PRIu64 " whereas it " + "should be 0", + (uint64_t)tif->tif_cur_cumulated_mem_alloc); + } + _TIFFfreeExt(NULL, tif); } diff --git a/contrib/libs/libtiff/tif_color.c b/contrib/libs/libtiff/tif_color.c index 2d7dcac6fe6..a52fdacba59 100644 --- a/contrib/libs/libtiff/tif_color.c +++ b/contrib/libs/libtiff/tif_color.c @@ -89,7 +89,7 @@ void TIFFCIELab16ToXYZ(TIFFCIELabToRGB *cielab, uint32_t l, int32_t a, void TIFFXYZToRGB(TIFFCIELabToRGB *cielab, float X, float Y, float Z, uint32_t *r, uint32_t *g, uint32_t *b) { - int i; + size_t i; float Yr, Yg, Yb; float *matrix = &cielab->display.d_mat[0][0]; @@ -109,16 +109,16 @@ void TIFFXYZToRGB(TIFFCIELabToRGB *cielab, float X, float Y, float Z, Yb = TIFFmin(Yb, cielab->display.d_YCB); /* Turn luminosity to colour value. */ - i = (int)((Yr - cielab->display.d_Y0R) / cielab->rstep); - i = TIFFmin(cielab->range, i); + i = (size_t)((Yr - cielab->display.d_Y0R) / cielab->rstep); + i = TIFFmin((size_t)cielab->range, i); *r = RINT(cielab->Yr2r[i]); - i = (int)((Yg - cielab->display.d_Y0G) / cielab->gstep); - i = TIFFmin(cielab->range, i); + i = (size_t)((Yg - cielab->display.d_Y0G) / cielab->gstep); + i = TIFFmin((size_t)cielab->range, i); *g = RINT(cielab->Yg2g[i]); - i = (int)((Yb - cielab->display.d_Y0B) / cielab->bstep); - i = TIFFmin(cielab->range, i); + i = (size_t)((Yb - cielab->display.d_Y0B) / cielab->bstep); + i = TIFFmin((size_t)cielab->range, i); *b = RINT(cielab->Yb2b[i]); /* Clip output. */ @@ -135,7 +135,7 @@ void TIFFXYZToRGB(TIFFCIELabToRGB *cielab, float X, float Y, float Z, int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, float *refWhite) { - int i; + size_t i; double dfGamma; cielab->range = CIELABTORGB_TABLE_RANGE; @@ -146,7 +146,7 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, dfGamma = 1.0 / cielab->display.d_gammaR; cielab->rstep = (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; - for (i = 0; i <= cielab->range; i++) + for (i = 0; i <= (size_t)cielab->range; i++) { cielab->Yr2r[i] = cielab->display.d_Vrwr * ((float)pow((double)i / cielab->range, dfGamma)); @@ -156,7 +156,7 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, dfGamma = 1.0 / cielab->display.d_gammaG; cielab->gstep = (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; - for (i = 0; i <= cielab->range; i++) + for (i = 0; i <= (size_t)cielab->range; i++) { cielab->Yg2g[i] = cielab->display.d_Vrwg * ((float)pow((double)i / cielab->range, dfGamma)); @@ -166,7 +166,7 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, dfGamma = 1.0 / cielab->display.d_gammaB; cielab->bstep = (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; - for (i = 0; i <= cielab->range; i++) + for (i = 0; i <= (size_t)cielab->range; i++) { cielab->Yb2b[i] = cielab->display.d_Vrwb * ((float)pow((double)i / cielab->range, dfGamma)); diff --git a/contrib/libs/libtiff/tif_config.h b/contrib/libs/libtiff/tif_config.h index 74d59662744..2d6ea9bf6bc 100644 --- a/contrib/libs/libtiff/tif_config.h +++ b/contrib/libs/libtiff/tif_config.h @@ -40,6 +40,18 @@ /* Define to 1 if you have the `getopt' function. */ #define HAVE_GETOPT 1 +/* Define to 1 if you have the <GLUT/glut.h> header file. */ +/* #undef HAVE_GLUT_GLUT_H */ + +/* Define to 1 if you have the <GL/glut.h> header file. */ +/* #undef HAVE_GL_GLUT_H */ + +/* Define to 1 if you have the <GL/glu.h> header file. */ +/* #undef HAVE_GL_GLU_H */ + +/* Define to 1 if you have the <GL/gl.h> header file. */ +/* #undef HAVE_GL_GL_H */ + /* Define to 1 if you have the <io.h> header file. */ /* #undef HAVE_IO_H */ @@ -49,6 +61,12 @@ /* Define to 1 if you have the `mmap' function. */ #define HAVE_MMAP 1 +/* Define to 1 if you have the <OpenGL/glu.h> header file. */ +/* #undef HAVE_OPENGL_GLU_H */ + +/* Define to 1 if you have the <OpenGL/gl.h> header file. */ +/* #undef HAVE_OPENGL_GL_H */ + /* Define to 1 if you have the `setmode' function. */ /* #undef HAVE_SETMODE */ diff --git a/contrib/libs/libtiff/tif_dir.c b/contrib/libs/libtiff/tif_dir.c index f1adf080a08..35375aa6e24 100644 --- a/contrib/libs/libtiff/tif_dir.c +++ b/contrib/libs/libtiff/tif_dir.c @@ -211,7 +211,7 @@ static uint16_t countInkNamesString(TIFF *tif, uint32_t slen, const char *s) } bad: TIFFErrorExtR(tif, "TIFFSetField", - "%s: Invalid InkNames value; no NUL at given buffer end " + "%s: Invalid InkNames value; no null at given buffer end " "location %" PRIu32 ", after %" PRIu16 " ink", tif->tif_name, slen, i); return (0); @@ -1652,6 +1652,17 @@ void TIFFFreeDirectory(TIFF *tif) _TIFFmemset(&(td->td_stripoffset_entry), 0, sizeof(TIFFDirEntry)); _TIFFmemset(&(td->td_stripbytecount_entry), 0, sizeof(TIFFDirEntry)); + + /* Reset some internal parameters for IFD data size checking. */ + tif->tif_dir.td_dirdatasize_read = 0; + tif->tif_dir.td_dirdatasize_write = 0; + if (tif->tif_dir.td_dirdatasize_offsets != NULL) + { + _TIFFfreeExt(tif, tif->tif_dir.td_dirdatasize_offsets); + tif->tif_dir.td_dirdatasize_offsets = NULL; + tif->tif_dir.td_dirdatasize_Noffsets = 0; + } + tif->tif_dir.td_iswrittentofile = FALSE; } #undef CleanupField @@ -1676,18 +1687,23 @@ TIFFExtendProc TIFFSetTagExtender(TIFFExtendProc extender) */ int TIFFCreateDirectory(TIFF *tif) { + /* Free previously allocated memory and setup default values. */ + TIFFFreeDirectory(tif); TIFFDefaultDirectory(tif); tif->tif_diroff = 0; tif->tif_nextdiroff = 0; tif->tif_curoff = 0; tif->tif_row = (uint32_t)-1; tif->tif_curstrip = (uint32_t)-1; + tif->tif_dir.td_iswrittentofile = FALSE; return 0; } int TIFFCreateCustomDirectory(TIFF *tif, const TIFFFieldArray *infoarray) { + /* Free previously allocated memory and setup default values. */ + TIFFFreeDirectory(tif); TIFFDefaultDirectory(tif); /* @@ -1848,7 +1864,9 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, if (((uint64_t)poffa != poff) || (poffb < poffa) || (poffb < (tmsize_t)sizeof(uint16_t)) || (poffb > tif->tif_size)) { - TIFFErrorExtR(tif, module, "Error fetching directory count"); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); *nextdiroff = 0; return (0); } @@ -1877,14 +1895,18 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, uint16_t dircount16; if (poff > (uint64_t)TIFF_TMSIZE_T_MAX - sizeof(uint64_t)) { - TIFFErrorExtR(tif, module, "Error fetching directory count"); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); return (0); } poffa = (tmsize_t)poff; poffb = poffa + sizeof(uint64_t); if (poffb > tif->tif_size) { - TIFFErrorExtR(tif, module, "Error fetching directory count"); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); return (0); } _TIFFmemcpy(&dircount64, tif->tif_base + poffa, sizeof(uint64_t)); @@ -1926,8 +1948,9 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, if (!SeekOK(tif, *nextdiroff) || !ReadOK(tif, &dircount, sizeof(uint16_t))) { - TIFFErrorExtR(tif, module, "%s: Error fetching directory count", - tif->tif_name); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); return (0); } if (tif->tif_flags & TIFF_SWAB) @@ -1953,15 +1976,18 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, if (!SeekOK(tif, *nextdiroff) || !ReadOK(tif, &dircount64, sizeof(uint64_t))) { - TIFFErrorExtR(tif, module, "%s: Error fetching directory count", - tif->tif_name); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); return (0); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8(&dircount64); if (dircount64 > 0xFFFF) { - TIFFErrorExtR(tif, module, "Error fetching directory count"); + TIFFErrorExtR(tif, module, + "%s:%d: %s: Error fetching directory count", + __FILE__, __LINE__, tif->tif_name); return (0); } dircount16 = (uint16_t)dircount64; @@ -2018,6 +2044,8 @@ tdir_t TIFFNumberOfDirectories(TIFF *tif) { ++n; } + /* Update number of main-IFDs in file. */ + tif->tif_curdircount = n; return (n); } @@ -2100,7 +2128,19 @@ int TIFFSetDirectory(TIFF *tif, tdir_t dirn) tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; else tif->tif_curdir--; - return (TIFFReadDirectory(tif)); + + tdir_t curdir = tif->tif_curdir; + + int retval = TIFFReadDirectory(tif); + + if (!retval && tif->tif_curdir == curdir) + { + /* If tif_curdir has not be incremented, TIFFFetchDirectory() in + * TIFFReadDirectory() has failed and tif_curdir shall be set + * specifically. */ + tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; + } + return (retval); } /* @@ -2125,8 +2165,11 @@ int TIFFSetSubDirectory(TIFF *tif, uint64_t diroff) int8_t probablySubIFD = 0; if (diroff == 0) { - /* Special case to invalidate the tif_lastdiroff member. */ + /* Special case to set tif_diroff=0, which is done in + * TIFFReadDirectory() below to indicate that the currently read IFD is + * treated as a new, fresh IFD. */ tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; + tif->tif_dir.td_iswrittentofile = FALSE; } else { @@ -2136,32 +2179,40 @@ int TIFFSetSubDirectory(TIFF *tif, uint64_t diroff) probablySubIFD = 1; } /* -1 because TIFFReadDirectory() will increment tif_curdir. */ - tif->tif_curdir = - curdir == 0 ? TIFF_NON_EXISTENT_DIR_NUMBER : curdir - 1; + if (curdir >= 1) + tif->tif_curdir = curdir - 1; + else + tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; } + curdir = tif->tif_curdir; tif->tif_nextdiroff = diroff; retval = TIFFReadDirectory(tif); - /* If failed, curdir was not incremented in TIFFReadDirectory(), so set it - * back, but leave it for diroff==0. */ - if (!retval && diroff != 0) + + /* tif_curdir is incremented in TIFFReadDirectory(), but if it has not been + * incremented, TIFFFetchDirectory() has failed there and tif_curdir shall + * be set specifically. */ + if (!retval && diroff != 0 && tif->tif_curdir == curdir) { - if (tif->tif_curdir == TIFF_NON_EXISTENT_DIR_NUMBER) - tif->tif_curdir = 0; - else - tif->tif_curdir++; + tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; } - if (retval && probablySubIFD) + + if (probablySubIFD) { - /* Reset IFD list to start new one for SubIFD chain and also start - * SubIFD chain with tif_curdir=0. */ - _TIFFCleanupIFDOffsetAndNumberMaps(tif); /* invalidate IFD loop lists */ - tif->tif_curdir = 0; /* first directory of new chain */ - /* add this offset to new IFD list */ - _TIFFCheckDirNumberAndOffset(tif, tif->tif_curdir, diroff); + if (retval) + { + /* Reset IFD list to start new one for SubIFD chain and also start + * SubIFD chain with tif_curdir=0 for IFD loop checking. */ + /* invalidate IFD loop lists */ + _TIFFCleanupIFDOffsetAndNumberMaps(tif); + tif->tif_curdir = 0; /* first directory of new chain */ + /* add this offset to new IFD list */ + _TIFFCheckDirNumberAndOffset(tif, tif->tif_curdir, diroff); + } /* To be able to return from SubIFD or custom-IFD to main-IFD */ tif->tif_setdirectory_force_absolute = TRUE; } + return (retval); } @@ -2257,9 +2308,11 @@ int TIFFUnlinkDirectory(TIFF *tif, tdir_t dirn) } else { + /* Need local swap because nextdir has to be used unswapped below. */ + uint64_t nextdir64 = nextdir; if (tif->tif_flags & TIFF_SWAB) - TIFFSwabLong8(&nextdir); - if (!WriteOK(tif, &nextdir, sizeof(uint64_t))) + TIFFSwabLong8(&nextdir64); + if (!WriteOK(tif, &nextdir64, sizeof(uint64_t))) { TIFFErrorExtR(tif, module, "Error writing directory link"); return (0); @@ -2303,6 +2356,10 @@ int TIFFUnlinkDirectory(TIFF *tif, tdir_t dirn) tif->tif_row = (uint32_t)-1; tif->tif_curstrip = (uint32_t)-1; tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; + if (tif->tif_curdircount > 0) + tif->tif_curdircount--; + else + tif->tif_curdircount = TIFF_NON_EXISTENT_DIR_NUMBER; _TIFFCleanupIFDOffsetAndNumberMaps(tif); /* invalidate IFD loop lists */ return (1); } diff --git a/contrib/libs/libtiff/tif_dir.h b/contrib/libs/libtiff/tif_dir.h index 9eaf22f8e62..ffad085e02e 100644 --- a/contrib/libs/libtiff/tif_dir.h +++ b/contrib/libs/libtiff/tif_dir.h @@ -65,6 +65,12 @@ typedef struct tif_dirread.c */ } TIFFDirEntry; +typedef struct +{ + uint64_t offset; + uint64_t length; +} TIFFEntryOffsetAndLength; /* auxiliary for evaluating size of IFD data */ + /* * Internal format of a TIFF directory entry. */ @@ -115,6 +121,9 @@ typedef struct #ifdef STRIPBYTECOUNTSORTED_UNUSED int td_stripbytecountsorted; /* is the bytecount array sorted ascending? */ #endif + /* Be aware that the parameters of td_stripoffset_entry and + * td_stripbytecount_entry are swapped but tdir_offset is not + * and has to be swapped when used. */ TIFFDirEntry td_stripoffset_entry; /* for deferred loading */ TIFFDirEntry td_stripbytecount_entry; /* for deferred loading */ uint16_t td_nsubifd; @@ -135,6 +144,24 @@ typedef struct unsigned char td_deferstrilearraywriting; /* see TIFFDeferStrileArrayWriting() */ + + unsigned char + td_iswrittentofile; /* indicates if current IFD is present on file */ + + /* LibTIFF writes all data that does not fit into the IFD entries directly + * after the IFD tag entry part. When reading, only the IFD data directly + * and continuously behind the IFD tags is taken into account for the IFD + * data size.*/ + uint64_t td_dirdatasize_write; /* auxiliary for evaluating size of IFD data + to be written */ + uint64_t td_dirdatasize_read; /* auxiliary for evaluating size of IFD data + read from file */ + uint32_t td_dirdatasize_Noffsets; /* auxiliary counter for + tif_dir.td_dirdatasize_offsets array */ + TIFFEntryOffsetAndLength + *td_dirdatasize_offsets; /* auxiliary array for all offsets of IFD tag + entries with data outside the IFD tag + entries. */ } TIFFDirectory; /* @@ -308,11 +335,10 @@ extern "C" TIFFDataType field_type; /* type of associated data */ uint32_t field_anonymous; /* if true, this is a unknown / anonymous tag */ - TIFFSetGetFieldType - set_field_type; /* type to be passed to TIFFSetField */ - TIFFSetGetFieldType - get_field_type; /* type to be passed to TIFFGetField */ - unsigned short field_bit; /* bit in fieldsset bit vector */ + TIFFSetGetFieldType set_field_type; /* type to be passed to TIFFSetField + and TIFFGetField*/ + TIFFSetGetFieldType get_field_type; /* not used */ + unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ char *field_name; /* ASCII name */ diff --git a/contrib/libs/libtiff/tif_dirinfo.c b/contrib/libs/libtiff/tif_dirinfo.c index 0e705e81e3d..432e6e1e448 100644 --- a/contrib/libs/libtiff/tif_dirinfo.c +++ b/contrib/libs/libtiff/tif_dirinfo.c @@ -213,8 +213,6 @@ static const TIFFField tiffFields[] = { {TIFFTAG_CURRENTPREPROFILEMATRIX, -1, -1, TIFF_SRATIONAL, 0, TIFF_SETGET_C16_FLOAT, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 1, "CurrentPreProfileMatrix", NULL}, {TIFFTAG_PERSAMPLE, 0, 0, TIFF_SHORT, 0, TIFF_SETGET_UNDEFINED, TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, "PerSample", NULL}, #if 0 - /* TODO: revert above #if 0 for TIFF 4.6.0 */ - /* begin DNG 1.2.0.0 tags */ {TIFFTAG_COLORIMETRICREFERENCE, 1, 1, TIFF_SHORT, 0, TIFF_SETGET_UINT16, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "ColorimetricReference", NULL}, {TIFFTAG_CAMERACALIBRATIONSIGNATURE, -1, -1, TIFF_BYTE, 0, TIFF_SETGET_C16_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 1, "CameraCalibrationSignature", NULL}, @@ -282,9 +280,11 @@ static const TIFFField tiffFields[] = { {TIFFTAG_ILLUMINANTDATA2, -3, -3, TIFF_UNDEFINED, 0, TIFF_SETGET_C32_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 1, "IlluminantData2", NULL}, {TIFFTAG_ILLUMINANTDATA3, -3, -3, TIFF_UNDEFINED, 0, TIFF_SETGET_C32_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 1, "IlluminantData3", NULL}, /* end DNG tags */ +#endif /* begin TIFF/EP tags */ {TIFFTAG_EP_CFAREPEATPATTERNDIM, 2, 2, TIFF_SHORT, 0, TIFF_SETGET_C0_UINT16, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "EP CFARepeatPatternDim", NULL}, {TIFFTAG_EP_CFAPATTERN, -1, -1, TIFF_BYTE, 0, TIFF_SETGET_C16_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 1, "EP CFAPattern", NULL}, +#if 0 /* TIFFTAG_EP_BATTERYLEVEL can be RATIONAL or ASCII. * LibTiff defines it as ASCII and converts RATIONAL to an ASCII string. */ {TIFFTAG_EP_BATTERYLEVEL, -1, -1, TIFF_ASCII, 0, TIFF_SETGET_ASCII, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "EP BatteryLevel", NULL}, @@ -374,7 +374,7 @@ static const TIFFField exifFields[] = { {EXIFTAG_BRIGHTNESSVALUE, 1, 1, TIFF_SRATIONAL, 0, TIFF_SETGET_FLOAT, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "BrightnessValue", NULL}, {EXIFTAG_EXPOSUREBIASVALUE, 1, 1, TIFF_SRATIONAL, 0, TIFF_SETGET_FLOAT, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "ExposureBiasValue", NULL}, {EXIFTAG_MAXAPERTUREVALUE, 1, 1, TIFF_RATIONAL, 0, TIFF_SETGET_FLOAT, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "MaxApertureValue", NULL}, - /*--: EXIFTAG_SUBJECTDISTANCE: LibTiff returns value of "-1" if numerator equals 4294967295 (0xFFFFFFFF) to indicate infinite distance! + /*--: EXIFTAG_SUBJECTDISTANCE: LibTiff returns value of "-1" if numerator equals 4294967295 (0xFFFFFFFF) to indicate infinite distance! * However, there are two other EXIF tags where numerator indicates a special value and six other cases where the denominator indicates special values, * which are not treated within LibTiff!! */ {EXIFTAG_SUBJECTDISTANCE, 1, 1, TIFF_RATIONAL, 0, TIFF_SETGET_FLOAT, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 1, 0, "SubjectDistance", NULL}, @@ -887,7 +887,7 @@ const TIFFField *_TIFFFindOrRegisterField(TIFF *tif, uint32_t tag, if (fld == NULL) { fld = _TIFFCreateAnonField(tif, tag, dt); - if (!_TIFFMergeFields(tif, fld, 1)) + if (fld == NULL || !_TIFFMergeFields(tif, fld, 1)) return NULL; } @@ -1197,12 +1197,24 @@ int TIFFMergeFieldInfo(TIFF *tif, const TIFFFieldInfo info[], uint32_t n) for (i = 0; i < n; i++) { tp->field_tag = info[i].field_tag; + if (info[i].field_readcount < TIFF_VARIABLE2 || + info[i].field_readcount == 0 || + info[i].field_writecount < TIFF_VARIABLE2 || + info[i].field_writecount == 0) + { + /* The fields (field_readcount) and (field_writecount) may use the + * values TIFF_VARIABLE (-1), TIFF_SPP (-2), TIFF_VARIABLE2 (-3). */ + TIFFErrorExtR(tif, module, + "The value of field_readcount and field_writecount " + "must be greater than or equal to -3 and not zero."); + return -1; + } tp->field_readcount = info[i].field_readcount; tp->field_writecount = info[i].field_writecount; tp->field_type = info[i].field_type; tp->field_anonymous = 0; tp->set_field_type = - _TIFFSetGetType(info[i].field_type, info[i].field_readcount, + _TIFFSetGetType(info[i].field_type, info[i].field_writecount, info[i].field_passcount); tp->get_field_type = _TIFFSetGetType(info[i].field_type, info[i].field_readcount, diff --git a/contrib/libs/libtiff/tif_dirread.c b/contrib/libs/libtiff/tif_dirread.c index 8ee560673c9..e079f23ed58 100644 --- a/contrib/libs/libtiff/tif_dirread.c +++ b/contrib/libs/libtiff/tif_dirread.c @@ -1016,16 +1016,7 @@ TIFFReadDirEntryFloat(TIFF *tif, TIFFDirEntry *direntry, float *value) err = TIFFReadDirEntryCheckedLong8(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk) return (err); -#if defined(__WIN32__) && (_MSC_VER < 1500) - /* - * XXX: MSVC 6.0 does not support conversion - * of 64-bit integers into floating point - * values. - */ - *value = _TIFFUInt64ToFloat(m); -#else *value = (float)m; -#endif return (TIFFReadDirEntryErrOk); } case TIFF_SLONG8: @@ -1130,16 +1121,7 @@ TIFFReadDirEntryDouble(TIFF *tif, TIFFDirEntry *direntry, double *value) err = TIFFReadDirEntryCheckedLong8(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk) return (err); -#if defined(__WIN32__) && (_MSC_VER < 1500) - /* - * XXX: MSVC 6.0 does not support conversion - * of 64-bit integers into floating point - * values. - */ - *value = _TIFFUInt64ToDouble(m); -#else *value = (double)m; -#endif return (TIFFReadDirEntryErrOk); } case TIFF_SLONG8: @@ -1308,6 +1290,24 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, datasize = (*count) * typesize; assert((tmsize_t)datasize > 0); + if (datasize > 100 * 1024 * 1024) + { + /* Before allocating a huge amount of memory for corrupted files, check + * if size of requested memory is not greater than file size. + */ + const uint64_t filesize = TIFFGetFileSize(tif); + if (datasize > filesize) + { + TIFFWarningExtR(tif, "ReadDirEntryArray", + "Requested memory size for tag %d (0x%x) %" PRIu32 + " is greater than filesize %" PRIu64 + ". Memory not allocated, tag not read", + direntry->tdir_tag, direntry->tdir_tag, datasize, + filesize); + return (TIFFReadDirEntryErrAlloc); + } + } + if (isMapped(tif) && datasize > (uint64_t)tif->tif_size) return TIFFReadDirEntryErrIo; @@ -2886,16 +2886,7 @@ TIFFReadDirEntryFloatArray(TIFF *tif, TIFFDirEntry *direntry, float **value) { if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8(ma); -#if defined(__WIN32__) && (_MSC_VER < 1500) - /* - * XXX: MSVC 6.0 does not support - * conversion of 64-bit integers into - * floating point values. - */ - *mb++ = _TIFFUInt64ToFloat(*ma++); -#else *mb++ = (float)(*ma++); -#endif } } break; @@ -3131,16 +3122,7 @@ TIFFReadDirEntryDoubleArray(TIFF *tif, TIFFDirEntry *direntry, double **value) { if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8(ma); -#if defined(__WIN32__) && (_MSC_VER < 1500) - /* - * XXX: MSVC 6.0 does not support - * conversion of 64-bit integers into - * floating point values. - */ - *mb++ = _TIFFUInt64ToDouble(*ma++); -#else *mb++ = (double)(*ma++); -#endif } } break; @@ -4096,6 +4078,155 @@ static int ByteCountLooksBad(TIFF *tif) } /* + * To evaluate the IFD data size when reading, save the offset and data size of + * all data that does not fit into the IFD entries themselves. + */ +static bool EvaluateIFDdatasizeReading(TIFF *tif, TIFFDirEntry *dp) +{ + const uint64_t data_width = TIFFDataWidth(dp->tdir_type); + if (data_width != 0 && dp->tdir_count > UINT64_MAX / data_width) + { + TIFFErrorExtR(tif, "EvaluateIFDdatasizeReading", + "Too large IFD data size"); + return false; + } + const uint64_t datalength = dp->tdir_count * data_width; + if (datalength > ((tif->tif_flags & TIFF_BIGTIFF) ? 0x8U : 0x4U)) + { + if (tif->tif_dir.td_dirdatasize_read > UINT64_MAX - datalength) + { + TIFFErrorExtR(tif, "EvaluateIFDdatasizeReading", + "Too large IFD data size"); + return false; + } + tif->tif_dir.td_dirdatasize_read += datalength; + if (!(tif->tif_flags & TIFF_BIGTIFF)) + { + /* The offset of TIFFDirEntry are not swapped when read in. That has + * to be done when used. */ + uint32_t offset = dp->tdir_offset.toff_long; + if (tif->tif_flags & TIFF_SWAB) + TIFFSwabLong(&offset); + tif->tif_dir + .td_dirdatasize_offsets[tif->tif_dir.td_dirdatasize_Noffsets] + .offset = (uint64_t)offset; + } + else + { + tif->tif_dir + .td_dirdatasize_offsets[tif->tif_dir.td_dirdatasize_Noffsets] + .offset = dp->tdir_offset.toff_long8; + if (tif->tif_flags & TIFF_SWAB) + TIFFSwabLong8( + &tif->tif_dir + .td_dirdatasize_offsets[tif->tif_dir + .td_dirdatasize_Noffsets] + .offset); + } + tif->tif_dir + .td_dirdatasize_offsets[tif->tif_dir.td_dirdatasize_Noffsets] + .length = datalength; + tif->tif_dir.td_dirdatasize_Noffsets++; + } + return true; +} + +/* + * Compare function for qsort() sorting TIFFEntryOffsetAndLength array entries. + */ +static int cmpTIFFEntryOffsetAndLength(const void *a, const void *b) +{ + const TIFFEntryOffsetAndLength *ta = (const TIFFEntryOffsetAndLength *)a; + const TIFFEntryOffsetAndLength *tb = (const TIFFEntryOffsetAndLength *)b; + /* Compare offsets */ + if (ta->offset > tb->offset) + return 1; + else if (ta->offset < tb->offset) + return -1; + else + return 0; +} + +/* + * Determine the IFD data size after reading an IFD from the file that can be + * overwritten and saving it in tif_dir.td_dirdatasize_read. This data size + * includes the IFD entries themselves as well as the data that does not fit + * directly into the IFD entries but is located directly after the IFD entries + * in the file. + */ +static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) +{ + /* IFD data size is only needed if file-writing is enabled. + * This also avoids the seek() to EOF to determine the file size, which + * causes the stdin-streaming-friendly mode of libtiff for GDAL to fail. */ + if (tif->tif_mode == O_RDONLY) + return; + + /* Sort TIFFEntryOffsetAndLength array in ascending order. */ + qsort(tif->tif_dir.td_dirdatasize_offsets, + tif->tif_dir.td_dirdatasize_Noffsets, + sizeof(TIFFEntryOffsetAndLength), cmpTIFFEntryOffsetAndLength); + + /* Get offset of end of IFD entry space. */ + uint64_t IFDendoffset; + if (!(tif->tif_flags & TIFF_BIGTIFF)) + IFDendoffset = tif->tif_diroff + 2 + dircount * 12 + 4; + else + IFDendoffset = tif->tif_diroff + 8 + dircount * 20 + 8; + + /* Check which offsets are right behind IFD entries. However, LibTIFF + * increments the writing address for every external data to an even offset. + * Thus gaps of 1 byte can occur. */ + uint64_t size = 0; + uint64_t offset; + uint32_t i; + for (i = 0; i < tif->tif_dir.td_dirdatasize_Noffsets; i++) + { + offset = tif->tif_dir.td_dirdatasize_offsets[i].offset; + if (offset == IFDendoffset) + { + size += tif->tif_dir.td_dirdatasize_offsets[i].length; + IFDendoffset += tif->tif_dir.td_dirdatasize_offsets[i].length; + } + else if (offset == IFDendoffset + 1) + { + /* Add gap byte after previous IFD data set. */ + size += tif->tif_dir.td_dirdatasize_offsets[i].length + 1; + IFDendoffset += tif->tif_dir.td_dirdatasize_offsets[i].length; + } + else + { + /* Further data is no more continuously after IFD */ + break; + } + } + /* Check for gap byte of some easy cases. This should cover 90% of cases. + * Otherwise, IFD will be re-written even it might be safely overwritten. */ + if (tif->tif_nextdiroff != 0) + { + if (tif->tif_nextdiroff == IFDendoffset + 1) + size++; + } + else + { + /* Check for IFD data ends at EOF. Then IFD can always be safely + * overwritten. */ + offset = TIFFSeekFile(tif, 0, SEEK_END); + if (offset == IFDendoffset) + { + tif->tif_dir.td_dirdatasize_read = UINT64_MAX; + return; + } + } + + /* Finally, add the size of the IFD tag entries themselves. */ + if (!(tif->tif_flags & TIFF_BIGTIFF)) + tif->tif_dir.td_dirdatasize_read = 2 + dircount * 12 + 4 + size; + else + tif->tif_dir.td_dirdatasize_read = 8 + dircount * 20 + 8 + size; +} /*-- CalcFinalIFDdatasizeReading() --*/ + +/* * Read the next TIFF directory from a file and convert it to the internal * format. We read directories sequentially. */ @@ -4150,7 +4281,6 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_curdir = 0; else tif->tif_curdir++; - (*tif->tif_cleanup)(tif); /* cleanup any previous compression state */ TIFFReadDirectoryCheckOrder(tif, dir, dircount); @@ -4180,8 +4310,27 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_flags &= ~TIFF_CHOPPEDUPARRAYS; /* free any old stuff and reinit */ + (*tif->tif_cleanup)(tif); /* cleanup any previous compression state */ TIFFFreeDirectory(tif); TIFFDefaultDirectory(tif); + + /* After setup a fresh directory indicate that now active IFD is also + * present on file, even if its entries could not be read successfully + * below. */ + tif->tif_dir.td_iswrittentofile = TRUE; + + /* Allocate arrays for offset values outside IFD entry for IFD data size + * checking. Note: Counter are reset within TIFFFreeDirectory(). */ + tif->tif_dir.td_dirdatasize_offsets = + (TIFFEntryOffsetAndLength *)_TIFFmallocExt( + tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + if (tif->tif_dir.td_dirdatasize_offsets == NULL) + { + TIFFErrorExtR( + tif, module, + "Failed to allocate memory for counting IFD data size at reading"); + goto bad; + } /* * Electronic Arts writes gray-scale TIFF files * without a PlanarConfiguration directory entry. @@ -4260,11 +4409,9 @@ int TIFFReadDirectory(TIFF *tif) dp->tdir_tag, dp->tdir_tag); /* the following knowingly leaks the anonymous field structure */ - if (!_TIFFMergeFields( - tif, - _TIFFCreateAnonField(tif, dp->tdir_tag, - (TIFFDataType)dp->tdir_type), - 1)) + const TIFFField *fld = _TIFFCreateAnonField( + tif, dp->tdir_tag, (TIFFDataType)dp->tdir_type); + if (fld == NULL || !_TIFFMergeFields(tif, fld, 1)) { TIFFWarningExtR( tif, module, @@ -4380,6 +4527,8 @@ int TIFFReadDirectory(TIFF *tif) uint16_t value; enum TIFFReadDirEntryErr err; err = TIFFReadDirEntryShort(tif, dp, &value); + if (!EvaluateIFDdatasizeReading(tif, dp)) + goto bad; if (err == TIFFReadDirEntryErrCount) err = TIFFReadDirEntryPersampleShort(tif, dp, &value); @@ -4410,6 +4559,8 @@ int TIFFReadDirectory(TIFF *tif) err = TIFFReadDirEntryErrCount; else err = TIFFReadDirEntryDoubleArray(tif, dp, &data); + if (!EvaluateIFDdatasizeReading(tif, dp)) + goto bad; if (err != TIFFReadDirEntryErrOk) { fip = TIFFFieldWithTag(tif, dp->tdir_tag); @@ -4429,6 +4580,7 @@ int TIFFReadDirectory(TIFF *tif) break; case TIFFTAG_STRIPOFFSETS: case TIFFTAG_TILEOFFSETS: + { switch (dp->tdir_type) { case TIFF_SHORT: @@ -4451,9 +4603,13 @@ int TIFFReadDirectory(TIFF *tif) } _TIFFmemcpy(&(tif->tif_dir.td_stripoffset_entry), dp, sizeof(TIFFDirEntry)); - break; + if (!EvaluateIFDdatasizeReading(tif, dp)) + goto bad; + } + break; case TIFFTAG_STRIPBYTECOUNTS: case TIFFTAG_TILEBYTECOUNTS: + { switch (dp->tdir_type) { case TIFF_SHORT: @@ -4476,7 +4632,10 @@ int TIFFReadDirectory(TIFF *tif) } _TIFFmemcpy(&(tif->tif_dir.td_stripbytecount_entry), dp, sizeof(TIFFDirEntry)); - break; + if (!EvaluateIFDdatasizeReading(tif, dp)) + goto bad; + } + break; case TIFFTAG_COLORMAP: case TIFFTAG_TRANSFERFUNCTION: { @@ -4530,6 +4689,8 @@ int TIFFReadDirectory(TIFF *tif) err = TIFFReadDirEntryErrCount; else err = TIFFReadDirEntryShortArray(tif, dp, &value); + if (!EvaluateIFDdatasizeReading(tif, dp)) + goto bad; if (err != TIFFReadDirEntryErrOk) { fip = TIFFFieldWithTag(tif, dp->tdir_tag); @@ -4620,9 +4781,12 @@ int TIFFReadDirectory(TIFF *tif) default: (void)TIFFFetchNormalTag(tif, dp, TRUE); break; - } - } /* -- if (!dp->tdir_ignore) */ - } /* -- for-loop -- */ + } /* -- switch (dp->tdir_tag) -- */ + } /* -- if (!dp->tdir_ignore) */ + } /* -- for-loop -- */ + + /* Evaluate final IFD data size. */ + CalcFinalIFDdatasizeReading(tif, dircount); /* * OJPEG hack: @@ -5028,7 +5192,7 @@ bad: if (dir) _TIFFfreeExt(tif, dir); return (0); -} +} /*-- TIFFReadDirectory() --*/ static void TIFFReadDirectoryCheckOrder(TIFF *tif, TIFFDirEntry *dir, uint16_t dircount) @@ -5112,8 +5276,7 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, uint16_t di; const TIFFField *fip; uint32_t fii; - (*tif->tif_cleanup)(tif); /* cleanup any previous compression state */ - _TIFFSetupFields(tif, infoarray); + dircount = TIFFFetchDirectory(tif, diroff, &dir, NULL); if (!dircount) { @@ -5122,9 +5285,56 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, diroff); return 0; } - TIFFFreeDirectory(tif); - _TIFFmemset(&tif->tif_dir, 0, sizeof(TIFFDirectory)); TIFFReadDirectoryCheckOrder(tif, dir, dircount); + + /* + * Mark duplicates of any tag to be ignored (bugzilla 1994) + * to avoid certain pathological problems. + */ + { + TIFFDirEntry *ma; + uint16_t mb; + for (ma = dir, mb = 0; mb < dircount; ma++, mb++) + { + TIFFDirEntry *na; + uint16_t nb; + for (na = ma + 1, nb = mb + 1; nb < dircount; na++, nb++) + { + if (ma->tdir_tag == na->tdir_tag) + { + na->tdir_ignore = TRUE; + } + } + } + } + + /* Free any old stuff and reinit. */ + (*tif->tif_cleanup)(tif); /* cleanup any previous compression state */ + TIFFFreeDirectory(tif); + /* Even if custom directories do not need the default settings of a standard + * IFD, the pointer to the TIFFSetField() and TIFFGetField() (i.e. + * tif->tif_tagmethods.vsetfield and tif->tif_tagmethods.vgetfield) need to + * be initialized, which is done in TIFFDefaultDirectory(). + * After that, the field array for the custom tags needs to be setup again. + */ + TIFFDefaultDirectory(tif); + _TIFFSetupFields(tif, infoarray); + + /* Allocate arrays for offset values outside IFD entry for IFD data size + * checking. Note: Counter are reset within TIFFFreeDirectory(). */ + tif->tif_dir.td_dirdatasize_offsets = + (TIFFEntryOffsetAndLength *)_TIFFmallocExt( + tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + if (tif->tif_dir.td_dirdatasize_offsets == NULL) + { + TIFFErrorExtR( + tif, module, + "Failed to allocate memory for counting IFD data size at reading"); + if (dir) + _TIFFfreeExt(tif, dir); + return 0; + } + for (di = 0, dp = dir; di < dircount; di++, dp++) { TIFFReadDirectoryFindFieldInfo(tif, dp->tdir_tag, &fii); @@ -5134,11 +5344,9 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, "Unknown field with tag %" PRIu16 " (0x%" PRIx16 ") encountered", dp->tdir_tag, dp->tdir_tag); - if (!_TIFFMergeFields( - tif, - _TIFFCreateAnonField(tif, dp->tdir_tag, - (TIFFDataType)dp->tdir_type), - 1)) + const TIFFField *fld = _TIFFCreateAnonField( + tif, dp->tdir_tag, (TIFFDataType)dp->tdir_type); + if (fld == NULL || !_TIFFMergeFields(tif, fld, 1)) { TIFFWarningExtR(tif, module, "Registering anonymous field with tag %" PRIu16 @@ -5221,6 +5429,9 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, } /*-- if (!dp->tdir_ignore) */ } } + /* Evaluate final IFD data size. */ + CalcFinalIFDdatasizeReading(tif, dircount); + /* To be able to return from SubIFD or custom-IFD to main-IFD */ tif->tif_setdirectory_force_absolute = TRUE; if (dir) @@ -5234,9 +5445,7 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, */ int TIFFReadEXIFDirectory(TIFF *tif, toff_t diroff) { - const TIFFFieldArray *exifFieldArray; - exifFieldArray = _TIFFGetExifFields(); - return TIFFReadCustomDirectory(tif, diroff, exifFieldArray); + return TIFFReadCustomDirectory(tif, diroff, _TIFFGetExifFields()); } /* @@ -5244,9 +5453,7 @@ int TIFFReadEXIFDirectory(TIFF *tif, toff_t diroff) */ int TIFFReadGPSDirectory(TIFF *tif, toff_t diroff) { - const TIFFFieldArray *gpsFieldArray; - gpsFieldArray = _TIFFGetGpsFields(); - return TIFFReadCustomDirectory(tif, diroff, gpsFieldArray); + return TIFFReadCustomDirectory(tif, diroff, _TIFFGetGpsFields()); } static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, @@ -5262,6 +5469,24 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, if (!_TIFFFillStrilesInternal(tif, 0)) return -1; + const uint64_t allocsize = (uint64_t)td->td_nstrips * sizeof(uint64_t); + uint64_t filesize = 0; + if (allocsize > 100 * 1024 * 1024) + { + /* Before allocating a huge amount of memory for corrupted files, check + * if size of requested memory is not greater than file size. */ + filesize = TIFFGetFileSize(tif); + if (allocsize > filesize) + { + TIFFWarningExtR( + tif, module, + "Requested memory size for StripByteCounts of %" PRIu64 + " is greater than filesize %" PRIu64 ". Memory not allocated", + allocsize, filesize); + return -1; + } + } + if (td->td_stripbytecount_p) _TIFFfreeExt(tif, td->td_stripbytecount_p); td->td_stripbytecount_p = (uint64_t *)_TIFFCheckMalloc( @@ -5272,9 +5497,7 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, if (td->td_compression != COMPRESSION_NONE) { uint64_t space; - uint64_t filesize; uint16_t n; - filesize = TIFFGetFileSize(tif); if (!(tif->tif_flags & TIFF_BIGTIFF)) space = sizeof(TIFFHeaderClassic) + 2 + dircount * 12 + 4; else @@ -5310,6 +5533,8 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, return -1; space += datasize; } + if (filesize == 0) + filesize = TIFFGetFileSize(tif); if (filesize < space) /* we should perhaps return in error ? */ space = filesize; @@ -5917,6 +6142,20 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, "directories not supported"); return 0; } + /* Before allocating a huge amount of memory for corrupted files, check + * if size of requested memory is not greater than file size. */ + uint64_t filesize = TIFFGetFileSize(tif); + uint64_t allocsize = (uint64_t)dircount16 * dirsize; + if (allocsize > filesize) + { + TIFFWarningExtR( + tif, module, + "Requested memory size for TIFF directory of %" PRIu64 + " is greater than filesize %" PRIu64 + ". Memory not allocated, TIFF directory not read", + allocsize, filesize); + return 0; + } origdir = _TIFFCheckMalloc(tif, dircount16, dirsize, "to read TIFF directory"); if (origdir == NULL) @@ -5964,6 +6203,8 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, } } } + /* No check against filesize needed here because "dir" should have same size + * than "origdir" checked above. */ dir = (TIFFDirEntry *)_TIFFCheckMalloc( tif, dircount16, sizeof(TIFFDirEntry), "to read TIFF directory"); if (dir == 0) @@ -6076,6 +6317,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } } } + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != NULL) + _TIFFfreeExt(tif, data); + return (0); + } if (mb + 1 < (uint32_t)dp->tdir_count) TIFFWarningExtR( tif, module, @@ -6085,15 +6332,15 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) fip->field_name); else if (mb + 1 > (uint32_t)dp->tdir_count) { - uint8_t *o; - TIFFWarningExtR( - tif, module, - "ASCII value for tag \"%s\" does not end in null byte", - fip->field_name); + TIFFWarningExtR(tif, module, + "ASCII value for tag \"%s\" does not end " + "in null byte. Forcing it to be null", + fip->field_name); /* TIFFReadDirEntryArrayWithLimit() ensures this can't be * larger than MAX_SIZE_TAG_DATA */ assert((uint32_t)dp->tdir_count + 1 == dp->tdir_count + 1); - o = _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = + _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -6203,6 +6450,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLong8(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + return 0; if (!TIFFSetField(tif, dp->tdir_tag, data)) return (0); } @@ -6216,6 +6465,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlong8(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + return 0; if (!TIFFSetField(tif, dp->tdir_tag, data)) return (0); } @@ -6229,6 +6480,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryFloat(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + return 0; if (!TIFFSetField(tif, dp->tdir_tag, data)) return (0); } @@ -6242,6 +6495,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryDouble(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + return 0; if (!TIFFSetField(tif, dp->tdir_tag, data)) return (0); } @@ -6255,6 +6510,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryIfd8(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + return 0; if (!TIFFSetField(tif, dp->tdir_tag, data)) return (0); } @@ -6304,6 +6561,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryByteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6333,6 +6596,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySbyteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6362,6 +6631,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryShortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6391,6 +6666,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySshortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6420,6 +6701,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6449,6 +6736,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6478,6 +6771,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6507,6 +6806,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6536,6 +6841,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryFloatArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6567,6 +6878,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryDoubleArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, data); if (data != 0) @@ -6589,16 +6906,39 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryByteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; if (data != 0 && dp->tdir_count > 0 && data[dp->tdir_count - 1] != '\0') { - TIFFWarningExtR( - tif, module, - "ASCII value for tag \"%s\" does not end in null " - "byte. Forcing it to be null", - fip->field_name); - data[dp->tdir_count - 1] = '\0'; + TIFFWarningExtR(tif, module, + "ASCII value for ASCII array tag " + "\"%s\" does not end in null " + "byte. Forcing it to be null", + fip->field_name); + /* Enlarge buffer and add terminating null. */ + uint8_t *o = + _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + if (o == NULL) + { + if (data != NULL) + _TIFFfreeExt(tif, data); + return (0); + } + if (dp->tdir_count > 0) + { + _TIFFmemcpy(o, data, (uint32_t)dp->tdir_count); + } + o[(uint32_t)dp->tdir_count] = 0; + dp->tdir_count++; /* Increment for added null. */ + if (data != 0) + _TIFFfreeExt(tif, data); + data = o; } m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6622,6 +6962,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryByteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6645,6 +6991,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySbyteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6668,6 +7020,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryShortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6691,6 +7049,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySshortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6714,6 +7078,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6737,6 +7107,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6760,6 +7136,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6783,6 +7165,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6806,6 +7194,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryFloatArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6829,6 +7223,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryDoubleArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6852,6 +7252,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryIfd8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint16_t)(dp->tdir_count), data); @@ -6871,15 +7277,39 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryByteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; if (data != 0 && dp->tdir_count > 0 && data[dp->tdir_count - 1] != '\0') { - TIFFWarningExtR(tif, module, - "ASCII value for tag \"%s\" does not end " - "in null byte. Forcing it to be null", - fip->field_name); - data[dp->tdir_count - 1] = '\0'; + TIFFWarningExtR( + tif, module, + "ASCII value for ASCII array tag \"%s\" does not end " + "in null byte. Forcing it to be null", + fip->field_name); + /* Enlarge buffer and add terminating null. */ + uint8_t *o = + _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + if (o == NULL) + { + if (data != NULL) + _TIFFfreeExt(tif, data); + return (0); + } + if (dp->tdir_count > 0) + { + _TIFFmemcpy(o, data, (uint32_t)dp->tdir_count); + } + o[(uint32_t)dp->tdir_count] = 0; + dp->tdir_count++; /* Increment for added null. */ + if (data != 0) + _TIFFfreeExt(tif, data); + data = o; } m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -6923,6 +7353,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, count, data); if (data != 0) @@ -6940,6 +7376,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySbyteArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -6958,6 +7400,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryShortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -6976,6 +7424,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySshortArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -6994,6 +7448,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7012,6 +7472,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlongArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7030,6 +7496,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryLong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7048,6 +7520,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntrySlong8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7066,6 +7544,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryFloatArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7084,6 +7568,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryDoubleArray(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7102,6 +7592,12 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) err = TIFFReadDirEntryIfd8Array(tif, dp, &data); if (err == TIFFReadDirEntryErrOk) { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } int m; m = TIFFSetField(tif, dp->tdir_tag, (uint32_t)(dp->tdir_count), data); @@ -7160,6 +7656,25 @@ static int TIFFFetchStripThing(TIFF *tif, TIFFDirEntry *dir, uint32_t nstrips, return (0); } + const uint64_t allocsize = (uint64_t)nstrips * sizeof(uint64_t); + if (allocsize > 100 * 1024 * 1024) + { + /* Before allocating a huge amount of memory for corrupted files, + * check if size of requested memory is not greater than file size. + */ + const uint64_t filesize = TIFFGetFileSize(tif); + if (allocsize > filesize) + { + TIFFWarningExtR( + tif, module, + "Requested memory size for StripArray of %" PRIu64 + " is greater than filesize %" PRIu64 + ". Memory not allocated", + allocsize, filesize); + _TIFFfreeExt(tif, data); + return (0); + } + } resizeddata = (uint64_t *)_TIFFCheckMalloc( tif, nstrips, sizeof(uint64_t), "for strip array"); if (resizeddata == 0) @@ -7259,6 +7774,26 @@ static void allocChoppedUpStripArrays(TIFF *tif, uint32_t nstrips, } bytecount = last_offset + last_bytecount - offset; + /* Before allocating a huge amount of memory for corrupted files, check if + * size of StripByteCount and StripOffset tags is not greater than + * file size. + */ + const uint64_t allocsize = (uint64_t)nstrips * sizeof(uint64_t) * 2; + if (allocsize > 100 * 1024 * 1024) + { + const uint64_t filesize = TIFFGetFileSize(tif); + if (allocsize > filesize) + { + TIFFWarningExtR(tif, "allocChoppedUpStripArrays", + "Requested memory size for StripByteCount and " + "StripOffsets %" PRIu64 + " is greater than filesize %" PRIu64 + ". Memory not allocated", + allocsize, filesize); + return; + } + } + newcounts = (uint64_t *)_TIFFCheckMalloc(tif, nstrips, sizeof(uint64_t), "for chopped \"StripByteCounts\" array"); @@ -7359,7 +7894,7 @@ static void ChopUpSingleUncompressedStrip(TIFF *tif) /* * never increase the number of rows per strip */ - if (rowsperstrip >= td->td_rowsperstrip) + if (rowsperstrip >= td->td_rowsperstrip || rowsperstrip == 0) return; nstrips = TIFFhowmany_32(td->td_imagelength, rowsperstrip); if (nstrips == 0) @@ -7455,6 +7990,8 @@ static void TryChopUpUncompressedBigTiff(TIFF *tif) stripbytes = rowblocksperstrip * rowblockbytes; assert(stripbytes <= 0x7FFFFFFFUL); + if (rowsperstrip == 0) + return; nstrips = TIFFhowmany_32(td->td_imagelength, rowsperstrip); if (nstrips == 0) return; diff --git a/contrib/libs/libtiff/tif_dirwrite.c b/contrib/libs/libtiff/tif_dirwrite.c index d8844bbd8a3..facdeaf7b16 100644 --- a/contrib/libs/libtiff/tif_dirwrite.c +++ b/contrib/libs/libtiff/tif_dirwrite.c @@ -303,12 +303,14 @@ int TIFFWriteCustomDirectory(TIFF *tif, uint64_t *pdiroff) } /* - * Similar to TIFFWriteDirectory(), but if the directory has already + * Similar to TIFFWriteDirectorySec(), but if the directory has already * been written once, it is relocated to the end of the file, in case it * has changed in size. Note that this will result in the loss of the * previously used directory space. */ -int TIFFRewriteDirectory(TIFF *tif) + +static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, + uint64_t *pdiroff) { static const char module[] = "TIFFRewriteDirectory"; @@ -464,10 +466,20 @@ int TIFFRewriteDirectory(TIFF *tif) } /* - * Now use TIFFWriteDirectory() normally. + * Now use TIFFWriteDirectorySec() normally. */ + return TIFFWriteDirectorySec(tif, isimage, imagedone, pdiroff); +} /*-- TIFFRewriteDirectorySec() --*/ - return TIFFWriteDirectory(tif); +/* + * Similar to TIFFWriteDirectory(), but if the directory has already + * been written once, it is relocated to the end of the file, in case it + * has changed in size. Note that this will result in the loss of the + * previously used directory space. + */ +int TIFFRewriteDirectory(TIFF *tif) +{ + return TIFFRewriteDirectorySec(tif, TRUE, TRUE, NULL); } static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, @@ -542,7 +554,12 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, dirsize = 0; while (1) { + /* The first loop only determines "ndir" and uses TIFFLinkDirectory() to + * set the offset at which the IFD is to be written to the file. + * The second loop writes IFD entries to the file. */ ndir = 0; + if (dir == NULL) + tif->tif_dir.td_dirdatasize_write = 0; if (isimage) { if (TIFFFieldSet(tif, FIELD_IMAGEDIMENSIONS)) @@ -852,7 +869,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, if ((o->field_bit >= FIELD_CODEC) && (TIFFFieldSet(tif, o->field_bit))) { - switch (o->get_field_type) + switch (o->set_field_type) { case TIFF_SETGET_ASCII: { @@ -1085,8 +1102,18 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, break; } } + /* "break" if IFD has been written above in second pass.*/ if (dir != NULL) break; + + /* Evaluate IFD data size: Finally, add the size of the IFD tag entries + * themselves. */ + if (!(tif->tif_flags & TIFF_BIGTIFF)) + tif->tif_dir.td_dirdatasize_write += 2 + ndir * 12 + 4; + else + tif->tif_dir.td_dirdatasize_write += 8 + ndir * 20 + 8; + + /* Setup a new directory within first pass. */ dir = _TIFFmallocExt(tif, ndir * sizeof(TIFFDirEntry)); if (dir == NULL) { @@ -1095,18 +1122,58 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (isimage) { - if ((tif->tif_diroff == 0) && (!TIFFLinkDirectory(tif))) - goto bad; + /* Check, weather the IFD to be written is new or an already written + * IFD can be overwritten or needs to be re-written to a different + * location in the file because the IFD is extended with additional + * tags or the IFD data size is increased. + * - tif_diroff == 0, if a new directory has to be linked. + * - tif_diroff != 0, IFD has been re-read from file and will be + * overwritten or re-written. + */ + if (tif->tif_diroff == 0) + { + if (!TIFFLinkDirectory(tif)) + goto bad; + } + else if (tif->tif_dir.td_dirdatasize_write > + tif->tif_dir.td_dirdatasize_read) + { + if (dir != NULL) + { + _TIFFfreeExt(tif, dir); + dir = NULL; + } + if (!TIFFRewriteDirectorySec(tif, isimage, imagedone, pdiroff)) + goto bad; + return (1); + } } else - tif->tif_diroff = - (TIFFSeekFile(tif, 0, SEEK_END) + 1) & (~((toff_t)1)); + { + /* For !isimage, which means custom-IFD like EXIFIFD or + * checkpointing an IFD, determine whether to overwrite or append at + * the end of the file. + */ + if (!((tif->tif_dir.td_dirdatasize_read > 0) && + (tif->tif_dir.td_dirdatasize_write <= + tif->tif_dir.td_dirdatasize_read))) + { + /* Append at end of file and increment to an even offset. */ + tif->tif_diroff = + (TIFFSeekFile(tif, 0, SEEK_END) + 1) & (~((toff_t)1)); + } + } + /* Return IFD offset */ if (pdiroff != NULL) *pdiroff = tif->tif_diroff; if (!(tif->tif_flags & TIFF_BIGTIFF)) dirsize = 2 + ndir * 12 + 4; else dirsize = 8 + ndir * 20 + 8; + /* Append IFD data stright after the IFD tag entries. + * Data that does not fit into an IFD tag entry is written to the file + * in the second pass of the while loop. That offset is stored in "dir". + */ tif->tif_dataoff = tif->tif_diroff + dirsize; if (!(tif->tif_flags & TIFF_BIGTIFF)) tif->tif_dataoff = (uint32_t)tif->tif_dataoff; @@ -1118,16 +1185,12 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (tif->tif_dataoff & 1) tif->tif_dataoff++; - if (isimage) - { - if (tif->tif_curdir == TIFF_NON_EXISTENT_DIR_NUMBER) - tif->tif_curdir = 0; - else - tif->tif_curdir++; - } - } + } /* while() */ if (isimage) { + /* For SubIFDs remember offset of SubIFD tag within main IFD. + * However, might be already done in TIFFWriteDirectoryTagSubifd() if + * there are more than one SubIFD. */ if (TIFFFieldSet(tif, FIELD_SUBIFD) && (tif->tif_subifdoff == 0)) { uint32_t na; @@ -1148,6 +1211,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, tif->tif_subifdoff = tif->tif_diroff + 8 + na * 20 + 12; } } + /* Copy/swab IFD entries from "dir" into "dirmem", + * which is then written to file. */ dirmem = _TIFFmallocExt(tif, dirsize); if (dirmem == NULL) { @@ -1227,7 +1292,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, dir = NULL; if (!SeekOK(tif, tif->tif_diroff)) { - TIFFErrorExtR(tif, module, "IO error writing directory"); + TIFFErrorExtR(tif, module, + "IO error writing directory at seek to offset"); goto bad; } if (!WriteOK(tif, dirmem, (tmsize_t)dirsize)) @@ -1236,18 +1302,83 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, goto bad; } _TIFFfreeExt(tif, dirmem); + + /* Increment tif_curdir if IFD wasn't already written to file and no error + * occurred during IFD writing above. */ + if (isimage && !tif->tif_dir.td_iswrittentofile) + { + if (!((tif->tif_flags & TIFF_INSUBIFD) && + !(TIFFFieldSet(tif, FIELD_SUBIFD)))) + { + /*-- Normal main-IFD case --*/ + if (tif->tif_curdircount != TIFF_NON_EXISTENT_DIR_NUMBER) + { + tif->tif_curdir = tif->tif_curdircount; + } + else + { + /*ToDo SU: NEW_IFD_CURDIR_INCREMENTING: Delete this + * unexpected case after some testing time. */ + /* Attention: tif->tif_curdircount is already set within + * TIFFNumberOfDirectories() */ + tif->tif_curdircount = TIFFNumberOfDirectories(tif); + tif->tif_curdir = tif->tif_curdircount; + TIFFErrorExtR( + tif, module, + "tif_curdircount is TIFF_NON_EXISTENT_DIR_NUMBER, " + "not expected !! Line %d", + __LINE__); + goto bad; + } + } + else + { + /*-- SubIFD case -- */ + /* tif_curdir is always set to 0 for all SubIFDs. */ + tif->tif_curdir = 0; + } + } + /* Increment tif_curdircount only if main-IFD of an image was not already + * present on file. */ + /* Check in combination with (... && !(TIFFFieldSet(tif, FIELD_SUBIFD))) + * is necessary here because TIFF_INSUBIFD was already set above for the + * next SubIFD when this main-IFD (with FIELD_SUBIFD) is currently being + * written. */ + if (isimage && !tif->tif_dir.td_iswrittentofile && + !((tif->tif_flags & TIFF_INSUBIFD) && + !(TIFFFieldSet(tif, FIELD_SUBIFD)))) + tif->tif_curdircount++; + + tif->tif_dir.td_iswrittentofile = TRUE; + + /* Reset SubIFD writing stage after last SubIFD has been written. */ + if (imagedone && (tif->tif_flags & TIFF_INSUBIFD) && tif->tif_nsubifd == 0) + tif->tif_flags &= ~TIFF_INSUBIFD; + + /* Add or update this directory to the IFD list. */ + if (!_TIFFCheckDirNumberAndOffset(tif, tif->tif_curdir, tif->tif_diroff)) + { + TIFFErrorExtR(tif, module, + "Starting directory %u at offset 0x%" PRIx64 " (%" PRIu64 + ") might cause an IFD loop", + tif->tif_curdir, tif->tif_diroff, tif->tif_diroff); + } + if (imagedone) { TIFFFreeDirectory(tif); tif->tif_flags &= ~TIFF_DIRTYDIRECT; tif->tif_flags &= ~TIFF_DIRTYSTRIP; (*tif->tif_cleanup)(tif); - /* - * Reset directory-related state for subsequent - * directories. - */ + /* Reset directory-related state for subsequent directories. */ TIFFCreateDirectory(tif); } + else + { + /* IFD is only checkpointed to file (or a custom IFD like EXIF is + * written), thus set IFD data size written to file. */ + tif->tif_dir.td_dirdatasize_read = tif->tif_dir.td_dirdatasize_write; + } return (1); bad: if (dir != NULL) @@ -1401,11 +1532,6 @@ static int TIFFWriteDirectoryTagAscii(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, char *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return ( TIFFWriteDirectoryTagCheckedAscii(tif, ndir, dir, tag, count, value)); } @@ -1414,11 +1540,6 @@ static int TIFFWriteDirectoryTagUndefinedArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, uint8_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedUndefinedArray(tif, ndir, dir, tag, count, value)); } @@ -1427,11 +1548,6 @@ static int TIFFWriteDirectoryTagByteArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, uint8_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedByteArray(tif, ndir, dir, tag, count, value)); } @@ -1440,11 +1556,6 @@ static int TIFFWriteDirectoryTagSbyteArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, int8_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedSbyteArray(tif, ndir, dir, tag, count, value)); } @@ -1453,11 +1564,6 @@ static int TIFFWriteDirectoryTagShort(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint16_t value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedShort(tif, ndir, dir, tag, value)); } @@ -1465,11 +1571,6 @@ static int TIFFWriteDirectoryTagShortArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, uint16_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedShortArray(tif, ndir, dir, tag, count, value)); } @@ -1485,8 +1586,9 @@ static int TIFFWriteDirectoryTagShortPerSample(TIFF *tif, uint32_t *ndir, int o; if (dir == NULL) { - (*ndir)++; - return (1); + /* only evaluate IFD data size and inc. ndir */ + return (TIFFWriteDirectoryTagCheckedShortArray( + tif, ndir, dir, tag, tif->tif_dir.td_samplesperpixel, NULL)); } m = _TIFFmallocExt(tif, tif->tif_dir.td_samplesperpixel * sizeof(uint16_t)); if (m == NULL) @@ -1506,11 +1608,6 @@ static int TIFFWriteDirectoryTagSshortArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, int16_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedSshortArray(tif, ndir, dir, tag, count, value)); } @@ -1519,11 +1616,6 @@ static int TIFFWriteDirectoryTagLong(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedLong(tif, ndir, dir, tag, value)); } @@ -1531,11 +1623,6 @@ static int TIFFWriteDirectoryTagLongArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, uint32_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedLongArray(tif, ndir, dir, tag, count, value)); } @@ -1544,11 +1631,6 @@ static int TIFFWriteDirectoryTagSlongArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, int32_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedSlongArray(tif, ndir, dir, tag, count, value)); } @@ -1572,8 +1654,9 @@ static int TIFFWriteDirectoryTagLong8Array(TIFF *tif, uint32_t *ndir, /* is this just a counting pass? */ if (dir == NULL) { - (*ndir)++; - return (1); + /* only evaluate IFD data size and inc. ndir */ + return (TIFFWriteDirectoryTagCheckedLong8Array(tif, ndir, dir, tag, + count, value)); } /* We always write Long8 for BigTIFF, no checking needed. */ @@ -1632,8 +1715,9 @@ static int TIFFWriteDirectoryTagSlong8Array(TIFF *tif, uint32_t *ndir, /* is this just a counting pass? */ if (dir == NULL) { - (*ndir)++; - return (1); + /* only evaluate IFD data size and inc. ndir */ + return (TIFFWriteDirectoryTagCheckedSlong8Array(tif, ndir, dir, tag, + count, value)); } /* We always write SLong8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) @@ -1686,11 +1770,6 @@ static int TIFFWriteDirectoryTagRational(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, double value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedRational(tif, ndir, dir, tag, value)); } @@ -1698,11 +1777,6 @@ static int TIFFWriteDirectoryTagRationalArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, float *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedRationalArray(tif, ndir, dir, tag, count, value)); } @@ -1711,11 +1785,6 @@ static int TIFFWriteDirectoryTagSrationalArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, float *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedSrationalArray(tif, ndir, dir, tag, count, value)); } @@ -1727,11 +1796,6 @@ static int TIFFWriteDirectoryTagRationalDoubleArray(TIFF *tif, uint32_t *ndir, uint32_t count, double *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedRationalDoubleArray(tif, ndir, dir, tag, count, value)); } @@ -1742,11 +1806,6 @@ static int TIFFWriteDirectoryTagSrationalDoubleArray(TIFF *tif, uint32_t *ndir, uint32_t count, double *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedSrationalDoubleArray( tif, ndir, dir, tag, count, value)); } @@ -1755,11 +1814,6 @@ static int TIFFWriteDirectoryTagFloatArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, float *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedFloatArray(tif, ndir, dir, tag, count, value)); } @@ -1768,11 +1822,6 @@ static int TIFFWriteDirectoryTagDoubleArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, double *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedDoubleArray(tif, ndir, dir, tag, count, value)); } @@ -1781,11 +1830,6 @@ static int TIFFWriteDirectoryTagIfdArray(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t count, uint32_t *value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } return (TIFFWriteDirectoryTagCheckedIfdArray(tif, ndir, dir, tag, count, value)); } @@ -1794,11 +1838,6 @@ static int TIFFWriteDirectoryTagShortLong(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir, uint16_t tag, uint32_t value) { - if (dir == NULL) - { - (*ndir)++; - return (1); - } if (value <= 0xFFFF) return (TIFFWriteDirectoryTagCheckedShort(tif, ndir, dir, tag, (uint16_t)value)); @@ -1824,7 +1863,7 @@ static int _WriteAsType(TIFF *tif, uint64_t strile_size, compression == COMPRESSION_WEBP || compression == COMPRESSION_JXL) { /* For a few select compression types, we assume that in the worst */ - /* case the compressed size will be 10 times the uncompressed size */ + /* case the compressed size will be 10 times the uncompressed size. */ /* This is overly pessismistic ! */ return strile_size >= uncompressed_threshold / 10; } @@ -1856,15 +1895,16 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, int o; int write_aslong4; - /* is this just a counting pass? */ - if (dir == NULL) - { - (*ndir)++; - return (1); - } - if (tif->tif_dir.td_deferstrilearraywriting) { + if (dir == NULL) + { + /* This is just a counting pass to count IFD entries. + * For deferstrilearraywriting no extra bytes will be written + * into IFD space. */ + (*ndir)++; + return 1; + } return TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_NOTYPE, 0, 0, NULL); } @@ -1872,12 +1912,10 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, if (tif->tif_flags & TIFF_BIGTIFF) { int write_aslong8 = 1; - /* In the case of ByteCounts array, we may be able to write them on */ - /* LONG if the strip/tilesize is not too big. */ - /* Also do that for count > 1 in the case someone would want to create - */ - /* a single-strip file with a growing height, in which case using */ - /* LONG8 will be safer. */ + /* In the case of ByteCounts array, we may be able to write them on LONG + * if the strip/tilesize is not too big. Also do that for count > 1 in + * the case someone would want to create a single-strip file with a + * growing height, in which case using LONG8 will be safer. */ if (count > 1 && tag == TIFFTAG_STRIPBYTECOUNTS) { write_aslong8 = WriteAsLong8(tif, TIFFStripSize64(tif)); @@ -1989,13 +2027,6 @@ static int TIFFWriteDirectoryTagIfdIfd8Array(TIFF *tif, uint32_t *ndir, uint32_t *q; int o; - /* is this just a counting pass? */ - if (dir == NULL) - { - (*ndir)++; - return (1); - } - /* We always write IFD8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) return TIFFWriteDirectoryTagCheckedIfd8Array(tif, ndir, dir, tag, count, @@ -2032,6 +2063,26 @@ static int TIFFWriteDirectoryTagIfdIfd8Array(TIFF *tif, uint32_t *ndir, return (o); } +/* + * Auxiliary function to determine the IFD data size to be written to the file. + * The IFD data size is finally the size of the IFD tag entries plus the IFD + * data that is written directly after the IFD tag entries. + */ +static void EvaluateIFDdatasizeWrite(TIFF *tif, uint32_t count, + uint32_t typesize, uint32_t *ndir) +{ + uint64_t datalength = (uint64_t)count * typesize; + if (datalength > ((tif->tif_flags & TIFF_BIGTIFF) ? 0x8U : 0x4U)) + { + /* LibTIFF increments write address to an even offset, thus datalength + * written is also incremented. */ + if (datalength & 1) + datalength++; + tif->tif_dir.td_dirdatasize_write += datalength; + } + (*ndir)++; +} + static int TIFFWriteDirectoryTagColormap(TIFF *tif, uint32_t *ndir, TIFFDirEntry *dir) { @@ -2039,12 +2090,13 @@ static int TIFFWriteDirectoryTagColormap(TIFF *tif, uint32_t *ndir, uint32_t m; uint16_t *n; int o; - if (dir == NULL) + m = (1 << tif->tif_dir.td_bitspersample); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - (*ndir)++; - return (1); + EvaluateIFDdatasizeWrite(tif, 3 * m, sizeof(uint16_t), ndir); + return 1; } - m = (1 << tif->tif_dir.td_bitspersample); + n = _TIFFmallocExt(tif, 3 * m * sizeof(uint16_t)); if (n == NULL) { @@ -2068,11 +2120,6 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, uint16_t n; uint16_t *o; int p; - if (dir == NULL) - { - (*ndir)++; - return (1); - } /* TIFFTAG_TRANSFERFUNCTION expects (1 or 3) pointer to arrays with * (1 << BitsPerSample) * uint16_t values. */ @@ -2086,9 +2133,9 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, { if (tif->tif_dir.td_transferfunction[i] == NULL) { - TIFFWarningExtR( - tif, module, - "Too few TransferFunctions provided. Tag not written to file"); + TIFFWarningExtR(tif, module, + "Too few TransferFunctions provided. Tag " + "not written to file"); return (1); /* Not an error; only tag is not written. */ } } @@ -2108,6 +2155,12 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, m * sizeof(uint16_t))) n = 1; } + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, n * m, 2, ndir); + return 1; + } + o = _TIFFmallocExt(tif, n * m * sizeof(uint16_t)); if (o == NULL) { @@ -2136,11 +2189,6 @@ static int TIFFWriteDirectoryTagSubifd(TIFF *tif, uint32_t *ndir, int n; if (tif->tif_dir.td_nsubifd == 0) return (1); - if (dir == NULL) - { - (*ndir)++; - return (1); - } m = tif->tif_dataoff; if (!(tif->tif_flags & TIFF_BIGTIFF)) { @@ -2178,6 +2226,12 @@ static int TIFFWriteDirectoryTagSubifd(TIFF *tif, uint32_t *ndir, n = TIFFWriteDirectoryTagCheckedIfd8Array( tif, ndir, dir, TIFFTAG_SUBIFD, tif->tif_dir.td_nsubifd, tif->tif_dir.td_subifd); + + if (dir == NULL) + /* Just have evaluated IFD data size and incremented ndir + * above in sub-functions. */ + return (n); + if (!n) return (0); /* @@ -2202,6 +2256,11 @@ static int TIFFWriteDirectoryTagCheckedAscii(TIFF *tif, uint32_t *ndir, uint32_t count, char *value) { assert(sizeof(char) == 1); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 1, ndir); + return 1; + } return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_ASCII, count, count, value)); } @@ -2213,6 +2272,11 @@ static int TIFFWriteDirectoryTagCheckedUndefinedArray(TIFF *tif, uint32_t *ndir, uint8_t *value) { assert(sizeof(uint8_t) == 1); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 1, ndir); + return 1; + } return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_UNDEFINED, count, count, value)); } @@ -2223,6 +2287,11 @@ static int TIFFWriteDirectoryTagCheckedByteArray(TIFF *tif, uint32_t *ndir, uint8_t *value) { assert(sizeof(uint8_t) == 1); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 1, ndir); + return 1; + } return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_BYTE, count, count, value)); } @@ -2233,6 +2302,11 @@ static int TIFFWriteDirectoryTagCheckedSbyteArray(TIFF *tif, uint32_t *ndir, int8_t *value) { assert(sizeof(int8_t) == 1); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 1, ndir); + return 1; + } return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_SBYTE, count, count, value)); } @@ -2243,6 +2317,12 @@ static int TIFFWriteDirectoryTagCheckedShort(TIFF *tif, uint32_t *ndir, { uint16_t m; assert(sizeof(uint16_t) == 2); + if (dir == NULL) + { + /* No additional data to IFD data size just increment ndir. */ + (*ndir)++; + return 1; + } m = value; if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&m); @@ -2257,6 +2337,11 @@ static int TIFFWriteDirectoryTagCheckedShortArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x80000000); assert(sizeof(uint16_t) == 2); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 2, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfShort(value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_SHORT, count, @@ -2270,6 +2355,11 @@ static int TIFFWriteDirectoryTagCheckedSshortArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x80000000); assert(sizeof(int16_t) == 2); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 2, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfShort((uint16_t *)value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_SSHORT, count, @@ -2282,6 +2372,12 @@ static int TIFFWriteDirectoryTagCheckedLong(TIFF *tif, uint32_t *ndir, { uint32_t m; assert(sizeof(uint32_t) == 4); + if (dir == NULL) + { + /* No additional data to IFD data size just increment ndir. */ + (*ndir)++; + return 1; + } m = value; if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong(&m); @@ -2296,6 +2392,11 @@ static int TIFFWriteDirectoryTagCheckedLongArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x40000000); assert(sizeof(uint32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 4, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_LONG, count, @@ -2309,6 +2410,11 @@ static int TIFFWriteDirectoryTagCheckedSlongArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x40000000); assert(sizeof(int32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 4, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_SLONG, count, @@ -2328,6 +2434,11 @@ static int TIFFWriteDirectoryTagCheckedLong8Array(TIFF *tif, uint32_t *ndir, "LONG8 not allowed for ClassicTIFF"); return (0); } + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 8, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_LONG8, count, @@ -2347,6 +2458,11 @@ static int TIFFWriteDirectoryTagCheckedSlong8Array(TIFF *tif, uint32_t *ndir, "SLONG8 not allowed for ClassicTIFF"); return (0); } + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 8, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8((uint64_t *)value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_SLONG8, count, @@ -2370,16 +2486,17 @@ static int TIFFWriteDirectoryTagCheckedRational(TIFF *tif, uint32_t *ndir, TIFFErrorExtR(tif, module, "Not-a-number value is illegal"); return 0; } - /*--Rational2Double: New function also used for non-custom rational tags. - * However, could be omitted here, because - * TIFFWriteDirectoryTagCheckedRational() is not used by code for custom - * tags, only by code for named-tiff-tags like FIELD_RESOLUTION and - * FIELD_POSITION */ - else + + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - DoubleToRational(value, &m[0], &m[1]); + tif->tif_dir.td_dirdatasize_write += + (tif->tif_flags & TIFF_BIGTIFF) ? 0 : 0x8U; + (*ndir)++; + return 1; } + DoubleToRational(value, &m[0], &m[1]); + if (tif->tif_flags & TIFF_SWAB) { TIFFSwabLong(&m[0]); @@ -2402,6 +2519,11 @@ static int TIFFWriteDirectoryTagCheckedRationalArray(TIFF *tif, uint32_t *ndir, uint32_t nc; int o; assert(sizeof(uint32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); + return 1; + } m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); if (m == NULL) { @@ -2433,6 +2555,11 @@ static int TIFFWriteDirectoryTagCheckedSrationalArray(TIFF *tif, uint32_t *ndir, uint32_t nc; int o; assert(sizeof(int32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); + return 1; + } m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); if (m == NULL) { @@ -2465,6 +2592,11 @@ TIFFWriteDirectoryTagCheckedRationalDoubleArray(TIFF *tif, uint32_t *ndir, uint32_t nc; int o; assert(sizeof(uint32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); + return 1; + } m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); if (m == NULL) { @@ -2495,6 +2627,11 @@ static int TIFFWriteDirectoryTagCheckedSrationalDoubleArray( uint32_t nc; int o; assert(sizeof(int32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); + return 1; + } m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); if (m == NULL) { @@ -2816,6 +2953,11 @@ static int TIFFWriteDirectoryTagCheckedFloatArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x40000000); assert(sizeof(float) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 4, ndir); + return 1; + } TIFFCvtNativeToIEEEFloat(tif, count, &value); if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfFloat(value, count); @@ -2830,6 +2972,11 @@ static int TIFFWriteDirectoryTagCheckedDoubleArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x20000000); assert(sizeof(double) == 8); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 8, ndir); + return 1; + } TIFFCvtNativeToIEEEDouble(tif, count, &value); if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfDouble(value, count); @@ -2843,6 +2990,11 @@ static int TIFFWriteDirectoryTagCheckedIfdArray(TIFF *tif, uint32_t *ndir, { assert(count < 0x40000000); assert(sizeof(uint32_t) == 4); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 4, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_IFD, count, @@ -2857,6 +3009,11 @@ static int TIFFWriteDirectoryTagCheckedIfd8Array(TIFF *tif, uint32_t *ndir, assert(count < 0x20000000); assert(sizeof(uint64_t) == 8); assert(tif->tif_flags & TIFF_BIGTIFF); + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, count, 8, ndir); + return 1; + } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(value, count); return (TIFFWriteDirectoryTagData(tif, ndir, dir, tag, TIFF_IFD8, count, @@ -2973,15 +3130,15 @@ static int TIFFLinkDirectory(TIFF *tif) "Error writing SubIFD directory link"); return (0); } + /* * Advance to the next SubIFD or, if this is - * the last one configured, revert back to the - * normal directory linkage. + * the last one configured, reverting back to the + * normal directory linkage is done in TIFFWriteDirectorySec() + * by tif->tif_flags &= ~TIFF_INSUBIFD;. */ if (--tif->tif_nsubifd) tif->tif_subifdoff += 4; - else - tif->tif_flags &= ~TIFF_INSUBIFD; return (1); } else @@ -2997,19 +3154,23 @@ static int TIFFLinkDirectory(TIFF *tif) "Error writing SubIFD directory link"); return (0); } + /* * Advance to the next SubIFD or, if this is - * the last one configured, revert back to the - * normal directory linkage. + * the last one configured, reverting back to the + * normal directory linkage is done in TIFFWriteDirectorySec() + * by tif->tif_flags &= ~TIFF_INSUBIFD;. */ if (--tif->tif_nsubifd) tif->tif_subifdoff += 8; - else - tif->tif_flags &= ~TIFF_INSUBIFD; return (1); } } + /* + * Handle main-IFDs + */ + tdir_t ndir = 1; /* count current number of main-IFDs */ if (!(tif->tif_flags & TIFF_BIGTIFF)) { uint32_t m; @@ -3030,18 +3191,26 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFErrorExtR(tif, tif->tif_name, "Error writing TIFF header"); return (0); } + if (!tif->tif_dir.td_iswrittentofile) + tif->tif_curdircount = 0; return (1); } /* * Not the first directory, search to the last and append. */ - if (tif->tif_lastdiroff != 0) + tdir_t dirn = 0; + if (tif->tif_lastdiroff != 0 && + _TIFFGetDirNumberFromOffset(tif, tif->tif_lastdiroff, &dirn)) { + /* Start searching from the lastely written IFD. Thus get its IFD + * number. */ nextdir = (uint32_t)tif->tif_lastdiroff; + ndir = dirn + 1; } else { nextdir = tif->tif_header.classic.tiff_diroff; + ndir = 1; /* start searching from the first IFD */ } while (1) @@ -3076,10 +3245,12 @@ static int TIFFLinkDirectory(TIFF *tif) break; } nextdir = nextnextdir; + ndir++; } } else { + /*- BigTIFF -*/ uint64_t m; uint64_t nextdir; m = tif->tif_diroff; @@ -3098,18 +3269,26 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFErrorExtR(tif, tif->tif_name, "Error writing TIFF header"); return (0); } + if (!tif->tif_dir.td_iswrittentofile) + tif->tif_curdircount = 0; return (1); } /* * Not the first directory, search to the last and append. */ - if (tif->tif_lastdiroff != 0) + tdir_t dirn = 0; + if (tif->tif_lastdiroff != 0 && + _TIFFGetDirNumberFromOffset(tif, tif->tif_lastdiroff, &dirn)) { + /* Start searching from the lastely written IFD. Thus get its IFD + * number. */ nextdir = tif->tif_lastdiroff; + ndir = dirn + 1; } else { nextdir = tif->tif_header.big.tiff_diroff; + ndir = 1; /* start searching from the first IFD */ } while (1) { @@ -3126,9 +3305,9 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFSwabLong8(&dircount64); if (dircount64 > 0xFFFF) { - TIFFErrorExtR( - tif, module, - "Sanity check on tag count failed, likely corrupt TIFF"); + TIFFErrorExtR(tif, module, + "Sanity check on tag count failed, " + "likely corrupt TIFF"); return (0); } dircount = (uint16_t)dircount64; @@ -3152,8 +3331,20 @@ static int TIFFLinkDirectory(TIFF *tif) break; } nextdir = nextnextdir; + ndir++; } } + /* Offset of next IFD is written to file. + * Update number of main-IFDs in file. + * However, tif_curdircount shall count only newly written main-IFDs with + * entries and not only number of linked offsets! Thus, tif_curdircount is + * incremented at the end of TIFFWriteDirectorySec(). + * TIFF_NON_EXISTENT_DIR_NUMBER means 'dont know number of IFDs' + * 0 means 'empty file opened for writing, but no IFD written yet' */ + if (!tif->tif_dir.td_iswrittentofile && !(tif->tif_flags & TIFF_INSUBIFD)) + { + tif->tif_curdircount = ndir; + } return (1); } @@ -3197,9 +3388,9 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, /* -------------------------------------------------------------------- */ if (isMapped(tif)) { - TIFFErrorExtR( - tif, module, - "Memory mapped files not currently supported for this operation."); + TIFFErrorExtR(tif, module, + "Memory mapped files not currently supported for " + "this operation."); return 0; } diff --git a/contrib/libs/libtiff/tif_fax3.c b/contrib/libs/libtiff/tif_fax3.c index a3c645cb68f..01a784730bd 100644 --- a/contrib/libs/libtiff/tif_fax3.c +++ b/contrib/libs/libtiff/tif_fax3.c @@ -41,6 +41,14 @@ #include "t4.h" #include <stdio.h> +#ifndef EOF_REACHED_COUNT_THRESHOLD +/* Arbitrary threshold to avoid corrupted single-strip files with extremely + * large imageheight to cause apparently endless looping, such as in + * https://gitlab.com/libtiff/libtiff/-/issues/583 + */ +#define EOF_REACHED_COUNT_THRESHOLD 8192 +#endif + /* * Compression+decompression state blocks are * derived from this ``base state'' block. @@ -77,6 +85,8 @@ typedef struct uint32_t data; /* current i/o byte/word */ int bit; /* current i/o bit in byte */ int EOLcnt; /* count of EOL codes recognized */ + int eofReachedCount; /* number of times decode has been called with + EOF already reached */ TIFFFaxFillFunc fill; /* fill routine */ uint32_t *runs; /* b&w runs for current/previous row */ uint32_t nruns; /* size of the refruns / curruns arrays */ @@ -120,6 +130,7 @@ typedef struct int EOLcnt; /* # EOL codes recognized */ \ const unsigned char *bitmap = sp->bitmap; /* input data bit reverser */ \ const TIFFFaxTabEnt *TabEnt + #define DECLARE_STATE_2D(tif, sp, mod) \ DECLARE_STATE(tif, sp, mod); \ int b1; /* next change on prev line */ \ @@ -162,6 +173,7 @@ static int Fax3PreDecode(TIFF *tif, uint16_t s) sp->bit = 0; /* force initial read */ sp->data = 0; sp->EOLcnt = 0; /* force initial scan for EOL */ + sp->eofReachedCount = 0; /* * Decoder assumes lsb-to-msb bit order. Note that we select * this here rather than in Fax3SetupState so that viewers can @@ -232,7 +244,12 @@ static void Fax3PrematureEOF(const char *module, TIFF *tif, uint32_t line, line, isTiled(tif) ? "tile" : "strip", (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); } -#define prematureEOF(a0) Fax3PrematureEOF(module, tif, sp->line, a0) +#define prematureEOF(a0) \ + do \ + { \ + Fax3PrematureEOF(module, tif, sp->line, a0); \ + ++sp->eofReachedCount; \ + } while (0) #define Nop @@ -252,6 +269,14 @@ static int Fax3Decode1D(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return (-1); } + if (sp->eofReachedCount >= EOF_REACHED_COUNT_THRESHOLD) + { + TIFFErrorExtR( + tif, module, + "End of file has already been reached %d times within that strip", + sp->eofReachedCount); + return (-1); + } CACHE_STATE(tif, sp); thisrun = sp->curruns; while (occ > 0) @@ -302,6 +327,14 @@ static int Fax3Decode2D(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return (-1); } + if (sp->eofReachedCount >= EOF_REACHED_COUNT_THRESHOLD) + { + TIFFErrorExtR( + tif, module, + "End of file has already been reached %d times within that strip", + sp->eofReachedCount); + return (-1); + } CACHE_STATE(tif, sp); while (occ > 0) { @@ -536,7 +569,11 @@ static int Fax3SetupState(TIFF *tif) TIFFroundup and TIFFSafeMultiply return zero on integer overflow */ - dsp->runs = (uint32_t *)NULL; + if (dsp->runs != NULL) + { + _TIFFfreeExt(tif, dsp->runs); + dsp->runs = (uint32_t *)NULL; + } dsp->nruns = TIFFroundup_32(rowpixels + 1, 32); if (needsRefLine) { @@ -578,6 +615,10 @@ static int Fax3SetupState(TIFF *tif) * is referenced. The reference line must * be initialized to be ``white'' (done elsewhere). */ + if (esp->refline != NULL) + { + _TIFFfreeExt(tif, esp->refline); + } esp->refline = (unsigned char *)_TIFFmallocExt(tif, rowbytes); if (esp->refline == NULL) { @@ -1514,7 +1555,16 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return (-1); } + if (sp->eofReachedCount >= EOF_REACHED_COUNT_THRESHOLD) + { + TIFFErrorExtR( + tif, module, + "End of file has already been reached %d times within that strip", + sp->eofReachedCount); + return (-1); + } CACHE_STATE(tif, sp); + int start = sp->line; while (occ > 0) { a0 = 0; @@ -1563,7 +1613,9 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) } (*sp->fill)(buf, thisrun, pa, lastx); UNCACHE_STATE(tif, sp); - return (sp->line ? 1 : -1); /* don't error on badly-terminated strips */ + return (sp->line != start + ? 1 + : -1); /* don't error on badly-terminated strips */ } UNCACHE_STATE(tif, sp); return (1); diff --git a/contrib/libs/libtiff/tif_getimage.c b/contrib/libs/libtiff/tif_getimage.c index 41f7dfd77e0..6c7b5031a19 100644 --- a/contrib/libs/libtiff/tif_getimage.c +++ b/contrib/libs/libtiff/tif_getimage.c @@ -760,6 +760,12 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, toskew = -(int32_t)(tw - w); } + if (tw == 0 || th == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), "tile width or height is zero"); + return (0); + } + /* * Leftmost tile is clipped on left side if col_offset > 0. */ @@ -916,6 +922,12 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, break; } + if (tw == 0 || th == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), "tile width or height is zero"); + return (0); + } + /* * Leftmost tile is clipped on left side if col_offset > 0. */ @@ -1092,6 +1104,11 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } TIFFGetFieldDefaulted(tif, TIFFTAG_ROWSPERSTRIP, &rowsperstrip); + if (rowsperstrip == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), "rowsperstrip is zero"); + return (0); + } scanline = TIFFScanlineSize(tif); fromskew = (w < imagewidth ? imagewidth - w : 0); @@ -1216,6 +1233,12 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } TIFFGetFieldDefaulted(tif, TIFFTAG_ROWSPERSTRIP, &rowsperstrip); + if (rowsperstrip == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), "rowsperstrip is zero"); + return (0); + } + scanline = TIFFScanlineSize(tif); fromskew = (w < imagewidth ? imagewidth - w : 0); for (row = 0; row < h; row += nrow) @@ -3213,6 +3236,13 @@ int TIFFReadRGBAStripExt(TIFF *tif, uint32_t row, uint32_t *raster, } TIFFGetFieldDefaulted(tif, TIFFTAG_ROWSPERSTRIP, &rowsperstrip); + + if (rowsperstrip == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), "rowsperstrip is zero"); + return (0); + } + if ((row % rowsperstrip) != 0) { TIFFErrorExtR( @@ -3224,6 +3254,13 @@ int TIFFReadRGBAStripExt(TIFF *tif, uint32_t row, uint32_t *raster, if (TIFFRGBAImageOK(tif, emsg) && TIFFRGBAImageBegin(&img, tif, stop_on_error, emsg)) { + if (row >= img.height) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Invalid row passed to TIFFReadRGBAStrip()."); + TIFFRGBAImageEnd(&img); + return (0); + } img.row_offset = row; img.col_offset = 0; @@ -3282,6 +3319,13 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, TIFFGetFieldDefaulted(tif, TIFFTAG_TILEWIDTH, &tile_xsize); TIFFGetFieldDefaulted(tif, TIFFTAG_TILELENGTH, &tile_ysize); + if (tile_xsize == 0 || tile_ysize == 0) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "tile_xsize or tile_ysize is zero"); + return (0); + } + if ((col % tile_xsize) != 0 || (row % tile_ysize) != 0) { TIFFErrorExtR(tif, TIFFFileName(tif), @@ -3301,6 +3345,14 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, return (0); } + if (col >= img.width || row >= img.height) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Invalid row/col passed to TIFFReadRGBATile()."); + TIFFRGBAImageEnd(&img); + return (0); + } + /* * The TIFFRGBAImageGet() function doesn't allow us to get off the * edge of the image, even to fill an otherwise valid tile. So we diff --git a/contrib/libs/libtiff/tif_hash_set.c b/contrib/libs/libtiff/tif_hash_set.c index 9792c63f47d..81dea3fcf26 100644 --- a/contrib/libs/libtiff/tif_hash_set.c +++ b/contrib/libs/libtiff/tif_hash_set.c @@ -146,7 +146,7 @@ TIFFHashSet *TIFFHashSetNew(TIFFHashSetHashFunc fnHashFunc, set->fnEqualFunc = fnEqualFunc ? fnEqualFunc : TIFFHashSetEqualPointer; set->fnFreeEltFunc = fnFreeEltFunc; set->nSize = 0; - set->tabList = (TIFFList **)(calloc(sizeof(TIFFList *), 53)); + set->tabList = (TIFFList **)(calloc(53, sizeof(TIFFList *))); if (set->tabList == NULL) { free(set); @@ -367,7 +367,7 @@ static bool TIFFHashSetRehash(TIFFHashSet *set) { int nNewAllocatedSize = anPrimes[set->nIndiceAllocatedSize]; TIFFList **newTabList = - (TIFFList **)(calloc(sizeof(TIFFList *), nNewAllocatedSize)); + (TIFFList **)(calloc(nNewAllocatedSize, sizeof(TIFFList *))); if (newTabList == NULL) return false; #ifdef HASH_DEBUG diff --git a/contrib/libs/libtiff/tif_jbig.c b/contrib/libs/libtiff/tif_jbig.c index a0f879f95d7..d0fae1f124a 100644 --- a/contrib/libs/libtiff/tif_jbig.c +++ b/contrib/libs/libtiff/tif_jbig.c @@ -92,6 +92,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) jbg_strerror(decodeStatus) #endif ); + memset(buffer, 0, (size_t)size); jbg_dec_free(&decoder); return 0; } @@ -99,6 +100,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) decodedSize = jbg_dec_getsize(&decoder); if ((tmsize_t)decodedSize < size) { + memset(buffer + decodedSize, 0, (size_t)(size - decodedSize)); TIFFWarningExtR(tif, "JBIG", "Only decoded %lu bytes, whereas %" TIFF_SSIZE_FORMAT " requested", diff --git a/contrib/libs/libtiff/tif_jpeg.c b/contrib/libs/libtiff/tif_jpeg.c index 250144f2112..10aed546358 100644 --- a/contrib/libs/libtiff/tif_jpeg.c +++ b/contrib/libs/libtiff/tif_jpeg.c @@ -73,43 +73,6 @@ int TIFFReInitJPEG_12(TIFF *tif, const JPEGOtherSettings *otherSettings, int scheme, int is_encode); int TIFFJPEGIsFullStripRequired_12(TIFF *tif); -/* We undefine FAR to avoid conflict with JPEG definition */ - -#ifdef FAR -#undef FAR -#endif - -/* - Libjpeg's jmorecfg.h defines INT16 and INT32, but only if XMD_H is - not defined. Unfortunately, the MinGW and Borland compilers include - a typedef for INT32, which causes a conflict. MSVC does not include - a conflicting typedef given the headers which are included. -*/ -#if defined(__BORLANDC__) || defined(__MINGW32__) -#define XMD_H 1 -#endif - -/* - The windows RPCNDR.H file defines boolean, but defines it with the - unsigned char size. You should compile JPEG library using appropriate - definitions in jconfig.h header, but many users compile library in wrong - way. That causes errors of the following type: - - "JPEGLib: JPEG parameter struct mismatch: library thinks size is 432, - caller expects 464" - - For such users we will fix the problem here. See install.doc file from - the JPEG library distribution for details. -*/ - -/* Define "boolean" as unsigned char, not int, per Windows custom. */ -#if defined(__WIN32__) && !defined(__MINGW32__) -#ifndef __RPCNDR_H__ /* don't conflict if rpcndr.h already read */ -typedef unsigned char boolean; -#endif -#define HAVE_BOOLEAN /* prevent jmorecfg.h from redefining it */ -#endif - #include "jerror.h" #include "jpeglib.h" @@ -125,18 +88,18 @@ typedef unsigned char boolean; * 16bit value? */ -/* HAVE_JPEGTURBO_DUAL_MODE_8_12 is defined for libjpeg-turbo >= 2.2 which +/* HAVE_JPEGTURBO_DUAL_MODE_8_12 is defined for libjpeg-turbo >= 3.0 which * adds a dual-mode 8/12 bit API in the same library. */ #if defined(HAVE_JPEGTURBO_DUAL_MODE_8_12) #define JPEG_DUAL_MODE_8_12 /* Start by undefining BITS_IN_JSAMPLE which is always set to 8 in libjpeg-turbo - * >= 2.2 Cf + * >= 3.0 Cf * https://github.com/libjpeg-turbo/libjpeg-turbo/commit/8b9bc4b9635a2a047fb23ebe70c9acd728d3f99b */ #undef BITS_IN_JSAMPLE -/* libjpeg-turbo >= 2.2 adds J12xxxx datatypes for the 12-bit mode. */ +/* libjpeg-turbo >= 3.0 adds J12xxxx datatypes for the 12-bit mode. */ #if defined(FROM_TIF_JPEG_12) #define BITS_IN_JSAMPLE 12 #define TIFF_JSAMPLE J12SAMPLE @@ -182,9 +145,20 @@ typedef unsigned char boolean; #define LONGJMP(jbuf, code) longjmp(jbuf, code) #define JMP_BUF jmp_buf +#ifndef TIFF_jpeg_destination_mgr_defined +#define TIFF_jpeg_destination_mgr_defined typedef struct jpeg_destination_mgr jpeg_destination_mgr; +#endif + +#ifndef TIFF_jpeg_source_mgr_defined +#define TIFF_jpeg_source_mgr_defined typedef struct jpeg_source_mgr jpeg_source_mgr; +#endif + +#ifndef TIFF_jpeg_error_mgr_defined +#define TIFF_jpeg_error_mgr_defined typedef struct jpeg_error_mgr jpeg_error_mgr; +#endif /* * State block for each open TIFF file using @@ -1241,6 +1215,12 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) * For PC 2, scale down the expected strip/tile size * to match a downsampled component */ + if (sp->h_sampling == 0 || sp->v_sampling == 0) + { + TIFFErrorExtR(tif, module, + "JPEG horizontal or vertical sampling is zero"); + return (0); + } segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); } @@ -1471,7 +1451,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) sp->src.bytes_in_buffer = (size_t)tif->tif_rawcc; if (sp->bytesperline == 0) + { + memset(buf, 0, (size_t)cc); return 0; + } nrows = cc / sp->bytesperline; if (cc % sp->bytesperline) @@ -1492,7 +1475,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) JSAMPROW bufptr = (JSAMPROW)buf; if (TIFFjpeg_read_scanlines(sp, &bufptr, 1) != 1) + { + memset(buf, 0, (size_t)cc); return (0); + } ++tif->tif_row; buf += sp->bytesperline; @@ -1526,7 +1512,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) sp->src.bytes_in_buffer = (size_t)tif->tif_rawcc; if (sp->bytesperline == 0) + { + memset(buf, 0, (size_t)cc); return 0; + } nrows = cc / sp->bytesperline; if (cc % sp->bytesperline) @@ -1562,7 +1551,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) * for 12bit data, which we need to repack. */ if (TIFFjpeg_read_scanlines(sp, &line_work_buf, 1) != 1) + { + memset(buf, 0, (size_t)cc); return (0); + } if (sp->cinfo.d.data_precision == 12) { @@ -2190,6 +2182,12 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) /* for PC 2, scale down the strip/tile size * to match a downsampled component */ + if (sp->h_sampling == 0 || sp->v_sampling == 0) + { + TIFFErrorExtR(tif, module, + "JPEG horizontal or vertical sampling is zero"); + return (0); + } segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); } diff --git a/contrib/libs/libtiff/tif_lerc.c b/contrib/libs/libtiff/tif_lerc.c index 19d330f3920..29be0110d20 100644 --- a/contrib/libs/libtiff/tif_lerc.c +++ b/contrib/libs/libtiff/tif_lerc.c @@ -71,6 +71,9 @@ typedef struct uint8_t *uncompressed_buffer; unsigned int uncompressed_offset; + uint8_t *uncompressed_buffer_multiband; + unsigned int uncompressed_buffer_multiband_alloc; + unsigned int mask_size; uint8_t *mask_buffer; @@ -86,9 +89,9 @@ typedef struct TIFFVSetMethod vsetparent; /* super-class method */ } LERCState; -#define LState(tif) ((LERCState *)(tif)->tif_data) -#define DecoderState(tif) LState(tif) -#define EncoderState(tif) LState(tif) +#define GetLERCState(tif) ((LERCState *)(tif)->tif_data) +#define LERCDecoderState(tif) GetLERCState(tif) +#define LERCEncoderState(tif) GetLERCState(tif) static int LERCEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); @@ -101,7 +104,7 @@ static int LERCFixupTags(TIFF *tif) static int LERCSetupDecode(TIFF *tif) { - LERCState *sp = DecoderState(tif); + LERCState *sp = LERCDecoderState(tif); assert(sp != NULL); @@ -168,7 +171,7 @@ static int GetLercDataType(TIFF *tif) return -1; } -static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) +static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) { TIFFDirectory *td = &tif->tif_dir; uint64_t new_size_64; @@ -202,8 +205,9 @@ static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) sp->uncompressed_size = new_size; /* add some margin as we are going to use it also to store deflate/zstd - * compressed data */ - new_alloc_64 = 100 + new_size_64 + new_size_64 / 3; + * compressed data. We also need extra margin when writing very small + * rasters with one mask per band. */ + new_alloc_64 = 256 + new_size_64 + new_size_64 / 3; #ifdef ZSTD_SUPPORT { size_t zstd_max = ZSTD_compressBound((size_t)new_size_64); @@ -243,11 +247,17 @@ static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) || (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - td->td_samplesperpixel == 1) && (td->td_bitspersample == 32 || td->td_bitspersample == 64))) { unsigned int mask_size = sp->segment_width * sp->segment_height; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG) + { + /* We may need one mask per band */ + mask_size *= td->td_samplesperpixel; + } +#endif if (sp->mask_size < mask_size) { void *mask_buffer = @@ -277,9 +287,9 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) static const char module[] = "LERCPreDecode"; lerc_status lerc_ret; TIFFDirectory *td = &tif->tif_dir; - LERCState *sp = DecoderState(tif); + LERCState *sp = LERCDecoderState(tif); int lerc_data_type; - unsigned int infoArray[8]; + unsigned int infoArray[9]; unsigned nomask_bands = td->td_samplesperpixel; int ndims; int use_mask = 0; @@ -295,7 +305,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (lerc_data_type < 0) return 0; - if (!SetupUncompressedBuffer(tif, sp, module)) + if (!SetupBuffers(tif, sp, module)) return 0; if (sp->additional_compression != LERC_ADD_COMPRESSION_NONE) @@ -400,7 +410,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } lerc_ret = - lerc_getBlobInfo(lerc_data, lerc_data_size, infoArray, NULL, 8, 0); + lerc_getBlobInfo(lerc_data, lerc_data_size, infoArray, NULL, 9, 0); if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_getBlobInfo() failed"); @@ -418,18 +428,16 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) use_mask = 1; nomask_bands--; } - else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - td->td_samplesperpixel == 1) && - (td->td_bitspersample == 32 || td->td_bitspersample == 64)) + else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP) { use_mask = 1; } ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1; - /* Info returned in infoArray is { version, dataType, nDim, nCols, - nRows, nBands, nValidPixels, blobSize } */ + /* Info returned in infoArray is { version, dataType, nDim/nDepth, nCols, + nRows, nBands, nValidPixels, blobSize, + and starting with liblerc 3.0 nRequestedMasks } */ if (infoArray[0] != (unsigned)sp->lerc_version) { TIFFWarningExtR(tif, module, @@ -442,12 +450,29 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) infoArray[1], lerc_data_type); return 0; } - if (infoArray[2] != (unsigned)ndims) + + const unsigned nFoundDims = infoArray[2]; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1) + { + if (nFoundDims != 1 && nFoundDims != (unsigned)ndims) + { + TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: 1 or %d", + nFoundDims, ndims); + return 0; + } + } + else +#endif + if (nFoundDims != (unsigned)ndims) { TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: %d", - infoArray[2], ndims); + nFoundDims, ndims); return 0; } + if (infoArray[3] != sp->segment_width) { TIFFErrorExtR(tif, module, "Unexpected nCols: %d. Expected: %du", @@ -460,12 +485,38 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) infoArray[4], sp->segment_height); return 0; } - if (infoArray[5] != 1) + + const unsigned nFoundBands = infoArray[5]; + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1 && nFoundDims == 1) + { +#if !LERC_AT_LEAST_VERSION(3, 0, 0) + if (nFoundBands == td->td_samplesperpixel) + { + TIFFErrorExtR( + tif, module, + "Unexpected nBands: %d. This file may have been generated with " + "a liblerc version >= 3.0, with one mask per band, and is not " + "supported by this older version of liblerc", + nFoundBands); + return 0; + } +#endif + if (nFoundBands != td->td_samplesperpixel) + { + TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + nFoundBands, td->td_samplesperpixel); + return 0; + } + } + else if (nFoundBands != 1) { TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", - infoArray[5], 1); + nFoundBands, 1); return 0; } + if (infoArray[7] != lerc_data_size) { TIFFErrorExtR(tif, module, "Unexpected blobSize: %d. Expected: %u", @@ -473,13 +524,75 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) return 0; } - lerc_ret = lerc_decode(lerc_data, lerc_data_size, + int nRequestedMasks = use_mask ? 1 : 0; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + const int nFoundMasks = infoArray[8]; + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1 && nFoundDims == 1) + { + if (nFoundMasks != 0 && nFoundMasks != td->td_samplesperpixel) + { + TIFFErrorExtR(tif, module, + "Unexpected nFoundMasks: %d. Expected: 0 or %d", + nFoundMasks, td->td_samplesperpixel); + return 0; + } + nRequestedMasks = nFoundMasks; + } + else + { + if (nFoundMasks != 0 && nFoundMasks != 1) + { + TIFFErrorExtR(tif, module, + "Unexpected nFoundMasks: %d. Expected: 0 or 1", + nFoundMasks); + return 0; + } + } + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && nFoundMasks == 0) + { + nRequestedMasks = 0; + use_mask = 0; + } +#endif + + const unsigned nb_pixels = sp->segment_width * sp->segment_height; + #if LERC_AT_LEAST_VERSION(3, 0, 0) - use_mask ? 1 : 0, + if (nRequestedMasks > 1) + { + unsigned int num_bytes_needed = + nb_pixels * td->td_samplesperpixel * (td->td_bitspersample / 8); + if (sp->uncompressed_buffer_multiband_alloc < num_bytes_needed) + { + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); + sp->uncompressed_buffer_multiband = + _TIFFmallocExt(tif, num_bytes_needed); + if (!sp->uncompressed_buffer_multiband) + { + sp->uncompressed_buffer_multiband_alloc = 0; + return 0; + } + sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; + } + lerc_ret = lerc_decode(lerc_data, lerc_data_size, nRequestedMasks, + sp->mask_buffer, nFoundDims, sp->segment_width, + sp->segment_height, nFoundBands, lerc_data_type, + sp->uncompressed_buffer_multiband); + } + else #endif - use_mask ? sp->mask_buffer : NULL, ndims, - sp->segment_width, sp->segment_height, 1, - lerc_data_type, sp->uncompressed_buffer); + { + lerc_ret = + lerc_decode(lerc_data, lerc_data_size, +#if LERC_AT_LEAST_VERSION(3, 0, 0) + nRequestedMasks, +#endif + use_mask ? sp->mask_buffer : NULL, nFoundDims, + sp->segment_width, sp->segment_height, nFoundBands, + lerc_data_type, sp->uncompressed_buffer); + } if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_decode() failed"); @@ -515,7 +628,6 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else if (use_mask && td->td_sampleformat == SAMPLEFORMAT_IEEEFP) { - const unsigned nb_pixels = sp->segment_width * sp->segment_height; unsigned i; #if WORDS_BIGENDIAN const unsigned char nan_bytes[] = {0x7f, 0xc0, 0, 0}; @@ -525,23 +637,104 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) float nan_float32; memcpy(&nan_float32, nan_bytes, 4); - if (td->td_bitspersample == 32) + if (td->td_planarconfig == PLANARCONFIG_SEPARATE || + td->td_samplesperpixel == 1) { - for (i = 0; i < nb_pixels; i++) + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + if (sp->mask_buffer[i] == 0) + ((float *)sp->uncompressed_buffer)[i] = nan_float32; + } + } + else + { + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + if (sp->mask_buffer[i] == 0) + ((double *)sp->uncompressed_buffer)[i] = nan_float64; + } + } + } + else if (nRequestedMasks == 1) + { + assert(nFoundDims == td->td_samplesperpixel); + assert(nFoundBands == 1); + + unsigned k = 0; + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i] == 0) + ((float *)sp->uncompressed_buffer)[k] = nan_float32; + ++k; + } + } + } + else { - if (sp->mask_buffer[i] == 0) - ((float *)sp->uncompressed_buffer)[i] = nan_float32; + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i] == 0) + ((double *)sp->uncompressed_buffer)[k] = + nan_float64; + ++k; + } + } } } +#if LERC_AT_LEAST_VERSION(3, 0, 0) else { - const double nan_float64 = nan_float32; - for (i = 0; i < nb_pixels; i++) + assert(nRequestedMasks == td->td_samplesperpixel); + assert(nFoundDims == 1); + assert(nFoundBands == td->td_samplesperpixel); + + unsigned k = 0; + if (td->td_bitspersample == 32) { - if (sp->mask_buffer[i] == 0) - ((double *)sp->uncompressed_buffer)[i] = nan_float64; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i + j * nb_pixels] == 0) + ((float *)sp->uncompressed_buffer)[k] = nan_float32; + else + ((float *)sp->uncompressed_buffer)[k] = + ((float *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels]; + ++k; + } + } + } + else + { + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i + j * nb_pixels] == 0) + ((double *)sp->uncompressed_buffer)[k] = + nan_float64; + else + ((double *)sp->uncompressed_buffer)[k] = + ((double *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels]; + ++k; + } + } } } +#endif } return 1; @@ -553,7 +746,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "LERCDecode"; - LERCState *sp = DecoderState(tif); + LERCState *sp = LERCDecoderState(tif); (void)s; assert(sp != NULL); @@ -561,6 +754,7 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (sp->uncompressed_buffer == 0) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Uncompressed buffer not allocated"); return 0; } @@ -568,6 +762,7 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if ((uint64_t)sp->uncompressed_offset + (uint64_t)occ > sp->uncompressed_size) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Too many bytes read"); return 0; } @@ -580,7 +775,7 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) static int LERCSetupEncode(TIFF *tif) { - LERCState *sp = EncoderState(tif); + LERCState *sp = LERCEncoderState(tif); assert(sp != NULL); if (sp->state & LSTATE_INIT_DECODE) @@ -599,7 +794,7 @@ static int LERCSetupEncode(TIFF *tif) static int LERCPreEncode(TIFF *tif, uint16_t s) { static const char module[] = "LERCPreEncode"; - LERCState *sp = EncoderState(tif); + LERCState *sp = LERCEncoderState(tif); int lerc_data_type; (void)s; @@ -611,7 +806,7 @@ static int LERCPreEncode(TIFF *tif, uint16_t s) if (lerc_data_type < 0) return 0; - if (!SetupUncompressedBuffer(tif, sp, module)) + if (!SetupBuffers(tif, sp, module)) return 0; return 1; @@ -623,7 +818,7 @@ static int LERCPreEncode(TIFF *tif, uint16_t s) static int LERCEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { static const char module[] = "LERCEncode"; - LERCState *sp = EncoderState(tif); + LERCState *sp = LERCEncoderState(tif); (void)s; assert(sp != NULL); @@ -649,8 +844,7 @@ static int LERCPostEncode(TIFF *tif) { lerc_status lerc_ret; static const char module[] = "LERCPostEncode"; - LERCState *sp = EncoderState(tif); - unsigned int numBytes = 0; + LERCState *sp = LERCEncoderState(tif); unsigned int numBytesWritten = 0; TIFFDirectory *td = &tif->tif_dir; int use_mask = 0; @@ -662,6 +856,9 @@ static int LERCPostEncode(TIFF *tif) return 0; } + int mask_count = 1; + const unsigned nb_pixels = sp->segment_width * sp->segment_height; + /* Extract alpha mask (if containing only 0 and 255 values, */ /* and compact array of regular bands */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && @@ -673,7 +870,6 @@ static int LERCPostEncode(TIFF *tif) const unsigned src_stride = td->td_samplesperpixel * (td->td_bitspersample / 8); unsigned i = 0; - const unsigned nb_pixels = sp->segment_width * sp->segment_height; use_mask = 1; for (i = 0; i < nb_pixels; i++) @@ -710,109 +906,247 @@ static int LERCPostEncode(TIFF *tif) } } else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - dst_nbands == 1) && (td->td_bitspersample == 32 || td->td_bitspersample == 64)) { /* Check for NaN values */ unsigned i; - const unsigned nb_pixels = sp->segment_width * sp->segment_height; if (td->td_bitspersample == 32) { - for (i = 0; i < nb_pixels; i++) + if (td->td_planarconfig == PLANARCONFIG_CONTIG && dst_nbands > 1) { - const float val = ((float *)sp->uncompressed_buffer)[i]; - if (val != val) + unsigned k = 0; + for (i = 0; i < nb_pixels; i++) { - use_mask = 1; - break; + int count_nan = 0; + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const float val = ((float *)sp->uncompressed_buffer)[k]; + ++k; + if (val != val) + { + ++count_nan; + } + } + if (count_nan > 0) + { + use_mask = 1; + if (count_nan < td->td_samplesperpixel) + { + mask_count = td->td_samplesperpixel; + break; + } + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const float val = ((float *)sp->uncompressed_buffer)[i]; + if (val != val) + { + use_mask = 1; + break; + } } } } else { - for (i = 0; i < nb_pixels; i++) + if (td->td_planarconfig == PLANARCONFIG_CONTIG && dst_nbands > 1) { - const double val = ((double *)sp->uncompressed_buffer)[i]; - if (val != val) + unsigned k = 0; + for (i = 0; i < nb_pixels; i++) + { + int count_nan = 0; + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const double val = + ((double *)sp->uncompressed_buffer)[k]; + ++k; + if (val != val) + { + ++count_nan; + } + } + if (count_nan > 0) + { + use_mask = 1; + if (count_nan < td->td_samplesperpixel) + { + mask_count = td->td_samplesperpixel; + break; + } + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) { - use_mask = 1; - break; + const double val = ((double *)sp->uncompressed_buffer)[i]; + if (val != val) + { + use_mask = 1; + break; + } } } } if (use_mask) { - if (td->td_bitspersample == 32) + if (mask_count > 1) { - for (i = 0; i < nb_pixels; i++) +#if LERC_AT_LEAST_VERSION(3, 0, 0) + unsigned int num_bytes_needed = + nb_pixels * dst_nbands * (td->td_bitspersample / 8); + if (sp->uncompressed_buffer_multiband_alloc < num_bytes_needed) { - const float val = ((float *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); + sp->uncompressed_buffer_multiband = + _TIFFmallocExt(tif, num_bytes_needed); + if (!sp->uncompressed_buffer_multiband) + { + sp->uncompressed_buffer_multiband_alloc = 0; + return 0; + } + sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; + } + + unsigned k = 0; + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const float val = + ((float *)sp->uncompressed_buffer)[k]; + ((float *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels] = val; + ++k; + sp->mask_buffer[i + j * nb_pixels] = + (val == val) ? 255 : 0; + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const double val = + ((double *)sp->uncompressed_buffer)[k]; + ((double *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels] = val; + ++k; + sp->mask_buffer[i + j * nb_pixels] = + (val == val) ? 255 : 0; + } + } + } +#else + TIFFErrorExtR(tif, module, + "lerc_encode() would need to create one mask per " + "sample, but this requires liblerc >= 3.0"); + return 0; +#endif + } + else if (td->td_planarconfig == PLANARCONFIG_CONTIG && + dst_nbands > 1) + { + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + const float val = + ((float *)sp->uncompressed_buffer)[i * dst_nbands]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const double val = + ((double *)sp->uncompressed_buffer)[i * dst_nbands]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } } } else { - for (i = 0; i < nb_pixels; i++) + if (td->td_bitspersample == 32) { - const double val = ((double *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + for (i = 0; i < nb_pixels; i++) + { + const float val = ((float *)sp->uncompressed_buffer)[i]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const double val = + ((double *)sp->uncompressed_buffer)[i]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } } } } } -#if 0 - lerc_ret = lerc_computeCompressedSize( - sp->uncompressed_buffer, - sp->lerc_version, - GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? - dst_nbands : 1, - sp->segment_width, - sp->segment_height, - 1, - use_mask ? sp->mask_buffer : NULL, - sp->maxzerror, - &numBytes); - if( lerc_ret != 0 ) - { - TIFFErrorExtR(tif, module, - "lerc_computeCompressedSize() failed"); - return 0; - } -#else - numBytes = sp->uncompressed_alloc; + unsigned int estimated_compressed_size = sp->uncompressed_alloc; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (mask_count > 1) + { + estimated_compressed_size += nb_pixels * mask_count / 8; + } #endif - if (sp->compressed_size < numBytes) + if (sp->compressed_size < estimated_compressed_size) { _TIFFfreeExt(tif, sp->compressed_buffer); - sp->compressed_buffer = _TIFFmallocExt(tif, numBytes); + sp->compressed_buffer = _TIFFmallocExt(tif, estimated_compressed_size); if (!sp->compressed_buffer) { sp->compressed_size = 0; return 0; } - sp->compressed_size = numBytes; + sp->compressed_size = estimated_compressed_size; } - lerc_ret = lerc_encodeForVersion( - sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, - sp->segment_width, sp->segment_height, 1, #if LERC_AT_LEAST_VERSION(3, 0, 0) - use_mask ? 1 : 0, + if (mask_count > 1) + { + lerc_ret = lerc_encodeForVersion( + sp->uncompressed_buffer_multiband, sp->lerc_version, + GetLercDataType(tif), 1, sp->segment_width, sp->segment_height, + dst_nbands, dst_nbands, sp->mask_buffer, sp->maxzerror, + sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + } + else #endif - use_mask ? sp->mask_buffer : NULL, sp->maxzerror, sp->compressed_buffer, - sp->compressed_size, &numBytesWritten); + { + lerc_ret = lerc_encodeForVersion( + sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), + td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, + sp->segment_width, sp->segment_height, 1, +#if LERC_AT_LEAST_VERSION(3, 0, 0) + use_mask ? 1 : 0, +#endif + use_mask ? sp->mask_buffer : NULL, sp->maxzerror, + sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + } if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_encode() failed"); return 0; } - assert(numBytesWritten < numBytes); + assert(numBytesWritten < estimated_compressed_size); if (sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE) { @@ -950,7 +1284,7 @@ static int LERCPostEncode(TIFF *tif) static void LERCCleanup(TIFF *tif) { - LERCState *sp = LState(tif); + LERCState *sp = GetLERCState(tif); assert(sp != 0); @@ -958,6 +1292,7 @@ static void LERCCleanup(TIFF *tif) tif->tif_tagmethods.vsetfield = sp->vsetparent; _TIFFfreeExt(tif, sp->uncompressed_buffer); + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); _TIFFfreeExt(tif, sp->compressed_buffer); _TIFFfreeExt(tif, sp->mask_buffer); @@ -995,7 +1330,7 @@ static const TIFFField LERCFields[] = { static int LERCVSetFieldBase(TIFF *tif, uint32_t tag, ...) { - LERCState *sp = LState(tif); + LERCState *sp = GetLERCState(tif); int ret; va_list ap; va_start(ap, tag); @@ -1007,7 +1342,7 @@ static int LERCVSetFieldBase(TIFF *tif, uint32_t tag, ...) static int LERCVSetField(TIFF *tif, uint32_t tag, va_list ap) { static const char module[] = "LERCVSetField"; - LERCState *sp = LState(tif); + LERCState *sp = GetLERCState(tif); switch (tag) { @@ -1115,7 +1450,7 @@ static int LERCVSetField(TIFF *tif, uint32_t tag, va_list ap) static int LERCVGetField(TIFF *tif, uint32_t tag, va_list ap) { - LERCState *sp = LState(tif); + LERCState *sp = GetLERCState(tif); switch (tag) { @@ -1163,7 +1498,7 @@ int TIFFInitLERC(TIFF *tif, int scheme) tif->tif_data = (uint8_t *)_TIFFcallocExt(tif, 1, sizeof(LERCState)); if (tif->tif_data == NULL) goto bad; - sp = LState(tif); + sp = GetLERCState(tif); /* * Override parent get/set field methods. diff --git a/contrib/libs/libtiff/tif_luv.c b/contrib/libs/libtiff/tif_luv.c index 021756d5d6d..d19653550c6 100644 --- a/contrib/libs/libtiff/tif_luv.c +++ b/contrib/libs/libtiff/tif_luv.c @@ -951,7 +951,8 @@ static int uv_encode(double u, double v, int em) /* encode (u',v') coordinates */ { - register int vi, ui; + unsigned int vi; + int ui; /* check for NaN */ if (u != u || v != v) @@ -980,8 +981,9 @@ static int uv_decode(double *up, double *vp, int c) /* decode (u',v') index */ { - int upper, lower; - register int ui, vi; + unsigned int upper, lower; + int ui; + unsigned int vi; if (c < 0 || c >= UV_NDIVS) return (-1); diff --git a/contrib/libs/libtiff/tif_lzma.c b/contrib/libs/libtiff/tif_lzma.c index 4cfd5e88212..db1c8b6829f 100644 --- a/contrib/libs/libtiff/tif_lzma.c +++ b/contrib/libs/libtiff/tif_lzma.c @@ -44,6 +44,8 @@ typedef struct { TIFFPredictorState predict; + int read_error; /* whether a read error has occurred, and which should cause + further reads in the same strip/tile to be aborted */ lzma_stream stream; lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_options_delta opt_delta; /* delta filter options */ @@ -58,9 +60,9 @@ typedef struct TIFFVSetMethod vsetparent; /* super-class method */ } LZMAState; -#define LState(tif) ((LZMAState *)(tif)->tif_data) -#define DecoderState(tif) LState(tif) -#define EncoderState(tif) LState(tif) +#define GetLZMAState(tif) ((LZMAState *)(tif)->tif_data) +#define LZMADecoderState(tif) GetLZMAState(tif) +#define LZMAEncoderState(tif) GetLZMAState(tif) static int LZMAEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); @@ -106,7 +108,7 @@ static int LZMAFixupTags(TIFF *tif) static int LZMASetupDecode(TIFF *tif) { - LZMAState *sp = DecoderState(tif); + LZMAState *sp = LZMADecoderState(tif); assert(sp != NULL); @@ -127,7 +129,7 @@ static int LZMASetupDecode(TIFF *tif) static int LZMAPreDecode(TIFF *tif, uint16_t s) { static const char module[] = "LZMAPreDecode"; - LZMAState *sp = DecoderState(tif); + LZMAState *sp = LZMADecoderState(tif); lzma_ret ret; (void)s; @@ -156,18 +158,31 @@ static int LZMAPreDecode(TIFF *tif, uint16_t s) LZMAStrerror(ret)); return 0; } + + sp->read_error = 0; + return 1; } static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "LZMADecode"; - LZMAState *sp = DecoderState(tif); + LZMAState *sp = LZMADecoderState(tif); (void)s; assert(sp != NULL); assert(sp->state == LSTATE_INIT_DECODE); + if (sp->read_error) + { + memset(op, 0, (size_t)occ); + TIFFErrorExtR(tif, module, + "LZMADecode: Scanline %" PRIu32 " cannot be read due to " + "previous error", + tif->tif_row); + return 0; + } + sp->stream.next_in = tif->tif_rawcp; sp->stream.avail_in = (size_t)tif->tif_rawcc; @@ -175,6 +190,9 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) sp->stream.avail_out = (size_t)occ; if ((tmsize_t)sp->stream.avail_out != occ) { + // read_error not set here as this is a usage issue that can be + // recovered in a following call. + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Liblzma cannot deal with buffers this size"); return 0; @@ -198,6 +216,8 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) lzma_stream_decoder(&sp->stream, lzma_memusage(&sp->stream), 0); if (r != LZMA_OK) { + sp->read_error = 1; + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Error initializing the stream decoder, %s", LZMAStrerror(r)); @@ -217,6 +237,8 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } while (sp->stream.avail_out > 0); if (sp->stream.avail_out != 0) { + sp->read_error = 1; + memset(sp->stream.next_out, 0, sp->stream.avail_out); TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" TIFF_SIZE_FORMAT " bytes)", @@ -232,7 +254,7 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) static int LZMASetupEncode(TIFF *tif) { - LZMAState *sp = EncoderState(tif); + LZMAState *sp = LZMAEncoderState(tif); assert(sp != NULL); if (sp->state & LSTATE_INIT_DECODE) @@ -251,7 +273,7 @@ static int LZMASetupEncode(TIFF *tif) static int LZMAPreEncode(TIFF *tif, uint16_t s) { static const char module[] = "LZMAPreEncode"; - LZMAState *sp = EncoderState(tif); + LZMAState *sp = LZMAEncoderState(tif); lzma_ret ret; (void)s; @@ -283,7 +305,7 @@ static int LZMAPreEncode(TIFF *tif, uint16_t s) static int LZMAEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { static const char module[] = "LZMAEncode"; - LZMAState *sp = EncoderState(tif); + LZMAState *sp = LZMAEncoderState(tif); assert(sp != NULL); assert(sp->state == LSTATE_INIT_ENCODE); @@ -329,7 +351,7 @@ static int LZMAEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) static int LZMAPostEncode(TIFF *tif) { static const char module[] = "LZMAPostEncode"; - LZMAState *sp = EncoderState(tif); + LZMAState *sp = LZMAEncoderState(tif); lzma_ret ret; sp->stream.avail_in = 0; @@ -365,7 +387,7 @@ static int LZMAPostEncode(TIFF *tif) static void LZMACleanup(TIFF *tif) { - LZMAState *sp = LState(tif); + LZMAState *sp = GetLZMAState(tif); assert(sp != 0); @@ -388,7 +410,7 @@ static void LZMACleanup(TIFF *tif) static int LZMAVSetField(TIFF *tif, uint32_t tag, va_list ap) { static const char module[] = "LZMAVSetField"; - LZMAState *sp = LState(tif); + LZMAState *sp = GetLZMAState(tif); switch (tag) { @@ -414,7 +436,7 @@ static int LZMAVSetField(TIFF *tif, uint32_t tag, va_list ap) static int LZMAVGetField(TIFF *tif, uint32_t tag, va_list ap) { - LZMAState *sp = LState(tif); + LZMAState *sp = GetLZMAState(tif); switch (tag) { @@ -457,7 +479,7 @@ int TIFFInitLZMA(TIFF *tif, int scheme) tif->tif_data = (uint8_t *)_TIFFmallocExt(tif, sizeof(LZMAState)); if (tif->tif_data == NULL) goto bad; - sp = LState(tif); + sp = GetLZMAState(tif); memcpy(&sp->stream, &tmp_stream, sizeof(lzma_stream)); /* diff --git a/contrib/libs/libtiff/tif_lzw.c b/contrib/libs/libtiff/tif_lzw.c index d631fa10494..4baf78e50b2 100644 --- a/contrib/libs/libtiff/tif_lzw.c +++ b/contrib/libs/libtiff/tif_lzw.c @@ -161,8 +161,8 @@ typedef struct } LZWCodecState; #define LZWState(tif) ((LZWBaseState *)(tif)->tif_data) -#define DecoderState(tif) ((LZWCodecState *)LZWState(tif)) -#define EncoderState(tif) ((LZWCodecState *)LZWState(tif)) +#define LZWDecoderState(tif) ((LZWCodecState *)LZWState(tif)) +#define LZWEncoderState(tif) ((LZWCodecState *)LZWState(tif)) static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s); #ifdef LZW_COMPAT @@ -183,7 +183,7 @@ static int LZWFixupTags(TIFF *tif) static int LZWSetupDecode(TIFF *tif) { static const char module[] = "LZWSetupDecode"; - LZWCodecState *sp = DecoderState(tif); + LZWCodecState *sp = LZWDecoderState(tif); int code; if (sp == NULL) @@ -199,7 +199,7 @@ static int LZWSetupDecode(TIFF *tif) return (0); } - sp = DecoderState(tif); + sp = LZWDecoderState(tif); sp->dec_codetab = NULL; sp->dec_decode = NULL; @@ -245,7 +245,7 @@ static int LZWSetupDecode(TIFF *tif) static int LZWPreDecode(TIFF *tif, uint16_t s) { static const char module[] = "LZWPreDecode"; - LZWCodecState *sp = DecoderState(tif); + LZWCodecState *sp = LZWDecoderState(tif); (void)s; assert(sp != NULL); @@ -329,10 +329,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) #ifdef WORDS_BIGENDIAN #define GetNextData(nextdata, bp) memcpy(&nextdata, bp, sizeof(nextdata)) #elif SIZEOF_WORDTYPE == 8 -#if defined(__GNUC__) && defined(__x86_64__) -#define GetNextData(nextdata, bp) \ - nextdata = __builtin_bswap64(*(uint64_t *)(bp)) -#elif defined(_M_X64) +#if defined(_M_X64) #define GetNextData(nextdata, bp) nextdata = _byteswap_uint64(*(uint64_t *)(bp)) #elif defined(__GNUC__) #define GetNextData(nextdata, bp) \ @@ -346,10 +343,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) (((uint64_t)bp[6]) << 8) | (((uint64_t)bp[7])) #endif #elif SIZEOF_WORDTYPE == 4 -#if defined(__GNUC__) && defined(__i386__) -#define GetNextData(nextdata, bp) \ - nextdata = __builtin_bswap32(*(uint32_t *)(bp)) -#elif defined(_M_X86) +#if defined(_M_X86) #define GetNextData(nextdata, bp) \ nextdata = _byteswap_ulong(*(unsigned long *)(bp)) #elif defined(__GNUC__) @@ -409,7 +403,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { static const char module[] = "LZWDecode"; - LZWCodecState *sp = DecoderState(tif); + LZWCodecState *sp = LZWDecoderState(tif); uint8_t *op = (uint8_t *)op0; tmsize_t occ = occ0; uint8_t *bp; @@ -423,6 +417,7 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) if (sp->read_error) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "LZWDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", @@ -737,6 +732,7 @@ after_loop: if (occ > 0) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" PRIu64 " bytes)", @@ -746,12 +742,14 @@ after_loop: return (1); no_eoi: + memset(op, 0, (size_t)occ); sp->read_error = 1; TIFFErrorExtR(tif, module, "LZWDecode: Strip %" PRIu32 " not terminated with EOI code", tif->tif_curstrip); return 0; error_code: + memset(op, 0, (size_t)occ); sp->read_error = 1; TIFFErrorExtR(tif, tif->tif_name, "Using code not yet in table"); return 0; @@ -800,7 +798,7 @@ error_code: static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { static const char module[] = "LZWDecodeCompat"; - LZWCodecState *sp = DecoderState(tif); + LZWCodecState *sp = LZWDecoderState(tif); uint8_t *op = (uint8_t *)op0; tmsize_t occ = occ0; uint8_t *tp; @@ -1026,7 +1024,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) static int LZWSetupEncode(TIFF *tif) { static const char module[] = "LZWSetupEncode"; - LZWCodecState *sp = EncoderState(tif); + LZWCodecState *sp = LZWEncoderState(tif); assert(sp != NULL); sp->enc_hashtab = (hash_t *)_TIFFmallocExt(tif, HSIZE * sizeof(hash_t)); @@ -1043,7 +1041,7 @@ static int LZWSetupEncode(TIFF *tif) */ static int LZWPreEncode(TIFF *tif, uint16_t s) { - LZWCodecState *sp = EncoderState(tif); + LZWCodecState *sp = LZWEncoderState(tif); (void)s; assert(sp != NULL); @@ -1114,7 +1112,7 @@ static int LZWPreEncode(TIFF *tif, uint16_t s) */ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { - register LZWCodecState *sp = EncoderState(tif); + register LZWCodecState *sp = LZWEncoderState(tif); register long fcode; register hash_t *hp; register int h, c; @@ -1299,7 +1297,7 @@ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) */ static int LZWPostEncode(TIFF *tif) { - register LZWCodecState *sp = EncoderState(tif); + register LZWCodecState *sp = LZWEncoderState(tif); uint8_t *op = tif->tif_rawcp; long nextbits = sp->lzw_nextbits; WordType nextdata = sp->lzw_nextdata; @@ -1381,11 +1379,11 @@ static void LZWCleanup(TIFF *tif) assert(tif->tif_data != 0); - if (DecoderState(tif)->dec_codetab) - _TIFFfreeExt(tif, DecoderState(tif)->dec_codetab); + if (LZWDecoderState(tif)->dec_codetab) + _TIFFfreeExt(tif, LZWDecoderState(tif)->dec_codetab); - if (EncoderState(tif)->enc_hashtab) - _TIFFfreeExt(tif, EncoderState(tif)->enc_hashtab); + if (LZWEncoderState(tif)->enc_hashtab) + _TIFFfreeExt(tif, LZWEncoderState(tif)->enc_hashtab); _TIFFfreeExt(tif, tif->tif_data); tif->tif_data = NULL; @@ -1404,9 +1402,9 @@ int TIFFInitLZW(TIFF *tif, int scheme) tif->tif_data = (uint8_t *)_TIFFmallocExt(tif, sizeof(LZWCodecState)); if (tif->tif_data == NULL) goto bad; - DecoderState(tif)->dec_codetab = NULL; - DecoderState(tif)->dec_decode = NULL; - EncoderState(tif)->enc_hashtab = NULL; + LZWDecoderState(tif)->dec_codetab = NULL; + LZWDecoderState(tif)->dec_decode = NULL; + LZWEncoderState(tif)->enc_hashtab = NULL; LZWState(tif)->rw_mode = tif->tif_mode; /* diff --git a/contrib/libs/libtiff/tif_ojpeg.c b/contrib/libs/libtiff/tif_ojpeg.c index ea572091e50..f94d2a4e45f 100644 --- a/contrib/libs/libtiff/tif_ojpeg.c +++ b/contrib/libs/libtiff/tif_ojpeg.c @@ -207,37 +207,21 @@ static const TIFFField ojpegFields[] = { #include <setjmp.h> #endif -/* We undefine FAR to avoid conflict with JPEG definition */ - -#ifdef FAR -#undef FAR -#endif +#include "jerror.h" +#include "jpeglib.h" -/* - Libjpeg's jmorecfg.h defines INT16 and INT32, but only if XMD_H is - not defined. Unfortunately, the MinGW and Borland compilers include - a typedef for INT32, which causes a conflict. MSVC does not include - a conflicting typedef given the headers which are included. -*/ -#if defined(__BORLANDC__) || defined(__MINGW32__) -#define XMD_H 1 +#ifndef TIFF_jpeg_source_mgr_defined +#define TIFF_jpeg_source_mgr_defined +typedef struct jpeg_source_mgr jpeg_source_mgr; #endif -/* Define "boolean" as unsigned char, not int, per Windows custom. */ -#if defined(__WIN32__) && !defined(__MINGW32__) -#ifndef __RPCNDR_H__ /* don't conflict if rpcndr.h already read */ -typedef unsigned char boolean; -#endif -#define HAVE_BOOLEAN /* prevent jmorecfg.h from redefining it */ +#ifndef TIFF_jpeg_error_mgr_defined +#define TIFF_jpeg_error_mgr_defined +typedef struct jpeg_error_mgr jpeg_error_mgr; #endif -#include "jerror.h" -#include "jpeglib.h" - -typedef struct jpeg_error_mgr jpeg_error_mgr; typedef struct jpeg_common_struct jpeg_common_struct; typedef struct jpeg_decompress_struct jpeg_decompress_struct; -typedef struct jpeg_source_mgr jpeg_source_mgr; typedef enum { @@ -771,6 +755,9 @@ static int OJPEGPreDecode(TIFF *tif, uint16_t s) if (OJPEGWriteHeaderInfo(tif) == 0) return (0); } + + sp->subsampling_convert_state = 0; + while (sp->write_curstrile < m) { if (sp->libjpeg_jpeg_query_style == 0) @@ -856,12 +843,14 @@ static int OJPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) (void)s; if (!sp->decoder_ok) { + memset(buf, 0, (size_t)cc); TIFFErrorExtR(tif, module, "Cannot decode: decoder not correctly initialized"); return 0; } if (sp->libjpeg_session_active == 0) { + memset(buf, 0, (size_t)cc); /* This should normally not happen, except that it does when */ /* using TIFFReadScanline() which calls OJPEGPostDecode() for */ /* each scanline, which assumes that a whole strile was read */ @@ -875,17 +864,24 @@ static int OJPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) } if (sp->error_in_raw_data_decoding) { + memset(buf, 0, (size_t)cc); return 0; } if (sp->libjpeg_jpeg_query_style == 0) { if (OJPEGDecodeRaw(tif, buf, cc) == 0) + { + memset(buf, 0, (size_t)cc); return (0); + } } else { if (OJPEGDecodeScanlines(tif, buf, cc) == 0) + { + memset(buf, 0, (size_t)cc); return (0); + } } return (1); } diff --git a/contrib/libs/libtiff/tif_open.c b/contrib/libs/libtiff/tif_open.c index 23fcf81c43f..59a07bee023 100644 --- a/contrib/libs/libtiff/tif_open.c +++ b/contrib/libs/libtiff/tif_open.c @@ -25,7 +25,13 @@ /* * TIFF Library. */ + +#ifdef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#undef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#endif + #include "tiffiop.h" +#include <assert.h> #include <limits.h> /* @@ -81,8 +87,9 @@ TIFFOpenOptions *TIFFOpenOptionsAlloc() void TIFFOpenOptionsFree(TIFFOpenOptions *opts) { _TIFFfree(opts); } /** Define a limit in bytes for a single memory allocation done by libtiff. - * If max_single_mem_alloc is set to 0, no other limit that the underlying - * _TIFFmalloc() will be applied, which is the default. + * If max_single_mem_alloc is set to 0, which is the default, no other limit + * that the underlying _TIFFmalloc() or + * TIFFOpenOptionsSetMaxCumulatedMemAlloc() will be applied. */ void TIFFOpenOptionsSetMaxSingleMemAlloc(TIFFOpenOptions *opts, tmsize_t max_single_mem_alloc) @@ -90,6 +97,18 @@ void TIFFOpenOptionsSetMaxSingleMemAlloc(TIFFOpenOptions *opts, opts->max_single_mem_alloc = max_single_mem_alloc; } +/** Define a limit in bytes for the cumulated memory allocations done by libtiff + * on a given TIFF handle. + * If max_cumulated_mem_alloc is set to 0, which is the default, no other limit + * that the underlying _TIFFmalloc() or + * TIFFOpenOptionsSetMaxSingleMemAlloc() will be applied. + */ +void TIFFOpenOptionsSetMaxCumulatedMemAlloc(TIFFOpenOptions *opts, + tmsize_t max_cumulated_mem_alloc) +{ + opts->max_cumulated_mem_alloc = max_cumulated_mem_alloc; +} + void TIFFOpenOptionsSetErrorHandlerExtR(TIFFOpenOptions *opts, TIFFErrorHandlerExtR handler, void *errorhandler_user_data) @@ -117,6 +136,30 @@ static void _TIFFEmitErrorAboveMaxSingleMemAlloc(TIFF *tif, (uint64_t)s, (uint64_t)tif->tif_max_single_mem_alloc); } +static void _TIFFEmitErrorAboveMaxCumulatedMemAlloc(TIFF *tif, + const char *pszFunction, + tmsize_t s) +{ + TIFFErrorExtR(tif, pszFunction, + "Cumulated memory allocation of %" PRIu64 " + %" PRIu64 + " bytes is beyond the %" PRIu64 + " cumulated byte limit defined in open options", + (uint64_t)tif->tif_cur_cumulated_mem_alloc, (uint64_t)s, + (uint64_t)tif->tif_max_cumulated_mem_alloc); +} + +/* When allocating memory, we write at the beginning of the buffer it size. + * This allows us to keep track of the total memory allocated when we + * malloc/calloc/realloc and free. In theory we need just SIZEOF_SIZE_T bytes + * for that, but on x86_64, allocations of more than 16 bytes are aligned on + * 16 bytes. Hence using 2 * SIZEOF_SIZE_T. + * It is critical that _TIFFmallocExt/_TIFFcallocExt/_TIFFreallocExt are + * paired with _TIFFfreeExt. + * CMakeLists.txt defines TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS, which in + * turn disables the definition of the non Ext version in tiffio.h + */ +#define LEADING_AREA_TO_STORE_ALLOC_SIZE (2 * SIZEOF_SIZE_T) + /** malloc() version that takes into account memory-specific open options */ void *_TIFFmallocExt(TIFF *tif, tmsize_t s) { @@ -126,16 +169,32 @@ void *_TIFFmallocExt(TIFF *tif, tmsize_t s) _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFmallocExt", s); return NULL; } + if (tif != NULL && tif->tif_max_cumulated_mem_alloc > 0) + { + if (s > tif->tif_max_cumulated_mem_alloc - + tif->tif_cur_cumulated_mem_alloc || + s > TIFF_TMSIZE_T_MAX - LEADING_AREA_TO_STORE_ALLOC_SIZE) + { + _TIFFEmitErrorAboveMaxCumulatedMemAlloc(tif, "_TIFFmallocExt", s); + return NULL; + } + void *ptr = _TIFFmalloc(LEADING_AREA_TO_STORE_ALLOC_SIZE + s); + if (!ptr) + return NULL; + tif->tif_cur_cumulated_mem_alloc += s; + memcpy(ptr, &s, sizeof(s)); + return (char *)ptr + LEADING_AREA_TO_STORE_ALLOC_SIZE; + } return _TIFFmalloc(s); } /** calloc() version that takes into account memory-specific open options */ void *_TIFFcallocExt(TIFF *tif, tmsize_t nmemb, tmsize_t siz) { + if (nmemb <= 0 || siz <= 0 || nmemb > TIFF_TMSIZE_T_MAX / siz) + return NULL; if (tif != NULL && tif->tif_max_single_mem_alloc > 0) { - if (nmemb <= 0 || siz <= 0 || nmemb > TIFF_TMSIZE_T_MAX / siz) - return NULL; if (nmemb * siz > tif->tif_max_single_mem_alloc) { _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFcallocExt", @@ -143,6 +202,23 @@ void *_TIFFcallocExt(TIFF *tif, tmsize_t nmemb, tmsize_t siz) return NULL; } } + if (tif != NULL && tif->tif_max_cumulated_mem_alloc > 0) + { + const tmsize_t s = nmemb * siz; + if (s > tif->tif_max_cumulated_mem_alloc - + tif->tif_cur_cumulated_mem_alloc || + s > TIFF_TMSIZE_T_MAX - LEADING_AREA_TO_STORE_ALLOC_SIZE) + { + _TIFFEmitErrorAboveMaxCumulatedMemAlloc(tif, "_TIFFcallocExt", s); + return NULL; + } + void *ptr = _TIFFcalloc(LEADING_AREA_TO_STORE_ALLOC_SIZE + s, 1); + if (!ptr) + return NULL; + tif->tif_cur_cumulated_mem_alloc += s; + memcpy(ptr, &s, sizeof(s)); + return (char *)ptr + LEADING_AREA_TO_STORE_ALLOC_SIZE; + } return _TIFFcalloc(nmemb, siz); } @@ -155,13 +231,49 @@ void *_TIFFreallocExt(TIFF *tif, void *p, tmsize_t s) _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFreallocExt", s); return NULL; } + if (tif != NULL && tif->tif_max_cumulated_mem_alloc > 0) + { + void *oldPtr = p; + tmsize_t oldSize = 0; + if (p) + { + oldPtr = (char *)p - LEADING_AREA_TO_STORE_ALLOC_SIZE; + memcpy(&oldSize, oldPtr, sizeof(oldSize)); + assert(oldSize <= tif->tif_cur_cumulated_mem_alloc); + } + if (s > oldSize && + (s > tif->tif_max_cumulated_mem_alloc - + (tif->tif_cur_cumulated_mem_alloc - oldSize) || + s > TIFF_TMSIZE_T_MAX - LEADING_AREA_TO_STORE_ALLOC_SIZE)) + { + _TIFFEmitErrorAboveMaxCumulatedMemAlloc(tif, "_TIFFreallocExt", + s - oldSize); + return NULL; + } + void *newPtr = + _TIFFrealloc(oldPtr, LEADING_AREA_TO_STORE_ALLOC_SIZE + s); + if (newPtr == NULL) + return NULL; + tif->tif_cur_cumulated_mem_alloc -= oldSize; + tif->tif_cur_cumulated_mem_alloc += s; + memcpy(newPtr, &s, sizeof(s)); + return (char *)newPtr + LEADING_AREA_TO_STORE_ALLOC_SIZE; + } return _TIFFrealloc(p, s); } /** free() version that takes into account memory-specific open options */ void _TIFFfreeExt(TIFF *tif, void *p) { - (void)tif; + if (p != NULL && tif != NULL && tif->tif_max_cumulated_mem_alloc > 0) + { + void *oldPtr = (char *)p - LEADING_AREA_TO_STORE_ALLOC_SIZE; + tmsize_t oldSize; + memcpy(&oldSize, oldPtr, sizeof(oldSize)); + assert(oldSize <= tif->tif_cur_cumulated_mem_alloc); + tif->tif_cur_cumulated_mem_alloc -= oldSize; + p = oldPtr; + } _TIFFfree(p); } @@ -231,6 +343,17 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, (uint64_t)opts->max_single_mem_alloc); goto bad2; } + if (opts && opts->max_cumulated_mem_alloc > 0 && + size_to_alloc > opts->max_cumulated_mem_alloc) + { + _TIFFErrorEarly(opts, clientdata, module, + "%s: Memory allocation of %" PRIu64 + " bytes is beyond the %" PRIu64 + " cumulated byte limit defined in open options", + name, (uint64_t)size_to_alloc, + (uint64_t)opts->max_cumulated_mem_alloc); + goto bad2; + } tif = (TIFF *)_TIFFmallocExt(NULL, size_to_alloc); if (tif == NULL) { @@ -243,6 +366,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, strcpy(tif->tif_name, name); tif->tif_mode = m & ~(O_CREAT | O_TRUNC); tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; /* non-existent directory */ + tif->tif_curdircount = TIFF_NON_EXISTENT_DIR_NUMBER; tif->tif_curoff = 0; tif->tif_curstrip = (uint32_t)-1; /* invalid strip */ tif->tif_row = (uint32_t)-1; /* read/write pre-increment */ @@ -261,6 +385,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_warnhandler = opts->warnhandler; tif->tif_warnhandler_user_data = opts->warnhandler_user_data; tif->tif_max_single_mem_alloc = opts->max_single_mem_alloc; + tif->tif_max_cumulated_mem_alloc = opts->max_cumulated_mem_alloc; } if (!readproc || !writeproc || !seekproc || !closeproc || !sizeproc) @@ -423,9 +548,9 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, TIFFErrorExtR(tif, name, "Cannot read TIFF header"); goto bad; } -/* - * Setup header and write. - */ + /* + * Setup header and write. + */ #ifdef WORDS_BIGENDIAN tif->tif_header.common.tiff_magic = (tif->tif_flags & TIFF_SWAB) ? TIFF_LITTLEENDIAN : TIFF_BIGENDIAN; @@ -433,13 +558,17 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_header.common.tiff_magic = (tif->tif_flags & TIFF_SWAB) ? TIFF_BIGENDIAN : TIFF_LITTLEENDIAN; #endif + TIFFHeaderUnion tif_header_swapped; if (!(tif->tif_flags & TIFF_BIGTIFF)) { tif->tif_header.common.tiff_version = TIFF_VERSION_CLASSIC; tif->tif_header.classic.tiff_diroff = 0; - if (tif->tif_flags & TIFF_SWAB) - TIFFSwabShort(&tif->tif_header.common.tiff_version); tif->tif_header_size = sizeof(TIFFHeaderClassic); + /* Swapped copy for writing */ + _TIFFmemcpy(&tif_header_swapped, &tif->tif_header, + sizeof(TIFFHeaderUnion)); + if (tif->tif_flags & TIFF_SWAB) + TIFFSwabShort(&tif_header_swapped.common.tiff_version); } else { @@ -447,12 +576,15 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_header.big.tiff_offsetsize = 8; tif->tif_header.big.tiff_unused = 0; tif->tif_header.big.tiff_diroff = 0; + tif->tif_header_size = sizeof(TIFFHeaderBig); + /* Swapped copy for writing */ + _TIFFmemcpy(&tif_header_swapped, &tif->tif_header, + sizeof(TIFFHeaderUnion)); if (tif->tif_flags & TIFF_SWAB) { - TIFFSwabShort(&tif->tif_header.common.tiff_version); - TIFFSwabShort(&tif->tif_header.big.tiff_offsetsize); + TIFFSwabShort(&tif_header_swapped.common.tiff_version); + TIFFSwabShort(&tif_header_swapped.big.tiff_offsetsize); } - tif->tif_header_size = sizeof(TIFFHeaderBig); } /* * The doc for "fopen" for some STD_C_LIBs says that if you @@ -462,27 +594,13 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, * on Solaris. */ TIFFSeekFile(tif, 0, SEEK_SET); - if (!WriteOK(tif, &tif->tif_header, (tmsize_t)(tif->tif_header_size))) + if (!WriteOK(tif, &tif_header_swapped, + (tmsize_t)(tif->tif_header_size))) { TIFFErrorExtR(tif, name, "Error writing TIFF header"); goto bad; } /* - * Setup the byte order handling. - */ - if (tif->tif_header.common.tiff_magic == TIFF_BIGENDIAN) - { -#ifndef WORDS_BIGENDIAN - tif->tif_flags |= TIFF_SWAB; -#endif - } - else - { -#ifdef WORDS_BIGENDIAN - tif->tif_flags |= TIFF_SWAB; -#endif - } - /* * Setup default directory. */ if (!TIFFDefaultDirectory(tif)) @@ -490,10 +608,14 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_diroff = 0; tif->tif_lastdiroff = 0; tif->tif_setdirectory_force_absolute = FALSE; + /* tif_curdircount = 0 means 'empty file opened for writing, but no IFD + * written yet' */ + tif->tif_curdircount = 0; return (tif); } + /* - * Setup the byte order handling. + * Setup the byte order handling according to the opened file for reading. */ if (tif->tif_header.common.tiff_magic != TIFF_BIGENDIAN && tif->tif_header.common.tiff_magic != TIFF_LITTLEENDIAN @@ -619,9 +741,17 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, * example, it may be broken) and want to proceed to other * directories. I this case we use the TIFF_HEADERONLY flag to open * file and return immediately after reading TIFF header. + * However, the pointer to TIFFSetField() and TIFFGetField() + * (i.e. tif->tif_tagmethods.vsetfield and + * tif->tif_tagmethods.vgetfield) need to be initialized, which is + * done in TIFFDefaultDirectory(). */ if (tif->tif_flags & TIFF_HEADERONLY) + { + if (!TIFFDefaultDirectory(tif)) + goto bad; return (tif); + } /* * Setup initial directory. @@ -764,10 +894,7 @@ int TIFFIsBigEndian(TIFF *tif) /* * Return nonzero if given file is BigTIFF style. */ -int TIFFIsBigTIFF(TIFF *tif) -{ - return (tif->tif_header.common.tiff_version == TIFF_VERSION_BIG); -} +int TIFFIsBigTIFF(TIFF *tif) { return ((tif->tif_flags & TIFF_BIGTIFF) != 0); } /* * Return pointer to file read method. diff --git a/contrib/libs/libtiff/tif_packbits.c b/contrib/libs/libtiff/tif_packbits.c index 62849f8f3c1..1ae50cbd47c 100644 --- a/contrib/libs/libtiff/tif_packbits.c +++ b/contrib/libs/libtiff/tif_packbits.c @@ -300,6 +300,7 @@ static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) tif->tif_rawcc = cc; if (occ > 0) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Not enough data for scanline %" PRIu32, tif->tif_row); return (0); diff --git a/contrib/libs/libtiff/tif_pixarlog.c b/contrib/libs/libtiff/tif_pixarlog.c index 5c0346b6eca..56cf416a7f6 100644 --- a/contrib/libs/libtiff/tif_pixarlog.c +++ b/contrib/libs/libtiff/tif_pixarlog.c @@ -670,8 +670,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) return 1; } -#define DecoderState(tif) ((PixarLogState *)(tif)->tif_data) -#define EncoderState(tif) ((PixarLogState *)(tif)->tif_data) +#define PixarLogDecoderState(tif) ((PixarLogState *)(tif)->tif_data) +#define PixarLogEncoderState(tif) ((PixarLogState *)(tif)->tif_data) static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); @@ -740,7 +740,7 @@ static int PixarLogSetupDecode(TIFF *tif) { static const char module[] = "PixarLogSetupDecode"; TIFFDirectory *td = &tif->tif_dir; - PixarLogState *sp = DecoderState(tif); + PixarLogState *sp = PixarLogDecoderState(tif); tmsize_t tbuf_size; uint32_t strip_height; @@ -813,7 +813,7 @@ static int PixarLogSetupDecode(TIFF *tif) static int PixarLogPreDecode(TIFF *tif, uint16_t s) { static const char module[] = "PixarLogPreDecode"; - PixarLogState *sp = DecoderState(tif); + PixarLogState *sp = PixarLogDecoderState(tif); (void)s; assert(sp != NULL); @@ -835,7 +835,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "PixarLogDecode"; TIFFDirectory *td = &tif->tif_dir; - PixarLogState *sp = DecoderState(tif); + PixarLogState *sp = PixarLogDecoderState(tif); tmsize_t i; tmsize_t nsamples; int llen; @@ -859,6 +859,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "%" PRIu16 " bit input not supported in PixarLog", td->td_bitspersample); + memset(op, 0, (size_t)occ); return 0; } @@ -879,12 +880,14 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (sp->stream.avail_out != nsamples * sizeof(uint16_t)) { TIFFErrorExtR(tif, module, "ZLib cannot deal with buffers this size"); + memset(op, 0, (size_t)occ); return (0); } /* Check that we will not fill more than what was allocated */ if ((tmsize_t)sp->stream.avail_out > sp->tbuf_size) { TIFFErrorExtR(tif, module, "sp->stream.avail_out > sp->tbuf_size"); + memset(op, 0, (size_t)occ); return (0); } do @@ -899,12 +902,14 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR( tif, module, "Decoding error at scanline %" PRIu32 ", %s", tif->tif_row, sp->stream.msg ? sp->stream.msg : "(null)"); + memset(op, 0, (size_t)occ); return (0); } if (state != Z_OK) { TIFFErrorExtR(tif, module, "ZLib error: %s", sp->stream.msg ? sp->stream.msg : "(null)"); + memset(op, 0, (size_t)occ); return (0); } } while (sp->stream.avail_out > 0); @@ -916,6 +921,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) "Not enough data at scanline %" PRIu32 " (short %u bytes)", tif->tif_row, sp->stream.avail_out); + memset(op, 0, (size_t)occ); return (0); } @@ -977,6 +983,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) default: TIFFErrorExtR(tif, module, "Unsupported bits/sample: %" PRIu16, td->td_bitspersample); + memset(op, 0, (size_t)occ); return (0); } } @@ -988,7 +995,7 @@ static int PixarLogSetupEncode(TIFF *tif) { static const char module[] = "PixarLogSetupEncode"; TIFFDirectory *td = &tif->tif_dir; - PixarLogState *sp = EncoderState(tif); + PixarLogState *sp = PixarLogEncoderState(tif); tmsize_t tbuf_size; assert(sp != NULL); @@ -1038,7 +1045,7 @@ static int PixarLogSetupEncode(TIFF *tif) static int PixarLogPreEncode(TIFF *tif, uint16_t s) { static const char module[] = "PixarLogPreEncode"; - PixarLogState *sp = EncoderState(tif); + PixarLogState *sp = PixarLogEncoderState(tif); (void)s; assert(sp != NULL); @@ -1294,7 +1301,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { static const char module[] = "PixarLogEncode"; TIFFDirectory *td = &tif->tif_dir; - PixarLogState *sp = EncoderState(tif); + PixarLogState *sp = PixarLogEncoderState(tif); tmsize_t i; tmsize_t n; int llen; @@ -1401,7 +1408,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) static int PixarLogPostEncode(TIFF *tif) { static const char module[] = "PixarLogPostEncode"; - PixarLogState *sp = EncoderState(tif); + PixarLogState *sp = PixarLogEncoderState(tif); int state; sp->stream.avail_in = 0; diff --git a/contrib/libs/libtiff/tif_read.c b/contrib/libs/libtiff/tif_read.c index 4fec83969ea..7efab59c6a3 100644 --- a/contrib/libs/libtiff/tif_read.c +++ b/contrib/libs/libtiff/tif_read.c @@ -105,8 +105,8 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, TIFFErrorExtR(tif, module, "Invalid buffer size"); return 0; } - new_rawdata = - (uint8_t *)_TIFFrealloc(tif->tif_rawdata, tif->tif_rawdatasize); + new_rawdata = (uint8_t *)_TIFFreallocExt(tif, tif->tif_rawdata, + tif->tif_rawdatasize); if (new_rawdata == 0) { TIFFErrorExtR(tif, module, @@ -464,6 +464,10 @@ int TIFFReadScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) if (e) (*tif->tif_postdecode)(tif, (uint8_t *)buf, tif->tif_scanlinesize); } + else + { + memset(buf, 0, (size_t)tif->tif_scanlinesize); + } return (e > 0 ? 1 : -1); } @@ -495,6 +499,11 @@ static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF *tif, uint32_t strip, rowsperstrip = td->td_rowsperstrip; if (rowsperstrip > td->td_imagelength) rowsperstrip = td->td_imagelength; + if (rowsperstrip == 0) + { + TIFFErrorExtR(tif, module, "rowsperstrip is zero"); + return ((tmsize_t)(-1)); + } stripsperplane = TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip); stripinplane = (strip % stripsperplane); @@ -544,7 +553,10 @@ tmsize_t TIFFReadEncodedStrip(TIFF *tif, uint32_t strip, void *buf, if ((size != (tmsize_t)(-1)) && (size < stripsize)) stripsize = size; if (!TIFFFillStrip(tif, strip)) + { + memset(buf, 0, (size_t)stripsize); return ((tmsize_t)(-1)); + } if ((*tif->tif_decodestrip)(tif, buf, stripsize, plane) <= 0) return ((tmsize_t)(-1)); (*tif->tif_postdecode)(tif, buf, stripsize); @@ -962,9 +974,13 @@ tmsize_t TIFFReadEncodedTile(TIFF *tif, uint32_t tile, void *buf, tmsize_t size) size = tilesize; else if (size > tilesize) size = tilesize; - if (TIFFFillTile(tif, tile) && - (*tif->tif_decodetile)(tif, (uint8_t *)buf, size, - (uint16_t)(tile / td->td_stripsperimage))) + if (!TIFFFillTile(tif, tile)) + { + memset(buf, 0, (size_t)size); + return ((tmsize_t)(-1)); + } + else if ((*tif->tif_decodetile)(tif, (uint8_t *)buf, size, + (uint16_t)(tile / td->td_stripsperimage))) { (*tif->tif_postdecode)(tif, (uint8_t *)buf, size); return (size); @@ -1449,6 +1465,11 @@ static int TIFFStartTile(TIFF *tif, uint32_t tile) tif->tif_flags |= TIFF_CODERSETUP; } tif->tif_curtile = tile; + if (td->td_tilewidth == 0) + { + TIFFErrorExtR(tif, module, "Zero tilewidth"); + return 0; + } howmany32 = TIFFhowmany_32(td->td_imagewidth, td->td_tilewidth); if (howmany32 == 0) { @@ -1545,9 +1566,14 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, if (TIFFIsTiled(tif)) { - if (!TIFFStartTile(tif, strile) || - !(*tif->tif_decodetile)(tif, (uint8_t *)outbuf, outsize, - (uint16_t)(strile / td->td_stripsperimage))) + if (!TIFFStartTile(tif, strile)) + { + ret = 0; + memset(outbuf, 0, (size_t)outsize); + } + else if (!(*tif->tif_decodetile)( + tif, (uint8_t *)outbuf, outsize, + (uint16_t)(strile / td->td_stripsperimage))) { ret = 0; } @@ -1558,14 +1584,27 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, uint32_t stripsperplane; if (rowsperstrip > td->td_imagelength) rowsperstrip = td->td_imagelength; - stripsperplane = - TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip); - if (!TIFFStartStrip(tif, strile) || - !(*tif->tif_decodestrip)(tif, (uint8_t *)outbuf, outsize, - (uint16_t)(strile / stripsperplane))) + if (rowsperstrip == 0) { + TIFFErrorExtR(tif, module, "rowsperstrip is zero"); ret = 0; } + else + { + stripsperplane = + TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip); + if (!TIFFStartStrip(tif, strile)) + { + ret = 0; + memset(outbuf, 0, (size_t)outsize); + } + else if (!(*tif->tif_decodestrip)( + tif, (uint8_t *)outbuf, outsize, + (uint16_t)(strile / stripsperplane))) + { + ret = 0; + } + } } if (ret) { diff --git a/contrib/libs/libtiff/tif_strip.c b/contrib/libs/libtiff/tif_strip.c index 1348b19e4f7..75b7b562662 100644 --- a/contrib/libs/libtiff/tif_strip.c +++ b/contrib/libs/libtiff/tif_strip.c @@ -38,6 +38,11 @@ uint32_t TIFFComputeStrip(TIFF *tif, uint32_t row, uint16_t sample) TIFFDirectory *td = &tif->tif_dir; uint32_t strip; + if (td->td_rowsperstrip == 0) + { + TIFFErrorExtR(tif, module, "Cannot compute strip: RowsPerStrip is zero"); + return 0; + } strip = row / td->td_rowsperstrip; if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { @@ -61,6 +66,11 @@ uint32_t TIFFNumberOfStrips(TIFF *tif) TIFFDirectory *td = &tif->tif_dir; uint32_t nstrips; + if (td->td_rowsperstrip == 0) + { + TIFFWarningExtR(tif, "TIFFNumberOfStrips", "RowsPerStrip is zero"); + return 0; + } nstrips = (td->td_rowsperstrip == (uint32_t)-1 ? 1 : TIFFhowmany_32(td->td_imagelength, td->td_rowsperstrip)); @@ -107,7 +117,8 @@ uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) if ((ycbcrsubsampling[0] != 1 && ycbcrsubsampling[0] != 2 && ycbcrsubsampling[0] != 4) || (ycbcrsubsampling[1] != 1 && ycbcrsubsampling[1] != 2 && - ycbcrsubsampling[1] != 4)) + ycbcrsubsampling[1] != 4) || + (ycbcrsubsampling[0] == 0 || ycbcrsubsampling[1] == 0)) { TIFFErrorExtR(tif, module, "Invalid YCbCr subsampling (%dx%d)", ycbcrsubsampling[0], ycbcrsubsampling[1]); @@ -267,7 +278,8 @@ uint64_t TIFFScanlineSize64(TIFF *tif) if (((ycbcrsubsampling[0] != 1) && (ycbcrsubsampling[0] != 2) && (ycbcrsubsampling[0] != 4)) || ((ycbcrsubsampling[1] != 1) && (ycbcrsubsampling[1] != 2) && - (ycbcrsubsampling[1] != 4))) + (ycbcrsubsampling[1] != 4)) || + ((ycbcrsubsampling[0] == 0) || (ycbcrsubsampling[1] == 0))) { TIFFErrorExtR(tif, module, "Invalid YCbCr subsampling"); return 0; @@ -287,7 +299,25 @@ uint64_t TIFFScanlineSize64(TIFF *tif) else { uint64_t scanline_samples; - scanline_samples = _TIFFMultiply64(tif, td->td_imagewidth, + uint32_t scanline_width = td->td_imagewidth; + +#if 0 + // Tries to fix https://gitlab.com/libtiff/libtiff/-/merge_requests/564 + // but causes regression when decoding legit files with tiffcp -c none + // Cf https://gitlab.com/libtiff/libtiff/-/merge_requests/644 + if (td->td_photometric == PHOTOMETRIC_YCBCR) + { + uint16_t subsampling_hor; + uint16_t ignored; + TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, + &subsampling_hor, &ignored); + if (subsampling_hor > 1) // roundup width for YCbCr + scanline_width = + TIFFroundup_32(scanline_width, subsampling_hor); + } +#endif + + scanline_samples = _TIFFMultiply64(tif, scanline_width, td->td_samplesperpixel, module); scanline_size = TIFFhowmany_64(_TIFFMultiply64(tif, scanline_samples, diff --git a/contrib/libs/libtiff/tif_thunder.c b/contrib/libs/libtiff/tif_thunder.c index 1f97362ca39..bac0607dbe5 100644 --- a/contrib/libs/libtiff/tif_thunder.c +++ b/contrib/libs/libtiff/tif_thunder.c @@ -82,13 +82,14 @@ static int ThunderSetupDecode(TIFF *tif) return (1); } -static int ThunderDecode(TIFF *tif, uint8_t *op, tmsize_t maxpixels) +static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) { static const char module[] = "ThunderDecode"; register unsigned char *bp; register tmsize_t cc; unsigned int lastpixel; tmsize_t npixels; + uint8_t *op = op0; bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; @@ -107,6 +108,8 @@ static int ThunderDecode(TIFF *tif, uint8_t *op, tmsize_t maxpixels) * Replicate the last pixel n times, * where n is the lower-order 6 bits. */ + if (n == 0) + break; if (npixels & 1) { op[0] |= lastpixel; @@ -117,11 +120,10 @@ static int ThunderDecode(TIFF *tif, uint8_t *op, tmsize_t maxpixels) else lastpixel |= lastpixel << 4; npixels += n; - if (npixels < maxpixels) - { - for (; n > 0; n -= 2) - *op++ = (uint8_t)lastpixel; - } + if (npixels > maxpixels) + break; + for (; n > 0; n -= 2) + *op++ = (uint8_t)lastpixel; if (n == -1) *--op &= 0xf0; lastpixel &= 0xf; @@ -154,6 +156,8 @@ static int ThunderDecode(TIFF *tif, uint8_t *op, tmsize_t maxpixels) tif->tif_rawcc = cc; if (npixels != maxpixels) { + uint8_t *op_end = op0 + (maxpixels + 1) / 2; + memset(op, 0, (size_t)(op_end - op)); TIFFErrorExtR(tif, module, "%s data at scanline %lu (%" PRIu64 " != %" PRIu64 ")", npixels < maxpixels ? "Not enough" : "Too much", diff --git a/contrib/libs/libtiff/tif_unix.c b/contrib/libs/libtiff/tif_unix.c index 34dd53b98c9..e65da872a3b 100644 --- a/contrib/libs/libtiff/tif_unix.c +++ b/contrib/libs/libtiff/tif_unix.c @@ -27,6 +27,10 @@ * Windows Common RunTime Library. */ +#ifdef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#undef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#endif + #include "tif_config.h" #ifdef HAVE_SYS_TYPES_H @@ -79,6 +83,8 @@ static tmsize_t _tiffReadProc(thandle_t fd, void *buf, tmsize_t size) size_t io_size = bytes_total - bytes_read; if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; + /* Below is an obvious false positive of Coverity Scan */ + /* coverity[overflow_sink] */ count = read(fdh.fd, buf_offset, (TIFFIOSize_t)io_size); if (count <= 0) break; @@ -106,6 +112,8 @@ static tmsize_t _tiffWriteProc(thandle_t fd, void *buf, tmsize_t size) size_t io_size = bytes_total - bytes_written; if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; + /* Below is an obvious false positive of Coverity Scan */ + /* coverity[overflow_sink] */ count = write(fdh.fd, buf_offset, (TIFFIOSize_t)io_size); if (count <= 0) break; @@ -258,7 +266,7 @@ TIFF *TIFFOpenExt(const char *name, const char *mode, TIFFOpenOptions *opts) return tif; } -#ifdef __WIN32__ +#ifdef _WIN32 #include <windows.h> /* * Open a TIFF file with a Unicode filename, for read/writing. diff --git a/contrib/libs/libtiff/tif_webp.c b/contrib/libs/libtiff/tif_webp.c index bf9d77eb9be..ccffef07037 100644 --- a/contrib/libs/libtiff/tif_webp.c +++ b/contrib/libs/libtiff/tif_webp.c @@ -47,7 +47,9 @@ typedef struct { uint16_t nSamples; /* number of samples per pixel */ - int lossless; /* lossy/lossless compression */ + int read_error; /* whether a read error has occurred, and which should cause + further reads in the same strip/tile to be aborted */ + int lossless; /* lossy/lossless compression */ int lossless_exact; /* lossless exact mode. If TRUE, R,G,B values in areas with alpha = 0 will be preserved */ int quality_level; /* compression level */ @@ -133,6 +135,16 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) assert(sp != NULL); assert(sp->state == LSTATE_INIT_DECODE); + if (sp->read_error) + { + memset(op, 0, (size_t)occ); + TIFFErrorExtR(tif, module, + "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " + "previous error", + tif->tif_row); + return 0; + } + if (sp->psDecoder == NULL) { TIFFDirectory *td = &tif->tif_dir; @@ -158,12 +170,16 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) : (uint32_t)tif->tif_rawcc, &webp_width, &webp_height)) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "WebPGetInfo() failed"); return 0; } if ((uint32_t)webp_width != segment_width || (uint32_t)webp_height != segment_height) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR( tif, module, "WebP blob dimension is %dx%d. Expected %ux%u", webp_width, webp_height, segment_width, segment_height); @@ -174,6 +190,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) WebPDecoderConfig config; if (!WebPInitDecoderConfig(&config)) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "WebPInitDecoderConfig() failed"); return 0; } @@ -189,6 +207,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (!bWebPGetFeaturesOK) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "WebPInitDecoderConfig() failed"); return 0; } @@ -202,6 +222,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) */ !(webp_bands == 3 && sp->nSamples == 4)) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "WebP blob band count is %d. Expected %d", webp_bands, sp->nSamples); @@ -228,6 +250,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (!sp->pBuffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); + memset(op, 0, (size_t)occ); + sp->read_error = 1; return 0; } sp->buffer_size = buffer_size; @@ -257,6 +281,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (sp->psDecoder == NULL) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "Unable to allocate WebP decoder."); return 0; } @@ -264,6 +290,10 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (occ % sp->sDecBuffer.u.RGBA.stride) { + // read_error not set here as this is a usage issue that can be + // recovered in a following call. + memset(op, 0, (size_t)occ); + /* Do not set read_error as could potentially be recovered */ TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return 0; } @@ -284,6 +314,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { TIFFErrorExtR(tif, module, "Unrecognized error."); } + memset(op, 0, (size_t)occ); + sp->read_error = 1; return 0; } else @@ -303,6 +335,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { if (current_y != numberOfExpectedLines) { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "Unable to decode WebP data: less lines than " "expected."); @@ -332,6 +366,8 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } else { + memset(op, 0, (size_t)occ); + sp->read_error = 1; TIFFErrorExtR(tif, module, "Unable to decode WebP data."); return 0; } @@ -451,6 +487,8 @@ static int TWebPPreDecode(TIFF *tif, uint16_t s) sp->psDecoder = NULL; } + sp->read_error = 0; + return 1; } diff --git a/contrib/libs/libtiff/tif_win32.c b/contrib/libs/libtiff/tif_win32.c index 1a6b86dffb5..d64ba49f0e4 100644 --- a/contrib/libs/libtiff/tif_win32.c +++ b/contrib/libs/libtiff/tif_win32.c @@ -27,6 +27,10 @@ * Scott Wagner ([email protected]), Itek Graphix, Rochester, NY USA */ +#ifdef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#undef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +#endif + #include "tiffiop.h" #include <stdlib.h> diff --git a/contrib/libs/libtiff/tif_zip.c b/contrib/libs/libtiff/tif_zip.c index 53344f3e9d9..9731dba2639 100644 --- a/contrib/libs/libtiff/tif_zip.c +++ b/contrib/libs/libtiff/tif_zip.c @@ -67,6 +67,8 @@ typedef struct { TIFFPredictorState predict; z_stream stream; + int read_error; /* whether a read error has occurred, and which should cause + further reads in the same strip/tile to be aborted */ int zipquality; /* compression level */ int state; /* state flags */ int subcodec; /* DEFLATE_SUBCODEC_ZLIB or DEFLATE_SUBCODEC_LIBDEFLATE */ @@ -83,9 +85,9 @@ typedef struct TIFFVSetMethod vsetparent; /* super-class method */ } ZIPState; -#define ZState(tif) ((ZIPState *)(tif)->tif_data) -#define DecoderState(tif) ZState(tif) -#define EncoderState(tif) ZState(tif) +#define GetZIPState(tif) ((ZIPState *)(tif)->tif_data) +#define ZIPDecoderState(tif) GetZIPState(tif) +#define ZIPEncoderState(tif) GetZIPState(tif) static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); @@ -99,7 +101,7 @@ static int ZIPFixupTags(TIFF *tif) static int ZIPSetupDecode(TIFF *tif) { static const char module[] = "ZIPSetupDecode"; - ZIPState *sp = DecoderState(tif); + ZIPState *sp = ZIPDecoderState(tif); assert(sp != NULL); @@ -131,7 +133,7 @@ static int ZIPSetupDecode(TIFF *tif) */ static int ZIPPreDecode(TIFF *tif, uint16_t s) { - ZIPState *sp = DecoderState(tif); + ZIPState *sp = ZIPDecoderState(tif); (void)s; assert(sp != NULL); @@ -150,18 +152,33 @@ static int ZIPPreDecode(TIFF *tif, uint16_t s) sp->stream.avail_in = (uint64_t)tif->tif_rawcc < 0xFFFFFFFFU ? (uInt)tif->tif_rawcc : 0xFFFFFFFFU; - return (inflateReset(&sp->stream) == Z_OK); + if (inflateReset(&sp->stream) == Z_OK) + { + sp->read_error = 0; + return 1; + } + return 0; } static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "ZIPDecode"; - ZIPState *sp = DecoderState(tif); + ZIPState *sp = ZIPDecoderState(tif); (void)s; assert(sp != NULL); assert(sp->state == ZSTATE_INIT_DECODE); + if (sp->read_error) + { + memset(op, 0, (size_t)occ); + TIFFErrorExtR(tif, module, + "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " + "previous error", + tif->tif_row); + return 0; + } + #if LIBDEFLATE_SUPPORT if (sp->libdeflate_state == 1) return 0; @@ -227,8 +244,10 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (res != LIBDEFLATE_SUCCESS && res != LIBDEFLATE_INSUFFICIENT_SPACE) { + memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu", (unsigned long)tif->tif_row); + sp->read_error = 1; return 0; } @@ -263,13 +282,17 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) break; if (state == Z_DATA_ERROR) { + memset(sp->stream.next_out, 0, sp->stream.avail_out); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu, %s", (unsigned long)tif->tif_row, SAFE_MSG(sp)); + sp->read_error = 1; return (0); } if (state != Z_OK) { + memset(sp->stream.next_out, 0, sp->stream.avail_out); TIFFErrorExtR(tif, module, "ZLib error: %s", SAFE_MSG(sp)); + sp->read_error = 1; return (0); } } while (occ > 0); @@ -279,6 +302,8 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) "Not enough data at scanline %lu (short %" PRIu64 " bytes)", (unsigned long)tif->tif_row, (uint64_t)occ); + memset(sp->stream.next_out, 0, sp->stream.avail_out); + sp->read_error = 1; return (0); } @@ -290,7 +315,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) static int ZIPSetupEncode(TIFF *tif) { static const char module[] = "ZIPSetupEncode"; - ZIPState *sp = EncoderState(tif); + ZIPState *sp = ZIPEncoderState(tif); int cappedQuality; assert(sp != NULL); @@ -321,7 +346,7 @@ static int ZIPSetupEncode(TIFF *tif) */ static int ZIPPreEncode(TIFF *tif, uint16_t s) { - ZIPState *sp = EncoderState(tif); + ZIPState *sp = ZIPEncoderState(tif); (void)s; assert(sp != NULL); @@ -348,7 +373,7 @@ static int ZIPPreEncode(TIFF *tif, uint16_t s) static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { static const char module[] = "ZIPEncode"; - ZIPState *sp = EncoderState(tif); + ZIPState *sp = ZIPEncoderState(tif); assert(sp != NULL); assert(sp->state == ZSTATE_INIT_ENCODE); @@ -487,7 +512,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) static int ZIPPostEncode(TIFF *tif) { static const char module[] = "ZIPPostEncode"; - ZIPState *sp = EncoderState(tif); + ZIPState *sp = ZIPEncoderState(tif); int state; #if LIBDEFLATE_SUPPORT @@ -526,7 +551,7 @@ static int ZIPPostEncode(TIFF *tif) static void ZIPCleanup(TIFF *tif) { - ZIPState *sp = ZState(tif); + ZIPState *sp = GetZIPState(tif); assert(sp != 0); @@ -562,7 +587,7 @@ static void ZIPCleanup(TIFF *tif) static int ZIPVSetField(TIFF *tif, uint32_t tag, va_list ap) { static const char module[] = "ZIPVSetField"; - ZIPState *sp = ZState(tif); + ZIPState *sp = GetZIPState(tif); switch (tag) { @@ -628,7 +653,7 @@ static int ZIPVSetField(TIFF *tif, uint32_t tag, va_list ap) static int ZIPVGetField(TIFF *tif, uint32_t tag, va_list ap) { - ZIPState *sp = ZState(tif); + ZIPState *sp = GetZIPState(tif); switch (tag) { @@ -680,7 +705,7 @@ int TIFFInitZIP(TIFF *tif, int scheme) tif->tif_data = (uint8_t *)_TIFFcallocExt(tif, sizeof(ZIPState), 1); if (tif->tif_data == NULL) goto bad; - sp = ZState(tif); + sp = GetZIPState(tif); sp->stream.zalloc = NULL; sp->stream.zfree = NULL; sp->stream.opaque = NULL; diff --git a/contrib/libs/libtiff/tif_zstd.c b/contrib/libs/libtiff/tif_zstd.c index 646993103d2..fc73ce9cf53 100644 --- a/contrib/libs/libtiff/tif_zstd.c +++ b/contrib/libs/libtiff/tif_zstd.c @@ -54,9 +54,9 @@ typedef struct TIFFVSetMethod vsetparent; /* super-class method */ } ZSTDState; -#define LState(tif) ((ZSTDState *)(tif)->tif_data) -#define DecoderState(tif) LState(tif) -#define EncoderState(tif) LState(tif) +#define GetZSTDState(tif) ((ZSTDState *)(tif)->tif_data) +#define ZSTDDecoderState(tif) GetZSTDState(tif) +#define ZSTDEncoderState(tif) GetZSTDState(tif) static int ZSTDEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); @@ -69,7 +69,7 @@ static int ZSTDFixupTags(TIFF *tif) static int ZSTDSetupDecode(TIFF *tif) { - ZSTDState *sp = DecoderState(tif); + ZSTDState *sp = ZSTDDecoderState(tif); assert(sp != NULL); @@ -91,7 +91,7 @@ static int ZSTDSetupDecode(TIFF *tif) static int ZSTDPreDecode(TIFF *tif, uint16_t s) { static const char module[] = "ZSTDPreDecode"; - ZSTDState *sp = DecoderState(tif); + ZSTDState *sp = ZSTDDecoderState(tif); size_t zstd_ret; (void)s; @@ -124,7 +124,7 @@ static int ZSTDPreDecode(TIFF *tif, uint16_t s) static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "ZSTDDecode"; - ZSTDState *sp = DecoderState(tif); + ZSTDState *sp = ZSTDDecoderState(tif); ZSTD_inBuffer in_buffer; ZSTD_outBuffer out_buffer; size_t zstd_ret; @@ -146,6 +146,7 @@ static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) zstd_ret = ZSTD_decompressStream(sp->dstream, &out_buffer, &in_buffer); if (ZSTD_isError(zstd_ret)) { + memset(op + out_buffer.pos, 0, out_buffer.size - out_buffer.pos); TIFFErrorExtR(tif, module, "Error in ZSTD_decompressStream(): %s", ZSTD_getErrorName(zstd_ret)); return 0; @@ -155,6 +156,7 @@ static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (out_buffer.pos < (size_t)occ) { + memset(op + out_buffer.pos, 0, out_buffer.size - out_buffer.pos); TIFFErrorExtR(tif, module, "Not enough data at scanline %lu (short %lu bytes)", (unsigned long)tif->tif_row, @@ -170,7 +172,7 @@ static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) static int ZSTDSetupEncode(TIFF *tif) { - ZSTDState *sp = EncoderState(tif); + ZSTDState *sp = ZSTDEncoderState(tif); assert(sp != NULL); if (sp->state & LSTATE_INIT_DECODE) @@ -190,7 +192,7 @@ static int ZSTDSetupEncode(TIFF *tif) static int ZSTDPreEncode(TIFF *tif, uint16_t s) { static const char module[] = "ZSTDPreEncode"; - ZSTDState *sp = EncoderState(tif); + ZSTDState *sp = ZSTDEncoderState(tif); size_t zstd_ret; (void)s; @@ -229,7 +231,7 @@ static int ZSTDPreEncode(TIFF *tif, uint16_t s) static int ZSTDEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { static const char module[] = "ZSTDEncode"; - ZSTDState *sp = EncoderState(tif); + ZSTDState *sp = ZSTDEncoderState(tif); ZSTD_inBuffer in_buffer; size_t zstd_ret; @@ -271,7 +273,7 @@ static int ZSTDEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) static int ZSTDPostEncode(TIFF *tif) { static const char module[] = "ZSTDPostEncode"; - ZSTDState *sp = EncoderState(tif); + ZSTDState *sp = ZSTDEncoderState(tif); size_t zstd_ret; do @@ -297,7 +299,7 @@ static int ZSTDPostEncode(TIFF *tif) static void ZSTDCleanup(TIFF *tif) { - ZSTDState *sp = LState(tif); + ZSTDState *sp = GetZSTDState(tif); assert(sp != 0); @@ -325,7 +327,7 @@ static void ZSTDCleanup(TIFF *tif) static int ZSTDVSetField(TIFF *tif, uint32_t tag, va_list ap) { static const char module[] = "ZSTDVSetField"; - ZSTDState *sp = LState(tif); + ZSTDState *sp = GetZSTDState(tif); switch (tag) { @@ -347,7 +349,7 @@ static int ZSTDVSetField(TIFF *tif, uint32_t tag, va_list ap) static int ZSTDVGetField(TIFF *tif, uint32_t tag, va_list ap) { - ZSTDState *sp = LState(tif); + ZSTDState *sp = GetZSTDState(tif); switch (tag) { @@ -389,7 +391,7 @@ int TIFFInitZSTD(TIFF *tif, int scheme) tif->tif_data = (uint8_t *)_TIFFmallocExt(tif, sizeof(ZSTDState)); if (tif->tif_data == NULL) goto bad; - sp = LState(tif); + sp = GetZSTDState(tif); /* * Override parent get/set field methods. diff --git a/contrib/libs/libtiff/tiffio.h b/contrib/libs/libtiff/tiffio.h index 20460542f66..225f3c1bfea 100644 --- a/contrib/libs/libtiff/tiffio.h +++ b/contrib/libs/libtiff/tiffio.h @@ -77,10 +77,6 @@ typedef tstrile_t ttile_t; /* tile number */ typedef tmsize_t tsize_t; /* i/o size in bytes */ typedef void *tdata_t; /* image data ref */ -#if !defined(__WIN32__) && (defined(_WIN32) || defined(WIN32)) -#define __WIN32__ -#endif - /* * On windows you should define USE_WIN32_FILEIO if you are using tif_win32.c * or AVOID_WIN32_FILEIO if you are using something else (like tif_unix.c). @@ -88,7 +84,7 @@ typedef void *tdata_t; /* image data ref */ * By default tif_unix.c is assumed. */ -#if defined(_WINDOWS) || defined(__WIN32__) || defined(_Windows) +#if defined(_WIN32) #if !defined(__CYGWIN) && !defined(AVOID_WIN32_FILEIO) && \ !defined(USE_WIN32_FILEIO) #define AVOID_WIN32_FILEIO @@ -98,11 +94,11 @@ typedef void *tdata_t; /* image data ref */ #if defined(USE_WIN32_FILEIO) #define VC_EXTRALEAN #include <windows.h> -#ifdef __WIN32__ +#ifdef _WIN32 DECLARE_HANDLE(thandle_t); /* Win32 file handle */ #else typedef HFILE thandle_t; /* client data handle */ -#endif /* __WIN32__ */ +#endif /* _WIN32 */ #else typedef void *thandle_t; /* client data handle */ #endif /* USE_WIN32_FILEIO */ @@ -311,14 +307,15 @@ extern "C" /* * Auxiliary functions. */ - +#ifndef TIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS extern void *_TIFFmalloc(tmsize_t s); extern void *_TIFFcalloc(tmsize_t nmemb, tmsize_t siz); extern void *_TIFFrealloc(void *p, tmsize_t s); + extern void _TIFFfree(void *p); +#endif extern void _TIFFmemset(void *p, int v, tmsize_t c); extern void _TIFFmemcpy(void *d, const void *s, tmsize_t c); extern int _TIFFmemcmp(const void *p1, const void *p2, tmsize_t c); - extern void _TIFFfree(void *p); /* ** Stuff, related to tag handling and creating custom tags. @@ -508,6 +505,9 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, TIFFOpenOptionsSetMaxSingleMemAlloc(TIFFOpenOptions *opts, tmsize_t max_single_mem_alloc); extern void + TIFFOpenOptionsSetMaxCumulatedMemAlloc(TIFFOpenOptions *opts, + tmsize_t max_cumulated_mem_alloc); + extern void TIFFOpenOptionsSetErrorHandlerExtR(TIFFOpenOptions *opts, TIFFErrorHandlerExtR handler, void *errorhandler_user_data); @@ -518,11 +518,11 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, extern TIFF *TIFFOpen(const char *, const char *); extern TIFF *TIFFOpenExt(const char *, const char *, TIFFOpenOptions *opts); -#ifdef __WIN32__ +#ifdef _WIN32 extern TIFF *TIFFOpenW(const wchar_t *, const char *); extern TIFF *TIFFOpenWExt(const wchar_t *, const char *, TIFFOpenOptions *opts); -#endif /* __WIN32__ */ +#endif /* _WIN32 */ extern TIFF *TIFFFdOpen(int, const char *, const char *); extern TIFF *TIFFFdOpenExt(int, const char *, const char *, TIFFOpenOptions *opts); diff --git a/contrib/libs/libtiff/tiffiop.h b/contrib/libs/libtiff/tiffiop.h index fbf7b0700bf..c4348206df5 100644 --- a/contrib/libs/libtiff/tiffiop.h +++ b/contrib/libs/libtiff/tiffiop.h @@ -102,6 +102,13 @@ struct TIFFOffsetAndDirNumber }; typedef struct TIFFOffsetAndDirNumber TIFFOffsetAndDirNumber; +typedef union +{ + TIFFHeaderCommon common; + TIFFHeaderClassic classic; + TIFFHeaderBig big; +} TIFFHeaderUnion; + struct tiff { char *tif_name; /* name of open file */ @@ -153,20 +160,35 @@ struct tiff TIFFDirectory tif_dir; /* internal rep of current directory */ TIFFDirectory tif_customdir; /* custom IFDs are separated from the main ones */ - union - { - TIFFHeaderCommon common; - TIFFHeaderClassic classic; - TIFFHeaderBig big; - } tif_header; - uint16_t tif_header_size; /* file's header block and its length */ - uint32_t tif_row; /* current scanline */ - tdir_t tif_curdir; /* current directory (index) */ + TIFFHeaderUnion tif_header; /* file's header block Classic/BigTIFF union */ + uint16_t tif_header_size; /* file's header block and its length */ + uint32_t tif_row; /* current scanline */ + + /* There are IFDs in the file and an "active" IFD in memory, + * from which fields are "set" and "get". + * tif_curdir is set to: + * a) TIFF_NON_EXISTENT_DIR_NUMBER if there is no IFD in the file + * or the state is unknown, + * or the last read (i.e. TIFFFetchDirectory()) failed, + * or a custom directory was written. + * b) IFD index of last IFD written in the file. In this case the + * active IFD is a new (empty) one and tif_diroff is zero. + * If writing fails, tif_curdir is not changed. + * c) IFD index of IFD read from file into memory (=active IFD), + * even if IFD is corrupt and TIFFReadDirectory() returns 0. + * Then tif_diroff contains the offset of the IFD in the file. + * d) IFD index 0, whenever a custom directory or an unchained SubIFD + * was read. */ + tdir_t tif_curdir; /* current directory (index) */ + /* tif_curdircount: number of directories (main-IFDs) in file: + * - TIFF_NON_EXISTENT_DIR_NUMBER means 'dont know number of IFDs'. + * - 0 means 'empty file opened for writing, but no IFD written yet' */ + tdir_t tif_curdircount; uint32_t tif_curstrip; /* current strip for read/write */ uint64_t tif_curoff; /* current offset for read/write */ uint64_t tif_lastvalidoff; /* last valid offset allowed for rewrite in place. Used only by TIFFAppendToStrip() */ - uint64_t tif_dataoff; /* current offset for writing dir */ + uint64_t tif_dataoff; /* current offset for writing dir (IFD) */ /* SubIFD support */ uint16_t tif_nsubifd; /* remaining subifds to write */ uint64_t tif_subifdoff; /* offset for patching SubIFD link */ @@ -233,7 +255,9 @@ struct tiff void *tif_errorhandler_user_data; TIFFErrorHandlerExtR tif_warnhandler; void *tif_warnhandler_user_data; - tmsize_t tif_max_single_mem_alloc; /* in bytes. 0 for unlimited */ + tmsize_t tif_max_single_mem_alloc; /* in bytes. 0 for unlimited */ + tmsize_t tif_max_cumulated_mem_alloc; /* in bytes. 0 for unlimited */ + tmsize_t tif_cur_cumulated_mem_alloc; /* in bytes */ }; struct TIFFOpenOptions @@ -243,6 +267,7 @@ struct TIFFOpenOptions TIFFErrorHandlerExtR warnhandler; /* may be NULL */ void *warnhandler_user_data; /* may be NULL */ tmsize_t max_single_mem_alloc; /* in bytes. 0 for unlimited */ + tmsize_t max_cumulated_mem_alloc; /* in bytes. 0 for unlimited */ }; #define isPseudoTag(t) (t > 0xffff) /* is tag value normal or pseudo */ @@ -331,7 +356,7 @@ struct TIFFOpenOptions #define ftell(stream, offset, whence) ftello(stream, offset, whence) #endif #endif -#if defined(__WIN32__) && !(defined(_MSC_VER) && _MSC_VER < 1400) && \ +#if defined(_WIN32) && \ !(defined(__MSVCRT_VERSION__) && __MSVCRT_VERSION__ < 0x800) typedef unsigned int TIFFIOSize_t; #define _TIFF_lseek_f(fildes, offset, whence) \ @@ -437,9 +462,6 @@ extern "C" extern void *_TIFFCheckRealloc(TIFF *, void *, tmsize_t, tmsize_t, const char *); - extern double _TIFFUInt64ToDouble(uint64_t); - extern float _TIFFUInt64ToFloat(uint64_t); - extern float _TIFFClampDoubleToFloat(double); extern uint32_t _TIFFClampDoubleToUInt32(double); diff --git a/contrib/libs/libtiff/tiffvers.h b/contrib/libs/libtiff/tiffvers.h index 4630580d478..9e886f133b2 100644 --- a/contrib/libs/libtiff/tiffvers.h +++ b/contrib/libs/libtiff/tiffvers.h @@ -8,7 +8,7 @@ * Furthermore, configure_file variables of type "" are * modified by clang-format and won't be substituted by CMake. */ -#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.6.0\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." +#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.7.0\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." /* * This define can be used in code that requires * compilation-related definitions specific to a @@ -16,13 +16,13 @@ * version checking should be done based on the * string returned by TIFFGetVersion. */ -#define TIFFLIB_VERSION 20230908 +#define TIFFLIB_VERSION 20240911 /* The following defines have been added in 4.5.0 */ #define TIFFLIB_MAJOR_VERSION 4 -#define TIFFLIB_MINOR_VERSION 6 +#define TIFFLIB_MINOR_VERSION 7 #define TIFFLIB_MICRO_VERSION 0 -#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.6.0" +#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.7.0" /* Macro added in 4.5.0. Returns TRUE if the current libtiff version is * greater or equal to major.minor.micro diff --git a/contrib/libs/libtiff/ya.make b/contrib/libs/libtiff/ya.make index e06d9c60b3a..ef90f0d9373 100644 --- a/contrib/libs/libtiff/ya.make +++ b/contrib/libs/libtiff/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(4.6.0) +VERSION(4.7.0) -ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.6.0) +ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.7.0) PEERDIR( contrib/libs/libjpeg-turbo @@ -34,6 +34,11 @@ NO_COMPILER_WARNINGS() NO_UTIL() +CFLAGS( + -DALLOW_TIFF_NON_EXT_ALLOC_FUNCTIONS + -DTIFF_DO_NOT_USE_NON_EXT_ALLOC_FUNCTIONS +) + SRCS( tif_aux.c tif_close.c diff --git a/contrib/python/anyio/.dist-info/METADATA b/contrib/python/anyio/.dist-info/METADATA deleted file mode 100644 index 7b114cdbe89..00000000000 --- a/contrib/python/anyio/.dist-info/METADATA +++ /dev/null @@ -1,104 +0,0 @@ -Metadata-Version: 2.1 -Name: anyio -Version: 4.8.0 -Summary: High level compatibility layer for multiple asynchronous event loop implementations -Author-email: Alex Grönholm <[email protected]> -License: MIT -Project-URL: Documentation, https://anyio.readthedocs.io/en/latest/ -Project-URL: Changelog, https://anyio.readthedocs.io/en/stable/versionhistory.html -Project-URL: Source code, https://github.com/agronholm/anyio -Project-URL: Issue tracker, https://github.com/agronholm/anyio/issues -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Framework :: AnyIO -Classifier: Typing :: Typed -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Requires-Python: >=3.9 -Description-Content-Type: text/x-rst -License-File: LICENSE -Requires-Dist: exceptiongroup>=1.0.2; python_version < "3.11" -Requires-Dist: idna>=2.8 -Requires-Dist: sniffio>=1.1 -Requires-Dist: typing_extensions>=4.5; python_version < "3.13" -Provides-Extra: trio -Requires-Dist: trio>=0.26.1; extra == "trio" -Provides-Extra: test -Requires-Dist: anyio[trio]; extra == "test" -Requires-Dist: coverage[toml]>=7; extra == "test" -Requires-Dist: exceptiongroup>=1.2.0; extra == "test" -Requires-Dist: hypothesis>=4.0; extra == "test" -Requires-Dist: psutil>=5.9; extra == "test" -Requires-Dist: pytest>=7.0; extra == "test" -Requires-Dist: trustme; extra == "test" -Requires-Dist: truststore>=0.9.1; python_version >= "3.10" and extra == "test" -Requires-Dist: uvloop>=0.21; (platform_python_implementation == "CPython" and platform_system != "Windows" and python_version < "3.14") and extra == "test" -Provides-Extra: doc -Requires-Dist: packaging; extra == "doc" -Requires-Dist: Sphinx~=7.4; extra == "doc" -Requires-Dist: sphinx_rtd_theme; extra == "doc" -Requires-Dist: sphinx-autodoc-typehints>=1.2.0; extra == "doc" - -.. image:: https://github.com/agronholm/anyio/actions/workflows/test.yml/badge.svg - :target: https://github.com/agronholm/anyio/actions/workflows/test.yml - :alt: Build Status -.. image:: https://coveralls.io/repos/github/agronholm/anyio/badge.svg?branch=master - :target: https://coveralls.io/github/agronholm/anyio?branch=master - :alt: Code Coverage -.. image:: https://readthedocs.org/projects/anyio/badge/?version=latest - :target: https://anyio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation -.. image:: https://badges.gitter.im/gitterHQ/gitter.svg - :target: https://gitter.im/python-trio/AnyIO - :alt: Gitter chat - -AnyIO is an asynchronous networking and concurrency library that works on top of either asyncio_ or -trio_. It implements trio-like `structured concurrency`_ (SC) on top of asyncio and works in harmony -with the native SC of trio itself. - -Applications and libraries written against AnyIO's API will run unmodified on either asyncio_ or -trio_. AnyIO can also be adopted into a library or application incrementally – bit by bit, no full -refactoring necessary. It will blend in with the native libraries of your chosen backend. - -Documentation -------------- - -View full documentation at: https://anyio.readthedocs.io/ - -Features --------- - -AnyIO offers the following functionality: - -* Task groups (nurseries_ in trio terminology) -* High-level networking (TCP, UDP and UNIX sockets) - - * `Happy eyeballs`_ algorithm for TCP connections (more robust than that of asyncio on Python - 3.8) - * async/await style UDP sockets (unlike asyncio where you still have to use Transports and - Protocols) - -* A versatile API for byte streams and object streams -* Inter-task synchronization and communication (locks, conditions, events, semaphores, object - streams) -* Worker threads -* Subprocesses -* Asynchronous file I/O (using worker threads) -* Signal handling - -AnyIO also comes with its own pytest_ plugin which also supports asynchronous fixtures. -It even works with the popular Hypothesis_ library. - -.. _asyncio: https://docs.python.org/3/library/asyncio.html -.. _trio: https://github.com/python-trio/trio -.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency -.. _nurseries: https://trio.readthedocs.io/en/stable/reference-core.html#nurseries-and-spawning -.. _Happy eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs -.. _pytest: https://docs.pytest.org/en/latest/ -.. _Hypothesis: https://hypothesis.works/ diff --git a/contrib/python/anyio/.dist-info/entry_points.txt b/contrib/python/anyio/.dist-info/entry_points.txt deleted file mode 100644 index 44dd9bdc303..00000000000 --- a/contrib/python/anyio/.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[pytest11] -anyio = anyio.pytest_plugin diff --git a/contrib/python/anyio/.dist-info/top_level.txt b/contrib/python/anyio/.dist-info/top_level.txt deleted file mode 100644 index c77c069ecc9..00000000000 --- a/contrib/python/anyio/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -anyio diff --git a/contrib/python/anyio/LICENSE b/contrib/python/anyio/LICENSE deleted file mode 100644 index 104eebf5a30..00000000000 --- a/contrib/python/anyio/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Alex Grönholm - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/anyio/README.rst b/contrib/python/anyio/README.rst deleted file mode 100644 index 35afc7e312d..00000000000 --- a/contrib/python/anyio/README.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. image:: https://github.com/agronholm/anyio/actions/workflows/test.yml/badge.svg - :target: https://github.com/agronholm/anyio/actions/workflows/test.yml - :alt: Build Status -.. image:: https://coveralls.io/repos/github/agronholm/anyio/badge.svg?branch=master - :target: https://coveralls.io/github/agronholm/anyio?branch=master - :alt: Code Coverage -.. image:: https://readthedocs.org/projects/anyio/badge/?version=latest - :target: https://anyio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation -.. image:: https://badges.gitter.im/gitterHQ/gitter.svg - :target: https://gitter.im/python-trio/AnyIO - :alt: Gitter chat - -AnyIO is an asynchronous networking and concurrency library that works on top of either asyncio_ or -trio_. It implements trio-like `structured concurrency`_ (SC) on top of asyncio and works in harmony -with the native SC of trio itself. - -Applications and libraries written against AnyIO's API will run unmodified on either asyncio_ or -trio_. AnyIO can also be adopted into a library or application incrementally – bit by bit, no full -refactoring necessary. It will blend in with the native libraries of your chosen backend. - -Documentation -------------- - -View full documentation at: https://anyio.readthedocs.io/ - -Features --------- - -AnyIO offers the following functionality: - -* Task groups (nurseries_ in trio terminology) -* High-level networking (TCP, UDP and UNIX sockets) - - * `Happy eyeballs`_ algorithm for TCP connections (more robust than that of asyncio on Python - 3.8) - * async/await style UDP sockets (unlike asyncio where you still have to use Transports and - Protocols) - -* A versatile API for byte streams and object streams -* Inter-task synchronization and communication (locks, conditions, events, semaphores, object - streams) -* Worker threads -* Subprocesses -* Asynchronous file I/O (using worker threads) -* Signal handling - -AnyIO also comes with its own pytest_ plugin which also supports asynchronous fixtures. -It even works with the popular Hypothesis_ library. - -.. _asyncio: https://docs.python.org/3/library/asyncio.html -.. _trio: https://github.com/python-trio/trio -.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency -.. _nurseries: https://trio.readthedocs.io/en/stable/reference-core.html#nurseries-and-spawning -.. _Happy eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs -.. _pytest: https://docs.pytest.org/en/latest/ -.. _Hypothesis: https://hypothesis.works/ diff --git a/contrib/python/anyio/anyio/__init__.py b/contrib/python/anyio/anyio/__init__.py deleted file mode 100644 index 098312599f6..00000000000 --- a/contrib/python/anyio/anyio/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -from ._core._eventloop import current_time as current_time -from ._core._eventloop import get_all_backends as get_all_backends -from ._core._eventloop import get_cancelled_exc_class as get_cancelled_exc_class -from ._core._eventloop import run as run -from ._core._eventloop import sleep as sleep -from ._core._eventloop import sleep_forever as sleep_forever -from ._core._eventloop import sleep_until as sleep_until -from ._core._exceptions import BrokenResourceError as BrokenResourceError -from ._core._exceptions import BrokenWorkerIntepreter as BrokenWorkerIntepreter -from ._core._exceptions import BrokenWorkerProcess as BrokenWorkerProcess -from ._core._exceptions import BusyResourceError as BusyResourceError -from ._core._exceptions import ClosedResourceError as ClosedResourceError -from ._core._exceptions import DelimiterNotFound as DelimiterNotFound -from ._core._exceptions import EndOfStream as EndOfStream -from ._core._exceptions import IncompleteRead as IncompleteRead -from ._core._exceptions import TypedAttributeLookupError as TypedAttributeLookupError -from ._core._exceptions import WouldBlock as WouldBlock -from ._core._fileio import AsyncFile as AsyncFile -from ._core._fileio import Path as Path -from ._core._fileio import open_file as open_file -from ._core._fileio import wrap_file as wrap_file -from ._core._resources import aclose_forcefully as aclose_forcefully -from ._core._signals import open_signal_receiver as open_signal_receiver -from ._core._sockets import connect_tcp as connect_tcp -from ._core._sockets import connect_unix as connect_unix -from ._core._sockets import create_connected_udp_socket as create_connected_udp_socket -from ._core._sockets import ( - create_connected_unix_datagram_socket as create_connected_unix_datagram_socket, -) -from ._core._sockets import create_tcp_listener as create_tcp_listener -from ._core._sockets import create_udp_socket as create_udp_socket -from ._core._sockets import create_unix_datagram_socket as create_unix_datagram_socket -from ._core._sockets import create_unix_listener as create_unix_listener -from ._core._sockets import getaddrinfo as getaddrinfo -from ._core._sockets import getnameinfo as getnameinfo -from ._core._sockets import wait_readable as wait_readable -from ._core._sockets import wait_socket_readable as wait_socket_readable -from ._core._sockets import wait_socket_writable as wait_socket_writable -from ._core._sockets import wait_writable as wait_writable -from ._core._streams import create_memory_object_stream as create_memory_object_stream -from ._core._subprocesses import open_process as open_process -from ._core._subprocesses import run_process as run_process -from ._core._synchronization import CapacityLimiter as CapacityLimiter -from ._core._synchronization import ( - CapacityLimiterStatistics as CapacityLimiterStatistics, -) -from ._core._synchronization import Condition as Condition -from ._core._synchronization import ConditionStatistics as ConditionStatistics -from ._core._synchronization import Event as Event -from ._core._synchronization import EventStatistics as EventStatistics -from ._core._synchronization import Lock as Lock -from ._core._synchronization import LockStatistics as LockStatistics -from ._core._synchronization import ResourceGuard as ResourceGuard -from ._core._synchronization import Semaphore as Semaphore -from ._core._synchronization import SemaphoreStatistics as SemaphoreStatistics -from ._core._tasks import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED -from ._core._tasks import CancelScope as CancelScope -from ._core._tasks import create_task_group as create_task_group -from ._core._tasks import current_effective_deadline as current_effective_deadline -from ._core._tasks import fail_after as fail_after -from ._core._tasks import move_on_after as move_on_after -from ._core._testing import TaskInfo as TaskInfo -from ._core._testing import get_current_task as get_current_task -from ._core._testing import get_running_tasks as get_running_tasks -from ._core._testing import wait_all_tasks_blocked as wait_all_tasks_blocked -from ._core._typedattr import TypedAttributeProvider as TypedAttributeProvider -from ._core._typedattr import TypedAttributeSet as TypedAttributeSet -from ._core._typedattr import typed_attribute as typed_attribute - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio."): - __value.__module__ = __name__ - -del __value diff --git a/contrib/python/anyio/anyio/_backends/__init__.py b/contrib/python/anyio/anyio/_backends/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/anyio/anyio/_backends/__init__.py +++ /dev/null diff --git a/contrib/python/anyio/anyio/_backends/_asyncio.py b/contrib/python/anyio/anyio/_backends/_asyncio.py deleted file mode 100644 index 76a400c1cb8..00000000000 --- a/contrib/python/anyio/anyio/_backends/_asyncio.py +++ /dev/null @@ -1,2807 +0,0 @@ -from __future__ import annotations - -import array -import asyncio -import concurrent.futures -import math -import os -import socket -import sys -import threading -import weakref -from asyncio import ( - AbstractEventLoop, - CancelledError, - all_tasks, - create_task, - current_task, - get_running_loop, - sleep, -) -from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined] -from collections import OrderedDict, deque -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from concurrent.futures import Future -from contextlib import AbstractContextManager, suppress -from contextvars import Context, copy_context -from dataclasses import dataclass -from functools import partial, wraps -from inspect import ( - CORO_RUNNING, - CORO_SUSPENDED, - getcoroutinestate, - iscoroutine, -) -from io import IOBase -from os import PathLike -from queue import Queue -from signal import Signals -from socket import AddressFamily, SocketKind -from threading import Thread -from types import CodeType, TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Optional, - TypeVar, - cast, -) -from weakref import WeakKeyDictionary - -import sniffio - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - TaskInfo, - abc, -) -from .._core._eventloop import claim_worker_thread, threadlocals -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, - WouldBlock, - iterate_exceptions, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from ..abc import ( - AsyncBackend, - IPSockAddrType, - SocketListener, - UDPPacketType, - UNIXDatagramPacketType, -) -from ..abc._eventloop import StrOrBytesPath -from ..lowlevel import RunVar -from ..streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info >= (3, 10): - from typing import ParamSpec -else: - from typing_extensions import ParamSpec - -if sys.version_info >= (3, 11): - from asyncio import Runner - from typing import TypeVarTuple, Unpack -else: - import contextvars - import enum - import signal - from asyncio import coroutines, events, exceptions, tasks - - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - - class _State(enum.Enum): - CREATED = "created" - INITIALIZED = "initialized" - CLOSED = "closed" - - class Runner: - # Copied from CPython 3.11 - def __init__( - self, - *, - debug: bool | None = None, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ): - self._state = _State.CREATED - self._debug = debug - self._loop_factory = loop_factory - self._loop: AbstractEventLoop | None = None - self._context = None - self._interrupt_count = 0 - self._set_event_loop = False - - def __enter__(self) -> Runner: - self._lazy_init() - return self - - def __exit__( - self, - exc_type: type[BaseException], - exc_val: BaseException, - exc_tb: TracebackType, - ) -> None: - self.close() - - def close(self) -> None: - """Shutdown and close event loop.""" - if self._state is not _State.INITIALIZED: - return - try: - loop = self._loop - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - if hasattr(loop, "shutdown_default_executor"): - loop.run_until_complete(loop.shutdown_default_executor()) - else: - loop.run_until_complete(_shutdown_default_executor(loop)) - finally: - if self._set_event_loop: - events.set_event_loop(None) - loop.close() - self._loop = None - self._state = _State.CLOSED - - def get_loop(self) -> AbstractEventLoop: - """Return embedded event loop.""" - self._lazy_init() - return self._loop - - def run(self, coro: Coroutine[T_Retval], *, context=None) -> T_Retval: - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError(f"a coroutine was expected, got {coro!r}") - - if events._get_running_loop() is not None: - # fail fast with short traceback - raise RuntimeError( - "Runner.run() cannot be called from a running event loop" - ) - - self._lazy_init() - - if context is None: - context = self._context - task = context.run(self._loop.create_task, coro) - - if ( - threading.current_thread() is threading.main_thread() - and signal.getsignal(signal.SIGINT) is signal.default_int_handler - ): - sigint_handler = partial(self._on_sigint, main_task=task) - try: - signal.signal(signal.SIGINT, sigint_handler) - except ValueError: - # `signal.signal` may throw if `threading.main_thread` does - # not support signals (e.g. embedded interpreter with signals - # not registered - see gh-91880) - sigint_handler = None - else: - sigint_handler = None - - self._interrupt_count = 0 - try: - return self._loop.run_until_complete(task) - except exceptions.CancelledError: - if self._interrupt_count > 0: - uncancel = getattr(task, "uncancel", None) - if uncancel is not None and uncancel() == 0: - raise KeyboardInterrupt() - raise # CancelledError - finally: - if ( - sigint_handler is not None - and signal.getsignal(signal.SIGINT) is sigint_handler - ): - signal.signal(signal.SIGINT, signal.default_int_handler) - - def _lazy_init(self) -> None: - if self._state is _State.CLOSED: - raise RuntimeError("Runner is closed") - if self._state is _State.INITIALIZED: - return - if self._loop_factory is None: - self._loop = events.new_event_loop() - if not self._set_event_loop: - # Call set_event_loop only once to avoid calling - # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) - self._set_event_loop = True - else: - self._loop = self._loop_factory() - if self._debug is not None: - self._loop.set_debug(self._debug) - self._context = contextvars.copy_context() - self._state = _State.INITIALIZED - - def _on_sigint(self, signum, frame, main_task: asyncio.Task) -> None: - self._interrupt_count += 1 - if self._interrupt_count == 1 and not main_task.done(): - main_task.cancel() - # wakeup loop if it is blocked by select() with long timeout - self._loop.call_soon_threadsafe(lambda: None) - return - raise KeyboardInterrupt() - - def _cancel_all_tasks(loop: AbstractEventLoop) -> None: - to_cancel = tasks.all_tasks(loop) - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - - loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) - - for task in to_cancel: - if task.cancelled(): - continue - if task.exception() is not None: - loop.call_exception_handler( - { - "message": "unhandled exception during asyncio.run() shutdown", - "exception": task.exception(), - "task": task, - } - ) - - async def _shutdown_default_executor(loop: AbstractEventLoop) -> None: - """Schedule the shutdown of the default executor.""" - - def _do_shutdown(future: asyncio.futures.Future) -> None: - try: - loop._default_executor.shutdown(wait=True) # type: ignore[attr-defined] - loop.call_soon_threadsafe(future.set_result, None) - except Exception as ex: - loop.call_soon_threadsafe(future.set_exception, ex) - - loop._executor_shutdown_called = True - if loop._default_executor is None: - return - future = loop.create_future() - thread = threading.Thread(target=_do_shutdown, args=(future,)) - thread.start() - try: - await future - finally: - thread.join() - - -T_Retval = TypeVar("T_Retval") -T_contra = TypeVar("T_contra", contravariant=True) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - -_root_task: RunVar[asyncio.Task | None] = RunVar("_root_task") - - -def find_root_task() -> asyncio.Task: - root_task = _root_task.get(None) - if root_task is not None and not root_task.done(): - return root_task - - # Look for a task that has been started via run_until_complete() - for task in all_tasks(): - if task._callbacks and not task.done(): - callbacks = [cb for cb, context in task._callbacks] - for cb in callbacks: - if ( - cb is _run_until_complete_cb - or getattr(cb, "__module__", None) == "uvloop.loop" - ): - _root_task.set(task) - return task - - # Look up the topmost task in the AnyIO task tree, if possible - task = cast(asyncio.Task, current_task()) - state = _task_states.get(task) - if state: - cancel_scope = state.cancel_scope - while cancel_scope and cancel_scope._parent_scope is not None: - cancel_scope = cancel_scope._parent_scope - - if cancel_scope is not None: - return cast(asyncio.Task, cancel_scope._host_task) - - return task - - -def get_callable_name(func: Callable) -> str: - module = getattr(func, "__module__", None) - qualname = getattr(func, "__qualname__", None) - return ".".join([x for x in (module, qualname) if x]) - - -# -# Event loop -# - -_run_vars: WeakKeyDictionary[asyncio.AbstractEventLoop, Any] = WeakKeyDictionary() - - -def _task_started(task: asyncio.Task) -> bool: - """Return ``True`` if the task has been started and has not finished.""" - # The task coro should never be None here, as we never add finished tasks to the - # task list - coro = task.get_coro() - assert coro is not None - try: - return getcoroutinestate(coro) in (CORO_RUNNING, CORO_SUSPENDED) - except AttributeError: - # task coro is async_genenerator_asend https://bugs.python.org/issue37771 - raise Exception(f"Cannot determine if task {task} has started or not") from None - - -# -# Timeouts and cancellation -# - - -def is_anyio_cancellation(exc: CancelledError) -> bool: - # Sometimes third party frameworks catch a CancelledError and raise a new one, so as - # a workaround we have to look at the previous ones in __context__ too for a - # matching cancel message - while True: - if ( - exc.args - and isinstance(exc.args[0], str) - and exc.args[0].startswith("Cancelled by cancel scope ") - ): - return True - - if isinstance(exc.__context__, CancelledError): - exc = exc.__context__ - continue - - return False - - -class CancelScope(BaseCancelScope): - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, deadline: float = math.inf, shield: bool = False): - self._deadline = deadline - self._shield = shield - self._parent_scope: CancelScope | None = None - self._child_scopes: set[CancelScope] = set() - self._cancel_called = False - self._cancelled_caught = False - self._active = False - self._timeout_handle: asyncio.TimerHandle | None = None - self._cancel_handle: asyncio.Handle | None = None - self._tasks: set[asyncio.Task] = set() - self._host_task: asyncio.Task | None = None - if sys.version_info >= (3, 11): - self._pending_uncancellations: int | None = 0 - else: - self._pending_uncancellations = None - - def __enter__(self) -> CancelScope: - if self._active: - raise RuntimeError( - "Each CancelScope may only be used for a single 'with' block" - ) - - self._host_task = host_task = cast(asyncio.Task, current_task()) - self._tasks.add(host_task) - try: - task_state = _task_states[host_task] - except KeyError: - task_state = TaskState(None, self) - _task_states[host_task] = task_state - else: - self._parent_scope = task_state.cancel_scope - task_state.cancel_scope = self - if self._parent_scope is not None: - # If using an eager task factory, the parent scope may not even contain - # the host task - self._parent_scope._child_scopes.add(self) - self._parent_scope._tasks.discard(host_task) - - self._timeout() - self._active = True - - # Start cancelling the host task if the scope was cancelled before entering - if self._cancel_called: - self._deliver_cancellation(self) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - del exc_tb - - if not self._active: - raise RuntimeError("This cancel scope is not active") - if current_task() is not self._host_task: - raise RuntimeError( - "Attempted to exit cancel scope in a different task than it was " - "entered in" - ) - - assert self._host_task is not None - host_task_state = _task_states.get(self._host_task) - if host_task_state is None or host_task_state.cancel_scope is not self: - raise RuntimeError( - "Attempted to exit a cancel scope that isn't the current tasks's " - "current cancel scope" - ) - - try: - self._active = False - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._tasks.remove(self._host_task) - if self._parent_scope is not None: - self._parent_scope._child_scopes.remove(self) - self._parent_scope._tasks.add(self._host_task) - - host_task_state.cancel_scope = self._parent_scope - - # Restart the cancellation effort in the closest visible, cancelled parent - # scope if necessary - self._restart_cancellation_in_parent() - - # We only swallow the exception iff it was an AnyIO CancelledError, either - # directly as exc_val or inside an exception group and there are no cancelled - # parent cancel scopes visible to us here - if self._cancel_called and not self._parent_cancellation_is_visible_to_us: - # For each level-cancel() call made on the host task, call uncancel() - while self._pending_uncancellations: - self._host_task.uncancel() - self._pending_uncancellations -= 1 - - # Update cancelled_caught and check for exceptions we must not swallow - cannot_swallow_exc_val = False - if exc_val is not None: - for exc in iterate_exceptions(exc_val): - if isinstance(exc, CancelledError) and is_anyio_cancellation( - exc - ): - self._cancelled_caught = True - else: - cannot_swallow_exc_val = True - - return self._cancelled_caught and not cannot_swallow_exc_val - else: - if self._pending_uncancellations: - assert self._parent_scope is not None - assert self._parent_scope._pending_uncancellations is not None - self._parent_scope._pending_uncancellations += ( - self._pending_uncancellations - ) - self._pending_uncancellations = 0 - - return False - finally: - self._host_task = None - del exc_val - - @property - def _effectively_cancelled(self) -> bool: - cancel_scope: CancelScope | None = self - while cancel_scope is not None: - if cancel_scope._cancel_called: - return True - - if cancel_scope.shield: - return False - - cancel_scope = cancel_scope._parent_scope - - return False - - @property - def _parent_cancellation_is_visible_to_us(self) -> bool: - return ( - self._parent_scope is not None - and not self.shield - and self._parent_scope._effectively_cancelled - ) - - def _timeout(self) -> None: - if self._deadline != math.inf: - loop = get_running_loop() - if loop.time() >= self._deadline: - self.cancel() - else: - self._timeout_handle = loop.call_at(self._deadline, self._timeout) - - def _deliver_cancellation(self, origin: CancelScope) -> bool: - """ - Deliver cancellation to directly contained tasks and nested cancel scopes. - - Schedule another run at the end if we still have tasks eligible for - cancellation. - - :param origin: the cancel scope that originated the cancellation - :return: ``True`` if the delivery needs to be retried on the next cycle - - """ - should_retry = False - current = current_task() - for task in self._tasks: - should_retry = True - if task._must_cancel: # type: ignore[attr-defined] - continue - - # The task is eligible for cancellation if it has started - if task is not current and (task is self._host_task or _task_started(task)): - waiter = task._fut_waiter # type: ignore[attr-defined] - if not isinstance(waiter, asyncio.Future) or not waiter.done(): - task.cancel(f"Cancelled by cancel scope {id(origin):x}") - if ( - task is origin._host_task - and origin._pending_uncancellations is not None - ): - origin._pending_uncancellations += 1 - - # Deliver cancellation to child scopes that aren't shielded or running their own - # cancellation callbacks - for scope in self._child_scopes: - if not scope._shield and not scope.cancel_called: - should_retry = scope._deliver_cancellation(origin) or should_retry - - # Schedule another callback if there are still tasks left - if origin is self: - if should_retry: - self._cancel_handle = get_running_loop().call_soon( - self._deliver_cancellation, origin - ) - else: - self._cancel_handle = None - - return should_retry - - def _restart_cancellation_in_parent(self) -> None: - """ - Restart the cancellation effort in the closest directly cancelled parent scope. - - """ - scope = self._parent_scope - while scope is not None: - if scope._cancel_called: - if scope._cancel_handle is None: - scope._deliver_cancellation(scope) - - break - - # No point in looking beyond any shielded scope - if scope._shield: - break - - scope = scope._parent_scope - - def cancel(self) -> None: - if not self._cancel_called: - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._cancel_called = True - if self._host_task is not None: - self._deliver_cancellation(self) - - @property - def deadline(self) -> float: - return self._deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self._deadline = float(value) - if self._timeout_handle is not None: - self._timeout_handle.cancel() - self._timeout_handle = None - - if self._active and not self._cancel_called: - self._timeout() - - @property - def cancel_called(self) -> bool: - return self._cancel_called - - @property - def cancelled_caught(self) -> bool: - return self._cancelled_caught - - @property - def shield(self) -> bool: - return self._shield - - @shield.setter - def shield(self, value: bool) -> None: - if self._shield != value: - self._shield = value - if not value: - self._restart_cancellation_in_parent() - - -# -# Task states -# - - -class TaskState: - """ - Encapsulates auxiliary task information that cannot be added to the Task instance - itself because there are no guarantees about its implementation. - """ - - __slots__ = "parent_id", "cancel_scope", "__weakref__" - - def __init__(self, parent_id: int | None, cancel_scope: CancelScope | None): - self.parent_id = parent_id - self.cancel_scope = cancel_scope - - -_task_states: WeakKeyDictionary[asyncio.Task, TaskState] = WeakKeyDictionary() - - -# -# Task groups -# - - -class _AsyncioTaskStatus(abc.TaskStatus): - def __init__(self, future: asyncio.Future, parent_id: int): - self._future = future - self._parent_id = parent_id - - def started(self, value: T_contra | None = None) -> None: - try: - self._future.set_result(value) - except asyncio.InvalidStateError: - if not self._future.cancelled(): - raise RuntimeError( - "called 'started' twice on the same task status" - ) from None - - task = cast(asyncio.Task, current_task()) - _task_states[task].parent_id = self._parent_id - - -if sys.version_info >= (3, 12): - _eager_task_factory_code: CodeType | None = asyncio.eager_task_factory.__code__ -else: - _eager_task_factory_code = None - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self.cancel_scope: CancelScope = CancelScope() - self._active = False - self._exceptions: list[BaseException] = [] - self._tasks: set[asyncio.Task] = set() - self._on_completed_fut: asyncio.Future[None] | None = None - - async def __aenter__(self) -> TaskGroup: - self.cancel_scope.__enter__() - self._active = True - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - try: - if exc_val is not None: - self.cancel_scope.cancel() - if not isinstance(exc_val, CancelledError): - self._exceptions.append(exc_val) - - loop = get_running_loop() - try: - if self._tasks: - with CancelScope() as wait_scope: - while self._tasks: - self._on_completed_fut = loop.create_future() - - try: - await self._on_completed_fut - except CancelledError as exc: - # Shield the scope against further cancellation attempts, - # as they're not productive (#695) - wait_scope.shield = True - self.cancel_scope.cancel() - - # Set exc_val from the cancellation exception if it was - # previously unset. However, we should not replace a native - # cancellation exception with one raise by a cancel scope. - if exc_val is None or ( - isinstance(exc_val, CancelledError) - and not is_anyio_cancellation(exc) - ): - exc_val = exc - - self._on_completed_fut = None - else: - # If there are no child tasks to wait on, run at least one checkpoint - # anyway - await AsyncIOBackend.cancel_shielded_checkpoint() - - self._active = False - if self._exceptions: - raise BaseExceptionGroup( - "unhandled errors in a TaskGroup", self._exceptions - ) - elif exc_val: - raise exc_val - except BaseException as exc: - if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__): - return True - - raise - - return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) - finally: - del exc_val, exc_tb, self._exceptions - - def _spawn( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - args: tuple[Unpack[PosArgsT]], - name: object, - task_status_future: asyncio.Future | None = None, - ) -> asyncio.Task: - def task_done(_task: asyncio.Task) -> None: - task_state = _task_states[_task] - assert task_state.cancel_scope is not None - assert _task in task_state.cancel_scope._tasks - task_state.cancel_scope._tasks.remove(_task) - self._tasks.remove(task) - del _task_states[_task] - - if self._on_completed_fut is not None and not self._tasks: - try: - self._on_completed_fut.set_result(None) - except asyncio.InvalidStateError: - pass - - try: - exc = _task.exception() - except CancelledError as e: - while isinstance(e.__context__, CancelledError): - e = e.__context__ - - exc = e - - if exc is not None: - # The future can only be in the cancelled state if the host task was - # cancelled, so return immediately instead of adding one more - # CancelledError to the exceptions list - if task_status_future is not None and task_status_future.cancelled(): - return - - if task_status_future is None or task_status_future.done(): - if not isinstance(exc, CancelledError): - self._exceptions.append(exc) - - if not self.cancel_scope._effectively_cancelled: - self.cancel_scope.cancel() - else: - task_status_future.set_exception(exc) - elif task_status_future is not None and not task_status_future.done(): - task_status_future.set_exception( - RuntimeError("Child exited without calling task_status.started()") - ) - - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - kwargs = {} - if task_status_future: - parent_id = id(current_task()) - kwargs["task_status"] = _AsyncioTaskStatus( - task_status_future, id(self.cancel_scope._host_task) - ) - else: - parent_id = id(self.cancel_scope._host_task) - - coro = func(*args, **kwargs) - if not iscoroutine(coro): - prefix = f"{func.__module__}." if hasattr(func, "__module__") else "" - raise TypeError( - f"Expected {prefix}{func.__qualname__}() to return a coroutine, but " - f"the return value ({coro!r}) is not a coroutine object" - ) - - name = get_callable_name(func) if name is None else str(name) - loop = asyncio.get_running_loop() - if ( - (factory := loop.get_task_factory()) - and getattr(factory, "__code__", None) is _eager_task_factory_code - and (closure := getattr(factory, "__closure__", None)) - ): - custom_task_constructor = closure[0].cell_contents - task = custom_task_constructor(coro, loop=loop, name=name) - else: - task = create_task(coro, name=name) - - # Make the spawned task inherit the task group's cancel scope - _task_states[task] = TaskState( - parent_id=parent_id, cancel_scope=self.cancel_scope - ) - self.cancel_scope._tasks.add(task) - self._tasks.add(task) - task.add_done_callback(task_done) - return task - - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - self._spawn(func, args, name) - - async def start( - self, func: Callable[..., Awaitable[Any]], *args: object, name: object = None - ) -> Any: - future: asyncio.Future = asyncio.Future() - task = self._spawn(func, args, name, future) - - # If the task raises an exception after sending a start value without a switch - # point between, the task group is cancelled and this method never proceeds to - # process the completed future. That's why we have to have a shielded cancel - # scope here. - try: - return await future - except CancelledError: - # Cancel the task and wait for it to exit before returning - task.cancel() - with CancelScope(shield=True), suppress(CancelledError): - await task - - raise - - -# -# Threads -# - -_Retval_Queue_Type = tuple[Optional[T_Retval], Optional[BaseException]] - - -class WorkerThread(Thread): - MAX_IDLE_TIME = 10 # seconds - - def __init__( - self, - root_task: asyncio.Task, - workers: set[WorkerThread], - idle_workers: deque[WorkerThread], - ): - super().__init__(name="AnyIO worker thread") - self.root_task = root_task - self.workers = workers - self.idle_workers = idle_workers - self.loop = root_task._loop - self.queue: Queue[ - tuple[Context, Callable, tuple, asyncio.Future, CancelScope] | None - ] = Queue(2) - self.idle_since = AsyncIOBackend.current_time() - self.stopping = False - - def _report_result( - self, future: asyncio.Future, result: Any, exc: BaseException | None - ) -> None: - self.idle_since = AsyncIOBackend.current_time() - if not self.stopping: - self.idle_workers.append(self) - - if not future.cancelled(): - if exc is not None: - if isinstance(exc, StopIteration): - new_exc = RuntimeError("coroutine raised StopIteration") - new_exc.__cause__ = exc - exc = new_exc - - future.set_exception(exc) - else: - future.set_result(result) - - def run(self) -> None: - with claim_worker_thread(AsyncIOBackend, self.loop): - while True: - item = self.queue.get() - if item is None: - # Shutdown command received - return - - context, func, args, future, cancel_scope = item - if not future.cancelled(): - result = None - exception: BaseException | None = None - threadlocals.current_cancel_scope = cancel_scope - try: - result = context.run(func, *args) - except BaseException as exc: - exception = exc - finally: - del threadlocals.current_cancel_scope - - if not self.loop.is_closed(): - self.loop.call_soon_threadsafe( - self._report_result, future, result, exception - ) - - self.queue.task_done() - - def stop(self, f: asyncio.Task | None = None) -> None: - self.stopping = True - self.queue.put_nowait(None) - self.workers.discard(self) - try: - self.idle_workers.remove(self) - except ValueError: - pass - - -_threadpool_idle_workers: RunVar[deque[WorkerThread]] = RunVar( - "_threadpool_idle_workers" -) -_threadpool_workers: RunVar[set[WorkerThread]] = RunVar("_threadpool_workers") - - -class BlockingPortal(abc.BlockingPortal): - def __new__(cls) -> BlockingPortal: - return object.__new__(cls) - - def __init__(self) -> None: - super().__init__() - self._loop = get_running_loop() - - def _spawn_task_from_thread( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - name: object, - future: Future[T_Retval], - ) -> None: - AsyncIOBackend.run_sync_from_thread( - partial(self._task_group.start_soon, name=name), - (self._call_func, func, args, kwargs, future), - self._loop, - ) - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class StreamReaderWrapper(abc.ByteReceiveStream): - _stream: asyncio.StreamReader - - async def receive(self, max_bytes: int = 65536) -> bytes: - data = await self._stream.read(max_bytes) - if data: - return data - else: - raise EndOfStream - - async def aclose(self) -> None: - self._stream.set_exception(ClosedResourceError()) - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class StreamWriterWrapper(abc.ByteSendStream): - _stream: asyncio.StreamWriter - - async def send(self, item: bytes) -> None: - self._stream.write(item) - await self._stream.drain() - - async def aclose(self) -> None: - self._stream.close() - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: asyncio.subprocess.Process - _stdin: StreamWriterWrapper | None - _stdout: StreamReaderWrapper | None - _stderr: StreamReaderWrapper | None - - async def aclose(self) -> None: - with CancelScope(shield=True) as scope: - if self._stdin: - await self._stdin.aclose() - if self._stdout: - await self._stdout.aclose() - if self._stderr: - await self._stderr.aclose() - - scope.shield = False - try: - await self.wait() - except BaseException: - scope.shield = True - self.kill() - await self.wait() - raise - - async def wait(self) -> int: - return await self._process.wait() - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: int) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -def _forcibly_shutdown_process_pool_on_exit( - workers: set[Process], _task: object -) -> None: - """ - Forcibly shuts down worker processes belonging to this event loop.""" - child_watcher: asyncio.AbstractChildWatcher | None = None - if sys.version_info < (3, 12): - try: - child_watcher = asyncio.get_event_loop_policy().get_child_watcher() - except NotImplementedError: - pass - - # Close as much as possible (w/o async/await) to avoid warnings - for process in workers: - if process.returncode is None: - continue - - process._stdin._stream._transport.close() # type: ignore[union-attr] - process._stdout._stream._transport.close() # type: ignore[union-attr] - process._stderr._stream._transport.close() # type: ignore[union-attr] - process.kill() - if child_watcher: - child_watcher.remove_child_handler(process.pid) - - -async def _shutdown_process_pool_on_exit(workers: set[abc.Process]) -> None: - """ - Shuts down worker processes belonging to this event loop. - - NOTE: this only works when the event loop was started using asyncio.run() or - anyio.run(). - - """ - process: abc.Process - try: - await sleep(math.inf) - except asyncio.CancelledError: - for process in workers: - if process.returncode is None: - process.kill() - - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class StreamProtocol(asyncio.Protocol): - read_queue: deque[bytes] - read_event: asyncio.Event - write_event: asyncio.Event - exception: Exception | None = None - is_at_eof: bool = False - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque() - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.write_event.set() - cast(asyncio.Transport, transport).set_write_buffer_limits(0) - - def connection_lost(self, exc: Exception | None) -> None: - if exc: - self.exception = BrokenResourceError() - self.exception.__cause__ = exc - - self.read_event.set() - self.write_event.set() - - def data_received(self, data: bytes) -> None: - # ProactorEventloop sometimes sends bytearray instead of bytes - self.read_queue.append(bytes(data)) - self.read_event.set() - - def eof_received(self) -> bool | None: - self.is_at_eof = True - self.read_event.set() - return True - - def pause_writing(self) -> None: - self.write_event = asyncio.Event() - - def resume_writing(self) -> None: - self.write_event.set() - - -class DatagramProtocol(asyncio.DatagramProtocol): - read_queue: deque[tuple[bytes, IPSockAddrType]] - read_event: asyncio.Event - write_event: asyncio.Event - exception: Exception | None = None - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque(maxlen=100) # arbitrary value - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.write_event.set() - - def connection_lost(self, exc: Exception | None) -> None: - self.read_event.set() - self.write_event.set() - - def datagram_received(self, data: bytes, addr: IPSockAddrType) -> None: - addr = convert_ipv6_sockaddr(addr) - self.read_queue.append((data, addr)) - self.read_event.set() - - def error_received(self, exc: Exception) -> None: - self.exception = exc - - def pause_writing(self) -> None: - self.write_event.clear() - - def resume_writing(self) -> None: - self.write_event.set() - - -class SocketStream(abc.SocketStream): - def __init__(self, transport: asyncio.Transport, protocol: StreamProtocol): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def receive(self, max_bytes: int = 65536) -> bytes: - with self._receive_guard: - if ( - not self._protocol.read_event.is_set() - and not self._transport.is_closing() - and not self._protocol.is_at_eof - ): - self._transport.resume_reading() - await self._protocol.read_event.wait() - self._transport.pause_reading() - else: - await AsyncIOBackend.checkpoint() - - try: - chunk = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - elif self._protocol.exception: - raise self._protocol.exception from None - else: - raise EndOfStream from None - - if len(chunk) > max_bytes: - # Split the oversized chunk - chunk, leftover = chunk[:max_bytes], chunk[max_bytes:] - self._protocol.read_queue.appendleft(leftover) - - # If the read queue is empty, clear the flag so that the next call will - # block until data is available - if not self._protocol.read_queue: - self._protocol.read_event.clear() - - return chunk - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - - if self._closed: - raise ClosedResourceError - elif self._protocol.exception is not None: - raise self._protocol.exception - - try: - self._transport.write(item) - except RuntimeError as exc: - if self._transport.is_closing(): - raise BrokenResourceError from exc - else: - raise - - await self._protocol.write_event.wait() - - async def send_eof(self) -> None: - try: - self._transport.write_eof() - except OSError: - pass - - async def aclose(self) -> None: - if not self._transport.is_closing(): - self._closed = True - try: - self._transport.write_eof() - except OSError: - pass - - self._transport.close() - await sleep(0) - self._transport.abort() - - -class _RawSocketMixin: - _receive_future: asyncio.Future | None = None - _send_future: asyncio.Future | None = None - _closing = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - def _wait_until_readable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._receive_future - loop.remove_reader(self.__raw_socket) - - f = self._receive_future = asyncio.Future() - loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - def _wait_until_writable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._send_future - loop.remove_writer(self.__raw_socket) - - f = self._send_future = asyncio.Future() - loop.add_writer(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - async def aclose(self) -> None: - if not self._closing: - self._closing = True - if self.__raw_socket.fileno() != -1: - self.__raw_socket.close() - - if self._receive_future: - self._receive_future.set_result(None) - if self._send_future: - self._send_future.set_result(None) - - -class UNIXSocketStream(_RawSocketMixin, abc.UNIXSocketStream): - async def send_eof(self) -> None: - with self._send_guard: - self._raw_socket.shutdown(socket.SHUT_WR) - - async def receive(self, max_bytes: int = 65536) -> bytes: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(max_bytes) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = self._raw_socket.send(view) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - view = view[bytes_sent:] - - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - loop = get_running_loop() - fds = array.array("i") - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = self._raw_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - loop = get_running_loop() - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - # The ignore can be removed after mypy picks up - # https://github.com/python/typeshed/pull/5545 - self._raw_socket.sendmsg( - [message], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fdarray)] - ) - break - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - -class TCPSocketListener(abc.SocketListener): - _accept_scope: CancelScope | None = None - _closed = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = cast(asyncio.BaseEventLoop, get_running_loop()) - self._accept_guard = ResourceGuard("accepting connections from") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - async def accept(self) -> abc.SocketStream: - if self._closed: - raise ClosedResourceError - - with self._accept_guard: - await AsyncIOBackend.checkpoint() - with CancelScope() as self._accept_scope: - try: - client_sock, _addr = await self._loop.sock_accept(self._raw_socket) - except asyncio.CancelledError: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - if self._closed: - raise ClosedResourceError from None - - raise - finally: - self._accept_scope = None - - client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - transport, protocol = await self._loop.connect_accepted_socket( - StreamProtocol, client_sock - ) - return SocketStream(transport, protocol) - - async def aclose(self) -> None: - if self._closed: - return - - self._closed = True - if self._accept_scope: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - self._accept_scope.cancel() - await sleep(0) - - self._raw_socket.close() - - -class UNIXSocketListener(abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = get_running_loop() - self._accept_guard = ResourceGuard("accepting connections from") - self._closed = False - - async def accept(self) -> abc.SocketStream: - await AsyncIOBackend.checkpoint() - with self._accept_guard: - while True: - try: - client_sock, _ = self.__raw_socket.accept() - client_sock.setblocking(False) - return UNIXSocketStream(client_sock) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - self._loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback( - lambda _: self._loop.remove_reader(self.__raw_socket) - ) - await f - except OSError as exc: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - async def aclose(self) -> None: - self._closed = True - self.__raw_socket.close() - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - -class UDPSocket(abc.UDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - if not self._transport.is_closing(): - self._closed = True - self._transport.close() - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - return self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(*item) - - -class ConnectedUDPSocket(abc.ConnectedUDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - if not self._transport.is_closing(): - self._closed = True - self._transport.close() - - async def receive(self) -> bytes: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - packet = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - return packet[0] - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(item) - - -class UNIXDatagramSocket(_RawSocketMixin, abc.UNIXDatagramSocket): - async def receive(self) -> UNIXDatagramPacketType: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recvfrom(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: UNIXDatagramPacketType) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.sendto(*item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -class ConnectedUNIXDatagramSocket(_RawSocketMixin, abc.ConnectedUNIXDatagramSocket): - async def receive(self) -> bytes: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.send(item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -_read_events: RunVar[dict[int, asyncio.Event]] = RunVar("read_events") -_write_events: RunVar[dict[int, asyncio.Event]] = RunVar("write_events") - - -# -# Synchronization -# - - -class Event(BaseEvent): - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self._event = asyncio.Event() - - def set(self) -> None: - self._event.set() - - def is_set(self) -> bool: - return self._event.is_set() - - async def wait(self) -> None: - if self.is_set(): - await AsyncIOBackend.checkpoint() - else: - await self._event.wait() - - def statistics(self) -> EventStatistics: - return EventStatistics(len(self._event._waiters)) - - -class Lock(BaseLock): - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self._owner_task: asyncio.Task | None = None - self._waiters: deque[tuple[asyncio.Task, asyncio.Future]] = deque() - - async def acquire(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._owner_task = task - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - if self._owner_task == task: - raise RuntimeError("Attempted to acquire an already held Lock") - - fut: asyncio.Future[None] = asyncio.Future() - item = task, fut - self._waiters.append(item) - try: - await fut - except CancelledError: - self._waiters.remove(item) - if self._owner_task is task: - self.release() - - raise - - self._waiters.remove(item) - - def acquire_nowait(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - self._owner_task = task - return - - if self._owner_task is task: - raise RuntimeError("Attempted to acquire an already held Lock") - - raise WouldBlock - - def locked(self) -> bool: - return self._owner_task is not None - - def release(self) -> None: - if self._owner_task != current_task(): - raise RuntimeError("The current task is not holding this lock") - - for task, fut in self._waiters: - if not fut.cancelled(): - self._owner_task = task - fut.set_result(None) - return - - self._owner_task = None - - def statistics(self) -> LockStatistics: - task_info = AsyncIOTaskInfo(self._owner_task) if self._owner_task else None - return LockStatistics(self.locked(), task_info, len(self._waiters)) - - -class Semaphore(BaseSemaphore): - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - super().__init__(initial_value, max_value=max_value) - self._value = initial_value - self._max_value = max_value - self._fast_acquire = fast_acquire - self._waiters: deque[asyncio.Future[None]] = deque() - - async def acquire(self) -> None: - if self._value > 0 and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._value -= 1 - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - fut: asyncio.Future[None] = asyncio.Future() - self._waiters.append(fut) - try: - await fut - except CancelledError: - try: - self._waiters.remove(fut) - except ValueError: - self.release() - - raise - - def acquire_nowait(self) -> None: - if self._value == 0: - raise WouldBlock - - self._value -= 1 - - def release(self) -> None: - if self._max_value is not None and self._value == self._max_value: - raise ValueError("semaphore released too many times") - - for fut in self._waiters: - if not fut.cancelled(): - fut.set_result(None) - self._waiters.remove(fut) - return - - self._value += 1 - - @property - def value(self) -> int: - return self._value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - return SemaphoreStatistics(len(self._waiters)) - - -class CapacityLimiter(BaseCapacityLimiter): - _total_tokens: float = 0 - - def __new__(cls, total_tokens: float) -> CapacityLimiter: - return object.__new__(cls) - - def __init__(self, total_tokens: float): - self._borrowers: set[Any] = set() - self._wait_queue: OrderedDict[Any, asyncio.Event] = OrderedDict() - self.total_tokens = total_tokens - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - @property - def total_tokens(self) -> float: - return self._total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and not math.isinf(value): - raise TypeError("total_tokens must be an int or math.inf") - if value < 1: - raise ValueError("total_tokens must be >= 1") - - waiters_to_notify = max(value - self._total_tokens, 0) - self._total_tokens = value - - # Notify waiting tasks that they have acquired the limiter - while self._wait_queue and waiters_to_notify: - event = self._wait_queue.popitem(last=False)[1] - event.set() - waiters_to_notify -= 1 - - @property - def borrowed_tokens(self) -> int: - return len(self._borrowers) - - @property - def available_tokens(self) -> float: - return self._total_tokens - len(self._borrowers) - - def acquire_nowait(self) -> None: - self.acquire_on_behalf_of_nowait(current_task()) - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - if borrower in self._borrowers: - raise RuntimeError( - "this borrower is already holding one of this CapacityLimiter's " - "tokens" - ) - - if self._wait_queue or len(self._borrowers) >= self._total_tokens: - raise WouldBlock - - self._borrowers.add(borrower) - - async def acquire(self) -> None: - return await self.acquire_on_behalf_of(current_task()) - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await AsyncIOBackend.checkpoint_if_cancelled() - try: - self.acquire_on_behalf_of_nowait(borrower) - except WouldBlock: - event = asyncio.Event() - self._wait_queue[borrower] = event - try: - await event.wait() - except BaseException: - self._wait_queue.pop(borrower, None) - raise - - self._borrowers.add(borrower) - else: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except BaseException: - self.release() - raise - - def release(self) -> None: - self.release_on_behalf_of(current_task()) - - def release_on_behalf_of(self, borrower: object) -> None: - try: - self._borrowers.remove(borrower) - except KeyError: - raise RuntimeError( - "this borrower isn't holding any of this CapacityLimiter's tokens" - ) from None - - # Notify the next task in line if this limiter has free capacity now - if self._wait_queue and len(self._borrowers) < self._total_tokens: - event = self._wait_queue.popitem(last=False)[1] - event.set() - - def statistics(self) -> CapacityLimiterStatistics: - return CapacityLimiterStatistics( - self.borrowed_tokens, - self.total_tokens, - tuple(self._borrowers), - len(self._wait_queue), - ) - - -_default_thread_limiter: RunVar[CapacityLimiter] = RunVar("_default_thread_limiter") - - -# -# Operating system signals -# - - -class _SignalReceiver: - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - self._loop = get_running_loop() - self._signal_queue: deque[Signals] = deque() - self._future: asyncio.Future = asyncio.Future() - self._handled_signals: set[Signals] = set() - - def _deliver(self, signum: Signals) -> None: - self._signal_queue.append(signum) - if not self._future.done(): - self._future.set_result(None) - - def __enter__(self) -> _SignalReceiver: - for sig in set(self._signals): - self._loop.add_signal_handler(sig, self._deliver, sig) - self._handled_signals.add(sig) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - for sig in self._handled_signals: - self._loop.remove_signal_handler(sig) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - await AsyncIOBackend.checkpoint() - if not self._signal_queue: - self._future = asyncio.Future() - await self._future - - return self._signal_queue.popleft() - - -# -# Testing and debugging -# - - -class AsyncIOTaskInfo(TaskInfo): - def __init__(self, task: asyncio.Task): - task_state = _task_states.get(task) - if task_state is None: - parent_id = None - else: - parent_id = task_state.parent_id - - coro = task.get_coro() - assert coro is not None, "created TaskInfo from a completed Task" - super().__init__(id(task), parent_id, task.get_name(), coro) - self._task = weakref.ref(task) - - def has_pending_cancellation(self) -> bool: - if not (task := self._task()): - # If the task isn't around anymore, it won't have a pending cancellation - return False - - if task._must_cancel: # type: ignore[attr-defined] - return True - elif ( - isinstance(task._fut_waiter, asyncio.Future) # type: ignore[attr-defined] - and task._fut_waiter.cancelled() # type: ignore[attr-defined] - ): - return True - - if task_state := _task_states.get(task): - if cancel_scope := task_state.cancel_scope: - return cancel_scope._effectively_cancelled - - return False - - -class TestRunner(abc.TestRunner): - _send_stream: MemoryObjectSendStream[tuple[Awaitable[Any], asyncio.Future[Any]]] - - def __init__( - self, - *, - debug: bool | None = None, - use_uvloop: bool = False, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ) -> None: - if use_uvloop and loop_factory is None: - import uvloop - - loop_factory = uvloop.new_event_loop - - self._runner = Runner(debug=debug, loop_factory=loop_factory) - self._exceptions: list[BaseException] = [] - self._runner_task: asyncio.Task | None = None - - def __enter__(self) -> TestRunner: - self._runner.__enter__() - self.get_loop().set_exception_handler(self._exception_handler) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._runner.__exit__(exc_type, exc_val, exc_tb) - - def get_loop(self) -> AbstractEventLoop: - return self._runner.get_loop() - - def _exception_handler( - self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] - ) -> None: - if isinstance(context.get("exception"), Exception): - self._exceptions.append(context["exception"]) - else: - loop.default_exception_handler(context) - - def _raise_async_exceptions(self) -> None: - # Re-raise any exceptions raised in asynchronous callbacks - if self._exceptions: - exceptions, self._exceptions = self._exceptions, [] - if len(exceptions) == 1: - raise exceptions[0] - elif exceptions: - raise BaseExceptionGroup( - "Multiple exceptions occurred in asynchronous callbacks", exceptions - ) - - async def _run_tests_and_fixtures( - self, - receive_stream: MemoryObjectReceiveStream[ - tuple[Awaitable[T_Retval], asyncio.Future[T_Retval]] - ], - ) -> None: - from _pytest.outcomes import OutcomeException - - with receive_stream, self._send_stream: - async for coro, future in receive_stream: - try: - retval = await coro - except CancelledError as exc: - if not future.cancelled(): - future.cancel(*exc.args) - - raise - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - if not isinstance(exc, (Exception, OutcomeException)): - raise - else: - if not future.cancelled(): - future.set_result(retval) - - async def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if not self._runner_task: - self._send_stream, receive_stream = create_memory_object_stream[ - tuple[Awaitable[Any], asyncio.Future] - ](1) - self._runner_task = self.get_loop().create_task( - self._run_tests_and_fixtures(receive_stream) - ) - - coro = func(*args, **kwargs) - future: asyncio.Future[T_Retval] = self.get_loop().create_future() - self._send_stream.send_nowait((coro, future)) - return await future - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - self._raise_async_exceptions() - - yield fixturevalue - - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - except StopAsyncIteration: - self._raise_async_exceptions() - else: - self.get_loop().run_until_complete(asyncgen.aclose()) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - retval = self.get_loop().run_until_complete( - self._call_in_runner_task(fixture_func, **kwargs) - ) - self._raise_async_exceptions() - return retval - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(test_func, **kwargs) - ) - except Exception as exc: - self._exceptions.append(exc) - - self._raise_async_exceptions() - - -class AsyncIOBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - @wraps(func) - async def wrapper() -> T_Retval: - task = cast(asyncio.Task, current_task()) - task.set_name(get_callable_name(func)) - _task_states[task] = TaskState(None, None) - - try: - return await func(*args) - finally: - del _task_states[task] - - debug = options.get("debug", None) - loop_factory = options.get("loop_factory", None) - if loop_factory is None and options.get("use_uvloop", False): - import uvloop - - loop_factory = uvloop.new_event_loop - - with Runner(debug=debug, loop_factory=loop_factory) as runner: - return runner.run(wrapper()) - - @classmethod - def current_token(cls) -> object: - return get_running_loop() - - @classmethod - def current_time(cls) -> float: - return get_running_loop().time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return CancelledError - - @classmethod - async def checkpoint(cls) -> None: - await sleep(0) - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - task = current_task() - if task is None: - return - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return - - while cancel_scope: - if cancel_scope.cancel_called: - await sleep(0) - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - with CancelScope(shield=True): - await sleep(0) - - @classmethod - async def sleep(cls, delay: float) -> None: - await sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - if (task := current_task()) is None: - return math.inf - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return math.inf - - deadline = math.inf - while cancel_scope: - deadline = min(deadline, cancel_scope.deadline) - if cancel_scope._cancel_called: - deadline = -math.inf - break - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - return deadline - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> abc.Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> abc.CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( # type: ignore[return] - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - await cls.checkpoint() - - # If this is the first run in this event loop thread, set up the necessary - # variables - try: - idle_workers = _threadpool_idle_workers.get() - workers = _threadpool_workers.get() - except LookupError: - idle_workers = deque() - workers = set() - _threadpool_idle_workers.set(idle_workers) - _threadpool_workers.set(workers) - - async with limiter or cls.current_default_thread_limiter(): - with CancelScope(shield=not abandon_on_cancel) as scope: - future = asyncio.Future[T_Retval]() - root_task = find_root_task() - if not idle_workers: - worker = WorkerThread(root_task, workers, idle_workers) - worker.start() - workers.add(worker) - root_task.add_done_callback(worker.stop) - else: - worker = idle_workers.pop() - - # Prune any other workers that have been idle for MAX_IDLE_TIME - # seconds or longer - now = cls.current_time() - while idle_workers: - if ( - now - idle_workers[0].idle_since - < WorkerThread.MAX_IDLE_TIME - ): - break - - expired_worker = idle_workers.popleft() - expired_worker.root_task.remove_done_callback( - expired_worker.stop - ) - expired_worker.stop() - - context = copy_context() - context.run(sniffio.current_async_library_cvar.set, None) - if abandon_on_cancel or scope._parent_scope is None: - worker_scope = scope - else: - worker_scope = scope._parent_scope - - worker.queue.put_nowait((context, func, args, future, worker_scope)) - return await future - - @classmethod - def check_cancelled(cls) -> None: - scope: CancelScope | None = threadlocals.current_cancel_scope - while scope is not None: - if scope.cancel_called: - raise CancelledError(f"Cancelled by cancel scope {id(scope):x}") - - if scope.shield: - return - - scope = scope._parent_scope - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - async def task_wrapper(scope: CancelScope) -> T_Retval: - __tracebackhide__ = True - task = cast(asyncio.Task, current_task()) - _task_states[task] = TaskState(None, scope) - scope._tasks.add(task) - try: - return await func(*args) - except CancelledError as exc: - raise concurrent.futures.CancelledError(str(exc)) from None - finally: - scope._tasks.discard(task) - - loop = cast(AbstractEventLoop, token) - context = copy_context() - context.run(sniffio.current_async_library_cvar.set, "asyncio") - wrapper = task_wrapper(threadlocals.current_cancel_scope) - f: concurrent.futures.Future[T_Retval] = context.run( - asyncio.run_coroutine_threadsafe, wrapper, loop - ) - return f.result() - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - @wraps(func) - def wrapper() -> None: - try: - sniffio.current_async_library_cvar.set("asyncio") - f.set_result(func(*args)) - except BaseException as exc: - f.set_exception(exc) - if not isinstance(exc, Exception): - raise - - f: concurrent.futures.Future[T_Retval] = Future() - loop = cast(AbstractEventLoop, token) - loop.call_soon_threadsafe(wrapper) - return f.result() - - @classmethod - def create_blocking_portal(cls) -> abc.BlockingPortal: - return BlockingPortal() - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - await cls.checkpoint() - if isinstance(command, PathLike): - command = os.fspath(command) - - if isinstance(command, (str, bytes)): - process = await asyncio.create_subprocess_shell( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - else: - process = await asyncio.create_subprocess_exec( - *command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - - stdin_stream = StreamWriterWrapper(process.stdin) if process.stdin else None - stdout_stream = StreamReaderWrapper(process.stdout) if process.stdout else None - stderr_stream = StreamReaderWrapper(process.stderr) if process.stderr else None - return Process(process, stdin_stream, stdout_stream, stderr_stream) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - create_task( - _shutdown_process_pool_on_exit(workers), - name="AnyIO process pool shutdown task", - ) - find_root_task().add_done_callback( - partial(_forcibly_shutdown_process_pool_on_exit, workers) # type:ignore[arg-type] - ) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> abc.SocketStream: - transport, protocol = cast( - tuple[asyncio.Transport, StreamProtocol], - await get_running_loop().create_connection( - StreamProtocol, host, port, local_addr=local_address - ), - ) - transport.pause_reading() - return SocketStream(transport, protocol) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - await cls.checkpoint() - loop = get_running_loop() - raw_socket = socket.socket(socket.AF_UNIX) - raw_socket.setblocking(False) - while True: - try: - raw_socket.connect(path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return UNIXSocketStream(raw_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, - local_addr=local_address, - remote_addr=remote_address, - family=family, - reuse_port=reuse_port, - ) - if protocol.exception: - transport.close() - raise protocol.exception - - if not remote_address: - return UDPSocket(transport, protocol) - else: - return ConnectedUDPSocket(transport, protocol) - - @classmethod - async def create_unix_datagram_socket( # type: ignore[override] - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - await cls.checkpoint() - loop = get_running_loop() - - if remote_path: - while True: - try: - raw_socket.connect(remote_path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return ConnectedUNIXDatagramSocket(raw_socket) - else: - return UNIXDatagramSocket(raw_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> list[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int], - ] - ]: - return await get_running_loop().getaddrinfo( - host, port, family=family, type=type, proto=proto, flags=flags - ) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await get_running_loop().getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - await cls.checkpoint() - try: - read_events = _read_events.get() - except LookupError: - read_events = {} - _read_events.set(read_events) - - if not isinstance(obj, int): - obj = obj.fileno() - - if read_events.get(obj): - raise BusyResourceError("reading from") - - loop = get_running_loop() - event = asyncio.Event() - try: - loop.add_reader(obj, event.set) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_reader(obj, event.set) - remove_reader = selector.remove_reader - else: - remove_reader = loop.remove_reader - - read_events[obj] = event - try: - await event.wait() - finally: - remove_reader(obj) - del read_events[obj] - - @classmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - await cls.checkpoint() - try: - write_events = _write_events.get() - except LookupError: - write_events = {} - _write_events.set(write_events) - - if not isinstance(obj, int): - obj = obj.fileno() - - if write_events.get(obj): - raise BusyResourceError("writing to") - - loop = get_running_loop() - event = asyncio.Event() - try: - loop.add_writer(obj, event.set) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_writer(obj, event.set) - remove_writer = selector.remove_writer - else: - remove_writer = loop.remove_writer - - write_events[obj] = event - try: - await event.wait() - finally: - del write_events[obj] - remove_writer(obj) - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _default_thread_limiter.get() - except LookupError: - limiter = CapacityLimiter(40) - _default_thread_limiter.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - return AsyncIOTaskInfo(current_task()) # type: ignore[arg-type] - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - return [AsyncIOTaskInfo(task) for task in all_tasks() if not task.done()] - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - await cls.checkpoint() - this_task = current_task() - while True: - for task in all_tasks(): - if task is this_task: - continue - - waiter = task._fut_waiter # type: ignore[attr-defined] - if waiter is None or waiter.done(): - await sleep(0.1) - break - else: - return - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = AsyncIOBackend diff --git a/contrib/python/anyio/anyio/_backends/_trio.py b/contrib/python/anyio/anyio/_backends/_trio.py deleted file mode 100644 index 32ae8ace7b4..00000000000 --- a/contrib/python/anyio/anyio/_backends/_trio.py +++ /dev/null @@ -1,1334 +0,0 @@ -from __future__ import annotations - -import array -import math -import os -import socket -import sys -import types -import weakref -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from concurrent.futures import Future -from contextlib import AbstractContextManager -from dataclasses import dataclass -from functools import partial -from io import IOBase -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind -from types import TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Generic, - NoReturn, - TypeVar, - cast, - overload, -) - -import trio.from_thread -import trio.lowlevel -from outcome import Error, Outcome, Value -from trio.lowlevel import ( - current_root_task, - current_task, - wait_readable, - wait_writable, -) -from trio.socket import SocketType as TrioSocketType -from trio.to_thread import run_sync - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - TaskInfo, - WouldBlock, - abc, -) -from .._core._eventloop import claim_worker_thread -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from ..abc import IPSockAddrType, UDPPacketType, UNIXDatagramPacketType -from ..abc._eventloop import AsyncBackend, StrOrBytesPath -from ..streams.memory import MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import HasFileno - -if sys.version_info >= (3, 10): - from typing import ParamSpec -else: - from typing_extensions import ParamSpec - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - -T = TypeVar("T") -T_Retval = TypeVar("T_Retval") -T_SockAddr = TypeVar("T_SockAddr", str, IPSockAddrType) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - - -# -# Event loop -# - -RunVar = trio.lowlevel.RunVar - - -# -# Timeouts and cancellation -# - - -class CancelScope(BaseCancelScope): - def __new__( - cls, original: trio.CancelScope | None = None, **kwargs: object - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, original: trio.CancelScope | None = None, **kwargs: Any) -> None: - self.__original = original or trio.CancelScope(**kwargs) - - def __enter__(self) -> CancelScope: - self.__original.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - return self.__original.__exit__(exc_type, exc_val, exc_tb) - - def cancel(self) -> None: - self.__original.cancel() - - @property - def deadline(self) -> float: - return self.__original.deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self.__original.deadline = value - - @property - def cancel_called(self) -> bool: - return self.__original.cancel_called - - @property - def cancelled_caught(self) -> bool: - return self.__original.cancelled_caught - - @property - def shield(self) -> bool: - return self.__original.shield - - @shield.setter - def shield(self, value: bool) -> None: - self.__original.shield = value - - -# -# Task groups -# - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self._active = False - self._nursery_manager = trio.open_nursery(strict_exception_groups=True) - self.cancel_scope = None # type: ignore[assignment] - - async def __aenter__(self) -> TaskGroup: - self._active = True - self._nursery = await self._nursery_manager.__aenter__() - self.cancel_scope = CancelScope(self._nursery.cancel_scope) - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - try: - # trio.Nursery.__exit__ returns bool; .open_nursery has wrong type - return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb) # type: ignore[return-value] - except BaseExceptionGroup as exc: - if not exc.split(trio.Cancelled)[1]: - raise trio.Cancelled._create() from exc - - raise - finally: - del exc_val, exc_tb - self._active = False - - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - self._nursery.start_soon(func, *args, name=name) - - async def start( - self, func: Callable[..., Awaitable[Any]], *args: object, name: object = None - ) -> Any: - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - return await self._nursery.start(func, *args, name=name) - - -# -# Threads -# - - -class BlockingPortal(abc.BlockingPortal): - def __new__(cls) -> BlockingPortal: - return object.__new__(cls) - - def __init__(self) -> None: - super().__init__() - self._token = trio.lowlevel.current_trio_token() - - def _spawn_task_from_thread( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - name: object, - future: Future[T_Retval], - ) -> None: - trio.from_thread.run_sync( - partial(self._task_group.start_soon, name=name), - self._call_func, - func, - args, - kwargs, - future, - trio_token=self._token, - ) - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class ReceiveStreamWrapper(abc.ByteReceiveStream): - _stream: trio.abc.ReceiveStream - - async def receive(self, max_bytes: int | None = None) -> bytes: - try: - data = await self._stream.receive_some(max_bytes) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - if data: - return data - else: - raise EndOfStream - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class SendStreamWrapper(abc.ByteSendStream): - _stream: trio.abc.SendStream - - async def send(self, item: bytes) -> None: - try: - await self._stream.send_all(item) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: trio.Process - _stdin: abc.ByteSendStream | None - _stdout: abc.ByteReceiveStream | None - _stderr: abc.ByteReceiveStream | None - - async def aclose(self) -> None: - with CancelScope(shield=True): - if self._stdin: - await self._stdin.aclose() - if self._stdout: - await self._stdout.aclose() - if self._stderr: - await self._stderr.aclose() - - try: - await self.wait() - except BaseException: - self.kill() - with CancelScope(shield=True): - await self.wait() - raise - - async def wait(self) -> int: - return await self._process.wait() - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: Signals) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -class _ProcessPoolShutdownInstrument(trio.abc.Instrument): - def after_run(self) -> None: - super().after_run() - - -current_default_worker_process_limiter: trio.lowlevel.RunVar = RunVar( - "current_default_worker_process_limiter" -) - - -async def _shutdown_process_pool(workers: set[abc.Process]) -> None: - try: - await trio.sleep(math.inf) - except trio.Cancelled: - for process in workers: - if process.returncode is None: - process.kill() - - with CancelScope(shield=True): - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class _TrioSocketMixin(Generic[T_SockAddr]): - def __init__(self, trio_socket: TrioSocketType) -> None: - self._trio_socket = trio_socket - self._closed = False - - def _check_closed(self) -> None: - if self._closed: - raise ClosedResourceError - if self._trio_socket.fileno() < 0: - raise BrokenResourceError - - @property - def _raw_socket(self) -> socket.socket: - return self._trio_socket._sock # type: ignore[attr-defined] - - async def aclose(self) -> None: - if self._trio_socket.fileno() >= 0: - self._closed = True - self._trio_socket.close() - - def _convert_socket_error(self, exc: BaseException) -> NoReturn: - if isinstance(exc, trio.ClosedResourceError): - raise ClosedResourceError from exc - elif self._trio_socket.fileno() < 0 and self._closed: - raise ClosedResourceError from None - elif isinstance(exc, OSError): - raise BrokenResourceError from exc - else: - raise exc - - -class SocketStream(_TrioSocketMixin, abc.SocketStream): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self, max_bytes: int = 65536) -> bytes: - with self._receive_guard: - try: - data = await self._trio_socket.recv(max_bytes) - except BaseException as exc: - self._convert_socket_error(exc) - - if data: - return data - else: - raise EndOfStream - - async def send(self, item: bytes) -> None: - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = await self._trio_socket.send(view) - except BaseException as exc: - self._convert_socket_error(exc) - - view = view[bytes_sent:] - - async def send_eof(self) -> None: - self._trio_socket.shutdown(socket.SHUT_WR) - - -class UNIXSocketStream(SocketStream, abc.UNIXSocketStream): - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - fds = array.array("i") - await trio.lowlevel.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = await self._trio_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BaseException as exc: - self._convert_socket_error(exc) - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await trio.lowlevel.checkpoint() - with self._send_guard: - while True: - try: - await self._trio_socket.sendmsg( - [message], - [ - ( - socket.SOL_SOCKET, - socket.SCM_RIGHTS, - fdarray, - ) - ], - ) - break - except BaseException as exc: - self._convert_socket_error(exc) - - -class TCPSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> SocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return SocketStream(trio_socket) - - -class UNIXSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> UNIXSocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - return UNIXSocketStream(trio_socket) - - -class UDPSocket(_TrioSocketMixin[IPSockAddrType], abc.UDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, convert_ipv6_sockaddr(addr) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUDPSocket(_TrioSocketMixin[IPSockAddrType], abc.ConnectedUDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class UNIXDatagramSocket(_TrioSocketMixin[str], abc.UNIXDatagramSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> UNIXDatagramPacketType: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, addr - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UNIXDatagramPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUNIXDatagramSocket( - _TrioSocketMixin[str], abc.ConnectedUNIXDatagramSocket -): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -# -# Synchronization -# - - -class Event(BaseEvent): - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self.__original = trio.Event() - - def is_set(self) -> bool: - return self.__original.is_set() - - async def wait(self) -> None: - return await self.__original.wait() - - def statistics(self) -> EventStatistics: - orig_statistics = self.__original.statistics() - return EventStatistics(tasks_waiting=orig_statistics.tasks_waiting) - - def set(self) -> None: - self.__original.set() - - -class Lock(BaseLock): - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self.__original = trio.Lock() - - @staticmethod - def _convert_runtime_error_msg(exc: RuntimeError) -> None: - if exc.args == ("attempt to re-acquire an already held Lock",): - exc.args = ("Attempted to acquire an already held Lock",) - - async def acquire(self) -> None: - if not self._fast_acquire: - try: - await self.__original.acquire() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def locked(self) -> bool: - return self.__original.locked() - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> LockStatistics: - orig_statistics = self.__original.statistics() - owner = TrioTaskInfo(orig_statistics.owner) if orig_statistics.owner else None - return LockStatistics( - orig_statistics.locked, owner, orig_statistics.tasks_waiting - ) - - -class Semaphore(BaseSemaphore): - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self.__original = trio.Semaphore(initial_value, max_value=max_value) - - async def acquire(self) -> None: - if not self._fast_acquire: - await self.__original.acquire() - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - - @property - def max_value(self) -> int | None: - return self.__original.max_value - - @property - def value(self) -> int: - return self.__original.value - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> SemaphoreStatistics: - orig_statistics = self.__original.statistics() - return SemaphoreStatistics(orig_statistics.tasks_waiting) - - -class CapacityLimiter(BaseCapacityLimiter): - def __new__( - cls, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> CapacityLimiter: - return object.__new__(cls) - - def __init__( - self, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> None: - if original is not None: - self.__original = original - else: - assert total_tokens is not None - self.__original = trio.CapacityLimiter(total_tokens) - - async def __aenter__(self) -> None: - return await self.__original.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.__original.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - return self.__original.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - self.__original.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - return self.__original.borrowed_tokens - - @property - def available_tokens(self) -> float: - return self.__original.available_tokens - - def acquire_nowait(self) -> None: - self.__original.acquire_nowait() - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - self.__original.acquire_on_behalf_of_nowait(borrower) - - async def acquire(self) -> None: - await self.__original.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self.__original.acquire_on_behalf_of(borrower) - - def release(self) -> None: - return self.__original.release() - - def release_on_behalf_of(self, borrower: object) -> None: - return self.__original.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - orig = self.__original.statistics() - return CapacityLimiterStatistics( - borrowed_tokens=orig.borrowed_tokens, - total_tokens=orig.total_tokens, - borrowers=tuple(orig.borrowers), - tasks_waiting=orig.tasks_waiting, - ) - - -_capacity_limiter_wrapper: trio.lowlevel.RunVar = RunVar("_capacity_limiter_wrapper") - - -# -# Signal handling -# - - -class _SignalReceiver: - _iterator: AsyncIterator[int] - - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - - def __enter__(self) -> _SignalReceiver: - self._cm = trio.open_signal_receiver(*self._signals) - self._iterator = self._cm.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_val, exc_tb) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - signum = await self._iterator.__anext__() - return Signals(signum) - - -# -# Testing and debugging -# - - -class TestRunner(abc.TestRunner): - def __init__(self, **options: Any) -> None: - from queue import Queue - - self._call_queue: Queue[Callable[[], object]] = Queue() - self._send_stream: MemoryObjectSendStream | None = None - self._options = options - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> None: - if self._send_stream: - self._send_stream.close() - while self._send_stream is not None: - self._call_queue.get()() - - async def _run_tests_and_fixtures(self) -> None: - self._send_stream, receive_stream = create_memory_object_stream(1) - with receive_stream: - async for coro, outcome_holder in receive_stream: - try: - retval = await coro - except BaseException as exc: - outcome_holder.append(Error(exc)) - else: - outcome_holder.append(Value(retval)) - - def _main_task_finished(self, outcome: object) -> None: - self._send_stream = None - - def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if self._send_stream is None: - trio.lowlevel.start_guest_run( - self._run_tests_and_fixtures, - run_sync_soon_threadsafe=self._call_queue.put, - done_callback=self._main_task_finished, - **self._options, - ) - while self._send_stream is None: - self._call_queue.get()() - - outcome_holder: list[Outcome] = [] - self._send_stream.send_nowait((func(*args, **kwargs), outcome_holder)) - while not outcome_holder: - self._call_queue.get()() - - return outcome_holder[0].unwrap() - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self._call_in_runner_task(asyncgen.asend, None) - - yield fixturevalue - - try: - self._call_in_runner_task(asyncgen.asend, None) - except StopAsyncIteration: - pass - else: - self._call_in_runner_task(asyncgen.aclose) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - return self._call_in_runner_task(fixture_func, **kwargs) - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - self._call_in_runner_task(test_func, **kwargs) - - -class TrioTaskInfo(TaskInfo): - def __init__(self, task: trio.lowlevel.Task): - parent_id = None - if task.parent_nursery and task.parent_nursery.parent_task: - parent_id = id(task.parent_nursery.parent_task) - - super().__init__(id(task), parent_id, task.name, task.coro) - self._task = weakref.proxy(task) - - def has_pending_cancellation(self) -> bool: - try: - return self._task._cancel_status.effectively_cancelled - except ReferenceError: - # If the task is no longer around, it surely doesn't have a cancellation - # pending - return False - - -class TrioBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - return trio.run(func, *args) - - @classmethod - def current_token(cls) -> object: - return trio.lowlevel.current_trio_token() - - @classmethod - def current_time(cls) -> float: - return trio.current_time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return trio.Cancelled - - @classmethod - async def checkpoint(cls) -> None: - await trio.lowlevel.checkpoint() - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - await trio.lowlevel.checkpoint_if_cancelled() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - await trio.lowlevel.cancel_shielded_checkpoint() - - @classmethod - async def sleep(cls, delay: float) -> None: - await trio.sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> abc.CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - return trio.current_effective_deadline() - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - def wrapper() -> T_Retval: - with claim_worker_thread(TrioBackend, token): - return func(*args) - - token = TrioBackend.current_token() - return await run_sync( - wrapper, - abandon_on_cancel=abandon_on_cancel, - limiter=cast(trio.CapacityLimiter, limiter), - ) - - @classmethod - def check_cancelled(cls) -> None: - trio.from_thread.check_cancelled() - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - return trio.from_thread.run(func, *args) - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - return trio.from_thread.run_sync(func, *args) - - @classmethod - def create_blocking_portal(cls) -> abc.BlockingPortal: - return BlockingPortal() - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - def convert_item(item: StrOrBytesPath) -> str: - str_or_bytes = os.fspath(item) - if isinstance(str_or_bytes, str): - return str_or_bytes - else: - return os.fsdecode(str_or_bytes) - - if isinstance(command, (str, bytes, PathLike)): - process = await trio.lowlevel.open_process( - convert_item(command), - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=True, - **kwargs, - ) - else: - process = await trio.lowlevel.open_process( - [convert_item(item) for item in command], - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=False, - **kwargs, - ) - - stdin_stream = SendStreamWrapper(process.stdin) if process.stdin else None - stdout_stream = ReceiveStreamWrapper(process.stdout) if process.stdout else None - stderr_stream = ReceiveStreamWrapper(process.stderr) if process.stderr else None - return Process(process, stdin_stream, stdout_stream, stderr_stream) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - trio.lowlevel.spawn_system_task(_shutdown_process_pool, workers) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - family = socket.AF_INET6 if ":" in host else socket.AF_INET - trio_socket = trio.socket.socket(family) - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - if local_address: - await trio_socket.bind(local_address) - - try: - await trio_socket.connect((host, port)) - except BaseException: - trio_socket.close() - raise - - return SocketStream(trio_socket) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - trio_socket = trio.socket.socket(socket.AF_UNIX) - try: - await trio_socket.connect(path) - except BaseException: - trio_socket.close() - raise - - return UNIXSocketStream(trio_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> abc.SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> abc.SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: socket.AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - trio_socket = trio.socket.socket(family=family, type=socket.SOCK_DGRAM) - - if reuse_port: - trio_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - if local_address: - await trio_socket.bind(local_address) - - if remote_address: - await trio_socket.connect(remote_address) - return ConnectedUDPSocket(trio_socket) - else: - return UDPSocket(trio_socket) - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: None - ) -> abc.UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes - ) -> abc.ConnectedUNIXDatagramSocket: ... - - @classmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - trio_socket = trio.socket.from_stdlib_socket(raw_socket) - - if remote_path: - await trio_socket.connect(remote_path) - return ConnectedUNIXDatagramSocket(trio_socket) - else: - return UNIXDatagramSocket(trio_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> list[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int], - ] - ]: - return await trio.socket.getaddrinfo(host, port, family, type, proto, flags) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await trio.socket.getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: HasFileno | int) -> None: - try: - await wait_readable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("reading from") from None - - @classmethod - async def wait_writable(cls, obj: HasFileno | int) -> None: - try: - await wait_writable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("writing to") from None - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _capacity_limiter_wrapper.get() - except LookupError: - limiter = CapacityLimiter( - original=trio.to_thread.current_default_thread_limiter() - ) - _capacity_limiter_wrapper.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - task = current_task() - return TrioTaskInfo(task) - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - root_task = current_root_task() - assert root_task - task_infos = [TrioTaskInfo(root_task)] - nurseries = root_task.child_nurseries - while nurseries: - new_nurseries: list[trio.Nursery] = [] - for nursery in nurseries: - for task in nursery.child_tasks: - task_infos.append(TrioTaskInfo(task)) - new_nurseries.extend(task.child_nurseries) - - nurseries = new_nurseries - - return task_infos - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - from trio.testing import wait_all_tasks_blocked - - await wait_all_tasks_blocked() - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = TrioBackend diff --git a/contrib/python/anyio/anyio/_core/__init__.py b/contrib/python/anyio/anyio/_core/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/anyio/anyio/_core/__init__.py +++ /dev/null diff --git a/contrib/python/anyio/anyio/_core/_asyncio_selector_thread.py b/contrib/python/anyio/anyio/_core/_asyncio_selector_thread.py deleted file mode 100644 index f4d18cf0429..00000000000 --- a/contrib/python/anyio/anyio/_core/_asyncio_selector_thread.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -import asyncio -import socket -import threading -from collections.abc import Callable -from selectors import EVENT_READ, EVENT_WRITE, DefaultSelector -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - -_selector_lock = threading.Lock() -_selector: Selector | None = None - - -class Selector: - def __init__(self) -> None: - self._thread = threading.Thread(target=self.run, name="AnyIO socket selector") - self._selector = DefaultSelector() - self._send, self._receive = socket.socketpair() - self._send.setblocking(False) - self._receive.setblocking(False) - # This somewhat reduces the amount of memory wasted queueing up data - # for wakeups. With these settings, maximum number of 1-byte sends - # before getting BlockingIOError: - # Linux 4.8: 6 - # macOS (darwin 15.5): 1 - # Windows 10: 525347 - # Windows you're weird. (And on Windows setting SNDBUF to 0 makes send - # blocking, even on non-blocking sockets, so don't do that.) - self._receive.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) - self._send.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) - # On Windows this is a TCP socket so this might matter. On other - # platforms this fails b/c AF_UNIX sockets aren't actually TCP. - try: - self._send.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - except OSError: - pass - - self._selector.register(self._receive, EVENT_READ) - self._closed = False - - def start(self) -> None: - self._thread.start() - threading._register_atexit(self._stop) # type: ignore[attr-defined] - - def _stop(self) -> None: - global _selector - self._closed = True - self._notify_self() - self._send.close() - self._thread.join() - self._selector.unregister(self._receive) - self._receive.close() - self._selector.close() - _selector = None - assert ( - not self._selector.get_map() - ), "selector still has registered file descriptors after shutdown" - - def _notify_self(self) -> None: - try: - self._send.send(b"\x00") - except BlockingIOError: - pass - - def add_reader(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_READ, {EVENT_READ: (loop, callback)}) - else: - if EVENT_READ in key.data: - raise ValueError( - "this file descriptor is already registered for reading" - ) - - key.data[EVENT_READ] = loop, callback - self._selector.modify(fd, key.events | EVENT_READ, key.data) - - self._notify_self() - - def add_writer(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_WRITE, {EVENT_WRITE: (loop, callback)}) - else: - if EVENT_WRITE in key.data: - raise ValueError( - "this file descriptor is already registered for writing" - ) - - key.data[EVENT_WRITE] = loop, callback - self._selector.modify(fd, key.events | EVENT_WRITE, key.data) - - self._notify_self() - - def remove_reader(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_READ: - del key.data[EVENT_READ] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def remove_writer(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_WRITE: - del key.data[EVENT_WRITE] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def run(self) -> None: - while not self._closed: - for key, events in self._selector.select(): - if key.fileobj is self._receive: - try: - while self._receive.recv(4096): - pass - except BlockingIOError: - pass - - continue - - if events & EVENT_READ: - loop, callback = key.data[EVENT_READ] - self.remove_reader(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - if events & EVENT_WRITE: - loop, callback = key.data[EVENT_WRITE] - self.remove_writer(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - -def get_selector() -> Selector: - global _selector - - with _selector_lock: - if _selector is None: - _selector = Selector() - _selector.start() - - return _selector diff --git a/contrib/python/anyio/anyio/_core/_eventloop.py b/contrib/python/anyio/anyio/_core/_eventloop.py deleted file mode 100644 index 6dcb4589818..00000000000 --- a/contrib/python/anyio/anyio/_core/_eventloop.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import math -import sys -import threading -from collections.abc import Awaitable, Callable, Generator -from contextlib import contextmanager -from importlib import import_module -from typing import TYPE_CHECKING, Any, TypeVar - -import sniffio - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from ..abc import AsyncBackend - -# This must be updated when new backends are introduced -BACKENDS = "asyncio", "trio" - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -threadlocals = threading.local() -loaded_backends: dict[str, type[AsyncBackend]] = {} - - -def run( - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - backend: str = "asyncio", - backend_options: dict[str, Any] | None = None, -) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param backend: name of the asynchronous event loop implementation – currently - either ``asyncio`` or ``trio`` - :param backend_options: keyword arguments to call the backend ``run()`` - implementation with (documented :ref:`here <backend options>`) - :return: the return value of the coroutine function - :raises RuntimeError: if an asynchronous event loop is already running in this - thread - :raises LookupError: if the named backend is not found - - """ - try: - asynclib_name = sniffio.current_async_library() - except sniffio.AsyncLibraryNotFoundError: - pass - else: - raise RuntimeError(f"Already running {asynclib_name} in this thread") - - try: - async_backend = get_async_backend(backend) - except ImportError as exc: - raise LookupError(f"No such backend: {backend}") from exc - - token = None - if sniffio.current_async_library_cvar.get(None) is None: - # Since we're in control of the event loop, we can cache the name of the async - # library - token = sniffio.current_async_library_cvar.set(backend) - - try: - backend_options = backend_options or {} - return async_backend.run(func, args, {}, backend_options) - finally: - if token: - sniffio.current_async_library_cvar.reset(token) - - -async def sleep(delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - - """ - return await get_async_backend().sleep(delay) - - -async def sleep_forever() -> None: - """ - Pause the current task until it's cancelled. - - This is a shortcut for ``sleep(math.inf)``. - - .. versionadded:: 3.1 - - """ - await sleep(math.inf) - - -async def sleep_until(deadline: float) -> None: - """ - Pause the current task until the given time. - - :param deadline: the absolute time to wake up at (according to the internal - monotonic clock of the event loop) - - .. versionadded:: 3.1 - - """ - now = current_time() - await sleep(max(deadline - now, 0)) - - -def current_time() -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - - """ - return get_async_backend().current_time() - - -def get_all_backends() -> tuple[str, ...]: - """Return a tuple of the names of all built-in backends.""" - return BACKENDS - - -def get_cancelled_exc_class() -> type[BaseException]: - """Return the current async library's cancellation exception class.""" - return get_async_backend().cancelled_exception_class() - - -# -# Private API -# - - -@contextmanager -def claim_worker_thread( - backend_class: type[AsyncBackend], token: object -) -> Generator[Any, None, None]: - threadlocals.current_async_backend = backend_class - threadlocals.current_token = token - try: - yield - finally: - del threadlocals.current_async_backend - del threadlocals.current_token - - -def get_async_backend(asynclib_name: str | None = None) -> type[AsyncBackend]: - if asynclib_name is None: - asynclib_name = sniffio.current_async_library() - - # We use our own dict instead of sys.modules to get the already imported back-end - # class because the appropriate modules in sys.modules could potentially be only - # partially initialized - try: - return loaded_backends[asynclib_name] - except KeyError: - module = import_module(f"anyio._backends._{asynclib_name}") - loaded_backends[asynclib_name] = module.backend_class - return module.backend_class diff --git a/contrib/python/anyio/anyio/_core/_exceptions.py b/contrib/python/anyio/anyio/_core/_exceptions.py deleted file mode 100644 index 16b94482c06..00000000000 --- a/contrib/python/anyio/anyio/_core/_exceptions.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Generator -from textwrap import dedent -from typing import Any - -if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup - - -class BrokenResourceError(Exception): - """ - Raised when trying to use a resource that has been rendered unusable due to external - causes (e.g. a send stream whose peer has disconnected). - """ - - -class BrokenWorkerProcess(Exception): - """ - Raised by :meth:`~anyio.to_process.run_sync` if the worker process terminates abruptly or - otherwise misbehaves. - """ - - -class BrokenWorkerIntepreter(Exception): - """ - Raised by :meth:`~anyio.to_interpreter.run_sync` if an unexpected exception is - raised in the subinterpreter. - """ - - def __init__(self, excinfo: Any): - # This was adapted from concurrent.futures.interpreter.ExecutionFailed - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f"{excinfo.type.__name__}: {excinfo.msg}" - else: - msg = excinfo.type.__name__ or excinfo.msg - - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self) -> str: - try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return dedent( - f""" - {super().__str__()} - - Uncaught in the interpreter: - - {formatted} - """.strip() - ) - - -class BusyResourceError(Exception): - """ - Raised when two tasks are trying to read from or write to the same resource - concurrently. - """ - - def __init__(self, action: str): - super().__init__(f"Another task is already {action} this resource") - - -class ClosedResourceError(Exception): - """Raised when trying to use a resource that has been closed.""" - - -class DelimiterNotFound(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - maximum number of bytes has been read without the delimiter being found. - """ - - def __init__(self, max_bytes: int) -> None: - super().__init__( - f"The delimiter was not found among the first {max_bytes} bytes" - ) - - -class EndOfStream(Exception): - """ - Raised when trying to read from a stream that has been closed from the other end. - """ - - -class IncompleteRead(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_exactly` or - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - connection is closed before the requested amount of bytes has been read. - """ - - def __init__(self) -> None: - super().__init__( - "The stream was closed before the read operation could be completed" - ) - - -class TypedAttributeLookupError(LookupError): - """ - Raised by :meth:`~anyio.TypedAttributeProvider.extra` when the given typed attribute - is not found and no default value has been given. - """ - - -class WouldBlock(Exception): - """Raised by ``X_nowait`` functions if ``X()`` would block.""" - - -def iterate_exceptions( - exception: BaseException, -) -> Generator[BaseException, None, None]: - if isinstance(exception, BaseExceptionGroup): - for exc in exception.exceptions: - yield from iterate_exceptions(exc) - else: - yield exception diff --git a/contrib/python/anyio/anyio/_core/_fileio.py b/contrib/python/anyio/anyio/_core/_fileio.py deleted file mode 100644 index 4e34f2addc5..00000000000 --- a/contrib/python/anyio/anyio/_core/_fileio.py +++ /dev/null @@ -1,729 +0,0 @@ -from __future__ import annotations - -import os -import pathlib -import sys -from collections.abc import ( - AsyncIterator, - Callable, - Iterable, - Iterator, - Sequence, -) -from dataclasses import dataclass -from functools import partial -from os import PathLike -from typing import ( - IO, - TYPE_CHECKING, - Any, - AnyStr, - Final, - Generic, - overload, -) - -from .. import to_thread -from ..abc import AsyncResource - -if TYPE_CHECKING: - from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer -else: - ReadableBuffer = OpenBinaryMode = OpenTextMode = WriteableBuffer = object - - -class AsyncFile(AsyncResource, Generic[AnyStr]): - """ - An asynchronous file object. - - This class wraps a standard file object and provides async friendly versions of the - following blocking methods (where available on the original file object): - - * read - * read1 - * readline - * readlines - * readinto - * readinto1 - * write - * writelines - * truncate - * seek - * tell - * flush - - All other methods are directly passed through. - - This class supports the asynchronous context manager protocol which closes the - underlying file at the end of the context block. - - This class also supports asynchronous iteration:: - - async with await open_file(...) as f: - async for line in f: - print(line) - """ - - def __init__(self, fp: IO[AnyStr]) -> None: - self._fp: Any = fp - - def __getattr__(self, name: str) -> object: - return getattr(self._fp, name) - - @property - def wrapped(self) -> IO[AnyStr]: - """The wrapped file object.""" - return self._fp - - async def __aiter__(self) -> AsyncIterator[AnyStr]: - while True: - line = await self.readline() - if line: - yield line - else: - break - - async def aclose(self) -> None: - return await to_thread.run_sync(self._fp.close) - - async def read(self, size: int = -1) -> AnyStr: - return await to_thread.run_sync(self._fp.read, size) - - async def read1(self: AsyncFile[bytes], size: int = -1) -> bytes: - return await to_thread.run_sync(self._fp.read1, size) - - async def readline(self) -> AnyStr: - return await to_thread.run_sync(self._fp.readline) - - async def readlines(self) -> list[AnyStr]: - return await to_thread.run_sync(self._fp.readlines) - - async def readinto(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto, b) - - async def readinto1(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto1, b) - - @overload - async def write(self: AsyncFile[bytes], b: ReadableBuffer) -> int: ... - - @overload - async def write(self: AsyncFile[str], b: str) -> int: ... - - async def write(self, b: ReadableBuffer | str) -> int: - return await to_thread.run_sync(self._fp.write, b) - - @overload - async def writelines( - self: AsyncFile[bytes], lines: Iterable[ReadableBuffer] - ) -> None: ... - - @overload - async def writelines(self: AsyncFile[str], lines: Iterable[str]) -> None: ... - - async def writelines(self, lines: Iterable[ReadableBuffer] | Iterable[str]) -> None: - return await to_thread.run_sync(self._fp.writelines, lines) - - async def truncate(self, size: int | None = None) -> int: - return await to_thread.run_sync(self._fp.truncate, size) - - async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: - return await to_thread.run_sync(self._fp.seek, offset, whence) - - async def tell(self) -> int: - return await to_thread.run_sync(self._fp.tell) - - async def flush(self) -> None: - return await to_thread.run_sync(self._fp.flush) - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., -) -> AsyncFile[bytes]: ... - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., -) -> AsyncFile[str]: ... - - -async def open_file( - file: str | PathLike[str] | int, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - closefd: bool = True, - opener: Callable[[str, int], int] | None = None, -) -> AsyncFile[Any]: - """ - Open a file asynchronously. - - The arguments are exactly the same as for the builtin :func:`open`. - - :return: an asynchronous file object - - """ - fp = await to_thread.run_sync( - open, file, mode, buffering, encoding, errors, newline, closefd, opener - ) - return AsyncFile(fp) - - -def wrap_file(file: IO[AnyStr]) -> AsyncFile[AnyStr]: - """ - Wrap an existing file as an asynchronous file. - - :param file: an existing file-like object - :return: an asynchronous file object - - """ - return AsyncFile(file) - - -@dataclass(eq=False) -class _PathIterator(AsyncIterator["Path"]): - iterator: Iterator[PathLike[str]] - - async def __anext__(self) -> Path: - nextval = await to_thread.run_sync( - next, self.iterator, None, abandon_on_cancel=True - ) - if nextval is None: - raise StopAsyncIteration from None - - return Path(nextval) - - -class Path: - """ - An asynchronous version of :class:`pathlib.Path`. - - This class cannot be substituted for :class:`pathlib.Path` or - :class:`pathlib.PurePath`, but it is compatible with the :class:`os.PathLike` - interface. - - It implements the Python 3.10 version of :class:`pathlib.Path` interface, except for - the deprecated :meth:`~pathlib.Path.link_to` method. - - Some methods may be unavailable or have limited functionality, based on the Python - version: - - * :meth:`~pathlib.Path.copy` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.copy_into` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.from_uri` (available on Python 3.13 or later) - * :meth:`~pathlib.Path.full_match` (available on Python 3.13 or later) - * :meth:`~pathlib.Path.is_junction` (available on Python 3.12 or later) - * :meth:`~pathlib.Path.match` (the ``case_sensitive`` paramater is only available on - Python 3.13 or later) - * :meth:`~pathlib.Path.move` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.move_into` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.relative_to` (the ``walk_up`` parameter is only available on - Python 3.12 or later) - * :meth:`~pathlib.Path.walk` (available on Python 3.12 or later) - - Any methods that do disk I/O need to be awaited on. These methods are: - - * :meth:`~pathlib.Path.absolute` - * :meth:`~pathlib.Path.chmod` - * :meth:`~pathlib.Path.cwd` - * :meth:`~pathlib.Path.exists` - * :meth:`~pathlib.Path.expanduser` - * :meth:`~pathlib.Path.group` - * :meth:`~pathlib.Path.hardlink_to` - * :meth:`~pathlib.Path.home` - * :meth:`~pathlib.Path.is_block_device` - * :meth:`~pathlib.Path.is_char_device` - * :meth:`~pathlib.Path.is_dir` - * :meth:`~pathlib.Path.is_fifo` - * :meth:`~pathlib.Path.is_file` - * :meth:`~pathlib.Path.is_junction` - * :meth:`~pathlib.Path.is_mount` - * :meth:`~pathlib.Path.is_socket` - * :meth:`~pathlib.Path.is_symlink` - * :meth:`~pathlib.Path.lchmod` - * :meth:`~pathlib.Path.lstat` - * :meth:`~pathlib.Path.mkdir` - * :meth:`~pathlib.Path.open` - * :meth:`~pathlib.Path.owner` - * :meth:`~pathlib.Path.read_bytes` - * :meth:`~pathlib.Path.read_text` - * :meth:`~pathlib.Path.readlink` - * :meth:`~pathlib.Path.rename` - * :meth:`~pathlib.Path.replace` - * :meth:`~pathlib.Path.resolve` - * :meth:`~pathlib.Path.rmdir` - * :meth:`~pathlib.Path.samefile` - * :meth:`~pathlib.Path.stat` - * :meth:`~pathlib.Path.symlink_to` - * :meth:`~pathlib.Path.touch` - * :meth:`~pathlib.Path.unlink` - * :meth:`~pathlib.Path.walk` - * :meth:`~pathlib.Path.write_bytes` - * :meth:`~pathlib.Path.write_text` - - Additionally, the following methods return an async iterator yielding - :class:`~.Path` objects: - - * :meth:`~pathlib.Path.glob` - * :meth:`~pathlib.Path.iterdir` - * :meth:`~pathlib.Path.rglob` - """ - - __slots__ = "_path", "__weakref__" - - __weakref__: Any - - def __init__(self, *args: str | PathLike[str]) -> None: - self._path: Final[pathlib.Path] = pathlib.Path(*args) - - def __fspath__(self) -> str: - return self._path.__fspath__() - - def __str__(self) -> str: - return self._path.__str__() - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.as_posix()!r})" - - def __bytes__(self) -> bytes: - return self._path.__bytes__() - - def __hash__(self) -> int: - return self._path.__hash__() - - def __eq__(self, other: object) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__eq__(target) - - def __lt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__lt__(target) - - def __le__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__le__(target) - - def __gt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__gt__(target) - - def __ge__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__ge__(target) - - def __truediv__(self, other: str | PathLike[str]) -> Path: - return Path(self._path / other) - - def __rtruediv__(self, other: str | PathLike[str]) -> Path: - return Path(other) / self - - @property - def parts(self) -> tuple[str, ...]: - return self._path.parts - - @property - def drive(self) -> str: - return self._path.drive - - @property - def root(self) -> str: - return self._path.root - - @property - def anchor(self) -> str: - return self._path.anchor - - @property - def parents(self) -> Sequence[Path]: - return tuple(Path(p) for p in self._path.parents) - - @property - def parent(self) -> Path: - return Path(self._path.parent) - - @property - def name(self) -> str: - return self._path.name - - @property - def suffix(self) -> str: - return self._path.suffix - - @property - def suffixes(self) -> list[str]: - return self._path.suffixes - - @property - def stem(self) -> str: - return self._path.stem - - async def absolute(self) -> Path: - path = await to_thread.run_sync(self._path.absolute) - return Path(path) - - def as_posix(self) -> str: - return self._path.as_posix() - - def as_uri(self) -> str: - return self._path.as_uri() - - if sys.version_info >= (3, 13): - parser = pathlib.Path.parser - - @classmethod - def from_uri(cls, uri: str) -> Path: - return Path(pathlib.Path.from_uri(uri)) - - def full_match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.full_match(path_pattern, case_sensitive=case_sensitive) - - def match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.match(path_pattern, case_sensitive=case_sensitive) - else: - - def match(self, path_pattern: str) -> bool: - return self._path.match(path_pattern) - - if sys.version_info >= (3, 14): - - async def copy( - self, - target: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - dirs_exist_ok: bool = False, - preserve_metadata: bool = False, - ) -> Path: - func = partial( - self._path.copy, - follow_symlinks=follow_symlinks, - dirs_exist_ok=dirs_exist_ok, - preserve_metadata=preserve_metadata, - ) - return Path(await to_thread.run_sync(func, target)) - - async def copy_into( - self, - target_dir: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - dirs_exist_ok: bool = False, - preserve_metadata: bool = False, - ) -> Path: - func = partial( - self._path.copy_into, - follow_symlinks=follow_symlinks, - dirs_exist_ok=dirs_exist_ok, - preserve_metadata=preserve_metadata, - ) - return Path(await to_thread.run_sync(func, target_dir)) - - async def move(self, target: str | os.PathLike[str]) -> Path: - # Upstream does not handle anyio.Path properly as a PathLike - target = pathlib.Path(target) - return Path(await to_thread.run_sync(self._path.move, target)) - - async def move_into( - self, - target_dir: str | os.PathLike[str], - ) -> Path: - return Path(await to_thread.run_sync(self._path.move_into, target_dir)) - - def is_relative_to(self, other: str | PathLike[str]) -> bool: - try: - self.relative_to(other) - return True - except ValueError: - return False - - async def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: - func = partial(os.chmod, follow_symlinks=follow_symlinks) - return await to_thread.run_sync(func, self._path, mode) - - @classmethod - async def cwd(cls) -> Path: - path = await to_thread.run_sync(pathlib.Path.cwd) - return cls(path) - - async def exists(self) -> bool: - return await to_thread.run_sync(self._path.exists, abandon_on_cancel=True) - - async def expanduser(self) -> Path: - return Path( - await to_thread.run_sync(self._path.expanduser, abandon_on_cancel=True) - ) - - def glob(self, pattern: str) -> AsyncIterator[Path]: - gen = self._path.glob(pattern) - return _PathIterator(gen) - - async def group(self) -> str: - return await to_thread.run_sync(self._path.group, abandon_on_cancel=True) - - async def hardlink_to( - self, target: str | bytes | PathLike[str] | PathLike[bytes] - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(os.link, target, self) - - @classmethod - async def home(cls) -> Path: - home_path = await to_thread.run_sync(pathlib.Path.home) - return cls(home_path) - - def is_absolute(self) -> bool: - return self._path.is_absolute() - - async def is_block_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_block_device, abandon_on_cancel=True - ) - - async def is_char_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_char_device, abandon_on_cancel=True - ) - - async def is_dir(self) -> bool: - return await to_thread.run_sync(self._path.is_dir, abandon_on_cancel=True) - - async def is_fifo(self) -> bool: - return await to_thread.run_sync(self._path.is_fifo, abandon_on_cancel=True) - - async def is_file(self) -> bool: - return await to_thread.run_sync(self._path.is_file, abandon_on_cancel=True) - - if sys.version_info >= (3, 12): - - async def is_junction(self) -> bool: - return await to_thread.run_sync(self._path.is_junction) - - async def is_mount(self) -> bool: - return await to_thread.run_sync( - os.path.ismount, self._path, abandon_on_cancel=True - ) - - def is_reserved(self) -> bool: - return self._path.is_reserved() - - async def is_socket(self) -> bool: - return await to_thread.run_sync(self._path.is_socket, abandon_on_cancel=True) - - async def is_symlink(self) -> bool: - return await to_thread.run_sync(self._path.is_symlink, abandon_on_cancel=True) - - def iterdir(self) -> AsyncIterator[Path]: - gen = self._path.iterdir() - return _PathIterator(gen) - - def joinpath(self, *args: str | PathLike[str]) -> Path: - return Path(self._path.joinpath(*args)) - - async def lchmod(self, mode: int) -> None: - await to_thread.run_sync(self._path.lchmod, mode) - - async def lstat(self) -> os.stat_result: - return await to_thread.run_sync(self._path.lstat, abandon_on_cancel=True) - - async def mkdir( - self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False - ) -> None: - await to_thread.run_sync(self._path.mkdir, mode, parents, exist_ok) - - @overload - async def open( - self, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[bytes]: ... - - @overload - async def open( - self, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[str]: ... - - async def open( - self, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> AsyncFile[Any]: - fp = await to_thread.run_sync( - self._path.open, mode, buffering, encoding, errors, newline - ) - return AsyncFile(fp) - - async def owner(self) -> str: - return await to_thread.run_sync(self._path.owner, abandon_on_cancel=True) - - async def read_bytes(self) -> bytes: - return await to_thread.run_sync(self._path.read_bytes) - - async def read_text( - self, encoding: str | None = None, errors: str | None = None - ) -> str: - return await to_thread.run_sync(self._path.read_text, encoding, errors) - - if sys.version_info >= (3, 12): - - def relative_to( - self, *other: str | PathLike[str], walk_up: bool = False - ) -> Path: - return Path(self._path.relative_to(*other, walk_up=walk_up)) - - else: - - def relative_to(self, *other: str | PathLike[str]) -> Path: - return Path(self._path.relative_to(*other)) - - async def readlink(self) -> Path: - target = await to_thread.run_sync(os.readlink, self._path) - return Path(target) - - async def rename(self, target: str | pathlib.PurePath | Path) -> Path: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.rename, target) - return Path(target) - - async def replace(self, target: str | pathlib.PurePath | Path) -> Path: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.replace, target) - return Path(target) - - async def resolve(self, strict: bool = False) -> Path: - func = partial(self._path.resolve, strict=strict) - return Path(await to_thread.run_sync(func, abandon_on_cancel=True)) - - def rglob(self, pattern: str) -> AsyncIterator[Path]: - gen = self._path.rglob(pattern) - return _PathIterator(gen) - - async def rmdir(self) -> None: - await to_thread.run_sync(self._path.rmdir) - - async def samefile(self, other_path: str | PathLike[str]) -> bool: - if isinstance(other_path, Path): - other_path = other_path._path - - return await to_thread.run_sync( - self._path.samefile, other_path, abandon_on_cancel=True - ) - - async def stat(self, *, follow_symlinks: bool = True) -> os.stat_result: - func = partial(os.stat, follow_symlinks=follow_symlinks) - return await to_thread.run_sync(func, self._path, abandon_on_cancel=True) - - async def symlink_to( - self, - target: str | bytes | PathLike[str] | PathLike[bytes], - target_is_directory: bool = False, - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.symlink_to, target, target_is_directory) - - async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: - await to_thread.run_sync(self._path.touch, mode, exist_ok) - - async def unlink(self, missing_ok: bool = False) -> None: - try: - await to_thread.run_sync(self._path.unlink) - except FileNotFoundError: - if not missing_ok: - raise - - if sys.version_info >= (3, 12): - - async def walk( - self, - top_down: bool = True, - on_error: Callable[[OSError], object] | None = None, - follow_symlinks: bool = False, - ) -> AsyncIterator[tuple[Path, list[str], list[str]]]: - def get_next_value() -> tuple[pathlib.Path, list[str], list[str]] | None: - try: - return next(gen) - except StopIteration: - return None - - gen = self._path.walk(top_down, on_error, follow_symlinks) - while True: - value = await to_thread.run_sync(get_next_value) - if value is None: - return - - root, dirs, paths = value - yield Path(root), dirs, paths - - def with_name(self, name: str) -> Path: - return Path(self._path.with_name(name)) - - def with_stem(self, stem: str) -> Path: - return Path(self._path.with_name(stem + self._path.suffix)) - - def with_suffix(self, suffix: str) -> Path: - return Path(self._path.with_suffix(suffix)) - - def with_segments(self, *pathsegments: str | PathLike[str]) -> Path: - return Path(*pathsegments) - - async def write_bytes(self, data: bytes) -> int: - return await to_thread.run_sync(self._path.write_bytes, data) - - async def write_text( - self, - data: str, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> int: - # Path.write_text() does not support the "newline" parameter before Python 3.10 - def sync_write_text() -> int: - with self._path.open( - "w", encoding=encoding, errors=errors, newline=newline - ) as fp: - return fp.write(data) - - return await to_thread.run_sync(sync_write_text) - - -PathLike.register(Path) diff --git a/contrib/python/anyio/anyio/_core/_resources.py b/contrib/python/anyio/anyio/_core/_resources.py deleted file mode 100644 index b9a5344aef2..00000000000 --- a/contrib/python/anyio/anyio/_core/_resources.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from ..abc import AsyncResource -from ._tasks import CancelScope - - -async def aclose_forcefully(resource: AsyncResource) -> None: - """ - Close an asynchronous resource in a cancelled scope. - - Doing this closes the resource without waiting on anything. - - :param resource: the resource to close - - """ - with CancelScope() as scope: - scope.cancel() - await resource.aclose() diff --git a/contrib/python/anyio/anyio/_core/_signals.py b/contrib/python/anyio/anyio/_core/_signals.py deleted file mode 100644 index f3451d302f5..00000000000 --- a/contrib/python/anyio/anyio/_core/_signals.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import AbstractContextManager -from signal import Signals - -from ._eventloop import get_async_backend - - -def open_signal_receiver( - *signals: Signals, -) -> AbstractContextManager[AsyncIterator[Signals]]: - """ - Start receiving operating system signals. - - :param signals: signals to receive (e.g. ``signal.SIGINT``) - :return: an asynchronous context manager for an asynchronous iterator which yields - signal numbers - - .. warning:: Windows does not support signals natively so it is best to avoid - relying on this in cross-platform applications. - - .. warning:: On asyncio, this permanently replaces any previous signal handler for - the given signals, as set via :meth:`~asyncio.loop.add_signal_handler`. - - """ - return get_async_backend().open_signal_receiver(*signals) diff --git a/contrib/python/anyio/anyio/_core/_sockets.py b/contrib/python/anyio/anyio/_core/_sockets.py deleted file mode 100644 index a822d060d72..00000000000 --- a/contrib/python/anyio/anyio/_core/_sockets.py +++ /dev/null @@ -1,787 +0,0 @@ -from __future__ import annotations - -import errno -import os -import socket -import ssl -import stat -import sys -from collections.abc import Awaitable -from ipaddress import IPv6Address, ip_address -from os import PathLike, chmod -from socket import AddressFamily, SocketKind -from typing import TYPE_CHECKING, Any, Literal, cast, overload - -from .. import to_thread -from ..abc import ( - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPAddressType, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, -) -from ..streams.stapled import MultiListener -from ..streams.tls import TLSStream -from ._eventloop import get_async_backend -from ._resources import aclose_forcefully -from ._synchronization import Event -from ._tasks import create_task_group, move_on_after - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -if sys.version_info < (3, 13): - from typing_extensions import deprecated -else: - from warnings import deprecated - -IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41) # https://bugs.python.org/issue29515 - -AnyIPAddressFamily = Literal[ - AddressFamily.AF_UNSPEC, AddressFamily.AF_INET, AddressFamily.AF_INET6 -] -IPAddressFamily = Literal[AddressFamily.AF_INET, AddressFamily.AF_INET6] - - -# tls_hostname given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str, - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# ssl_context given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - ssl_context: ssl.SSLContext, - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=True -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - tls: Literal[True], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=False -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - tls: Literal[False], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -# No TLS arguments -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = None, - tls: bool = False, - ssl_context: ssl.SSLContext | None = None, - tls_standard_compatible: bool = True, - tls_hostname: str | None = None, - happy_eyeballs_delay: float = 0.25, -) -> SocketStream | TLSStream: - """ - Connect to a host using the TCP protocol. - - This function implements the stateless version of the Happy Eyeballs algorithm (RFC - 6555). If ``remote_host`` is a host name that resolves to multiple IP addresses, - each one is tried until one connection attempt succeeds. If the first attempt does - not connected within 250 milliseconds, a second attempt is started using the next - address in the list, and so on. On IPv6 enabled systems, an IPv6 address (if - available) is tried first. - - When the connection has been established, a TLS handshake will be done if either - ``ssl_context`` or ``tls_hostname`` is not ``None``, or if ``tls`` is ``True``. - - :param remote_host: the IP address or host name to connect to - :param remote_port: port on the target host to connect to - :param local_host: the interface address or name to bind the socket to before - connecting - :param tls: ``True`` to do a TLS handshake with the connected stream and return a - :class:`~anyio.streams.tls.TLSStream` instead - :param ssl_context: the SSL context object to use (if omitted, a default context is - created) - :param tls_standard_compatible: If ``True``, performs the TLS shutdown handshake - before closing the stream and requires that the server does this as well. - Otherwise, :exc:`~ssl.SSLEOFError` may be raised during reads from the stream. - Some protocols, such as HTTP, require this option to be ``False``. - See :meth:`~ssl.SSLContext.wrap_socket` for details. - :param tls_hostname: host name to check the server certificate against (defaults to - the value of ``remote_host``) - :param happy_eyeballs_delay: delay (in seconds) before starting the next connection - attempt - :return: a socket stream object if no TLS handshake was done, otherwise a TLS stream - :raises OSError: if the connection attempt fails - - """ - # Placed here due to https://github.com/python/mypy/issues/7057 - connected_stream: SocketStream | None = None - - async def try_connect(remote_host: str, event: Event) -> None: - nonlocal connected_stream - try: - stream = await asynclib.connect_tcp(remote_host, remote_port, local_address) - except OSError as exc: - oserrors.append(exc) - return - else: - if connected_stream is None: - connected_stream = stream - tg.cancel_scope.cancel() - else: - await stream.aclose() - finally: - event.set() - - asynclib = get_async_backend() - local_address: IPSockAddrType | None = None - family = socket.AF_UNSPEC - if local_host: - gai_res = await getaddrinfo(str(local_host), None) - family, *_, local_address = gai_res[0] - - target_host = str(remote_host) - try: - addr_obj = ip_address(remote_host) - except ValueError: - addr_obj = None - - if addr_obj is not None: - if isinstance(addr_obj, IPv6Address): - target_addrs = [(socket.AF_INET6, addr_obj.compressed)] - else: - target_addrs = [(socket.AF_INET, addr_obj.compressed)] - else: - # getaddrinfo() will raise an exception if name resolution fails - gai_res = await getaddrinfo( - target_host, remote_port, family=family, type=socket.SOCK_STREAM - ) - - # Organize the list so that the first address is an IPv6 address (if available) - # and the second one is an IPv4 addresses. The rest can be in whatever order. - v6_found = v4_found = False - target_addrs = [] - for af, *rest, sa in gai_res: - if af == socket.AF_INET6 and not v6_found: - v6_found = True - target_addrs.insert(0, (af, sa[0])) - elif af == socket.AF_INET and not v4_found and v6_found: - v4_found = True - target_addrs.insert(1, (af, sa[0])) - else: - target_addrs.append((af, sa[0])) - - oserrors: list[OSError] = [] - async with create_task_group() as tg: - for i, (af, addr) in enumerate(target_addrs): - event = Event() - tg.start_soon(try_connect, addr, event) - with move_on_after(happy_eyeballs_delay): - await event.wait() - - if connected_stream is None: - cause = ( - oserrors[0] - if len(oserrors) == 1 - else ExceptionGroup("multiple connection attempts failed", oserrors) - ) - raise OSError("All connection attempts failed") from cause - - if tls or tls_hostname or ssl_context: - try: - return await TLSStream.wrap( - connected_stream, - server_side=False, - hostname=tls_hostname or str(remote_host), - ssl_context=ssl_context, - standard_compatible=tls_standard_compatible, - ) - except BaseException: - await aclose_forcefully(connected_stream) - raise - - return connected_stream - - -async def connect_unix(path: str | bytes | PathLike[Any]) -> UNIXSocketStream: - """ - Connect to the given UNIX socket. - - Not available on Windows. - - :param path: path to the socket - :return: a socket stream object - - """ - path = os.fspath(path) - return await get_async_backend().connect_unix(path) - - -async def create_tcp_listener( - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - family: AnyIPAddressFamily = socket.AddressFamily.AF_UNSPEC, - backlog: int = 65536, - reuse_port: bool = False, -) -> MultiListener[SocketStream]: - """ - Create a TCP socket listener. - - :param local_port: port number to listen on - :param local_host: IP address of the interface to listen on. If omitted, listen on - all IPv4 and IPv6 interfaces. To listen on all interfaces on a specific address - family, use ``0.0.0.0`` for IPv4 or ``::`` for IPv6. - :param family: address family (used if ``local_host`` was omitted) - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a list of listener objects - - """ - asynclib = get_async_backend() - backlog = min(backlog, 65536) - local_host = str(local_host) if local_host is not None else None - gai_res = await getaddrinfo( - local_host, - local_port, - family=family, - type=socket.SocketKind.SOCK_STREAM if sys.platform == "win32" else 0, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - listeners: list[SocketListener] = [] - try: - # The set() is here to work around a glibc bug: - # https://sourceware.org/bugzilla/show_bug.cgi?id=14969 - sockaddr: tuple[str, int] | tuple[str, int, int, int] - for fam, kind, *_, sockaddr in sorted(set(gai_res)): - # Workaround for an uvloop bug where we don't get the correct scope ID for - # IPv6 link-local addresses when passing type=socket.SOCK_STREAM to - # getaddrinfo(): https://github.com/MagicStack/uvloop/issues/539 - if sys.platform != "win32" and kind is not SocketKind.SOCK_STREAM: - continue - - raw_socket = socket.socket(fam) - raw_socket.setblocking(False) - - # For Windows, enable exclusive address use. For others, enable address - # reuse. - if sys.platform == "win32": - raw_socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - else: - raw_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - if reuse_port: - raw_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - # If only IPv6 was requested, disable dual stack operation - if fam == socket.AF_INET6: - raw_socket.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) - - # Workaround for #554 - if "%" in sockaddr[0]: - addr, scope_id = sockaddr[0].split("%", 1) - sockaddr = (addr, sockaddr[1], 0, int(scope_id)) - - raw_socket.bind(sockaddr) - raw_socket.listen(backlog) - listener = asynclib.create_tcp_listener(raw_socket) - listeners.append(listener) - except BaseException: - for listener in listeners: - await listener.aclose() - - raise - - return MultiListener(listeners) - - -async def create_unix_listener( - path: str | bytes | PathLike[Any], - *, - mode: int | None = None, - backlog: int = 65536, -) -> SocketListener: - """ - Create a UNIX socket listener. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :return: a listener object - - .. versionchanged:: 3.0 - If a socket already exists on the file system in the given path, it will be - removed first. - - """ - backlog = min(backlog, 65536) - raw_socket = await setup_unix_local_socket(path, mode, socket.SOCK_STREAM) - try: - raw_socket.listen(backlog) - return get_async_backend().create_unix_listener(raw_socket) - except BaseException: - raw_socket.close() - raise - - -async def create_udp_socket( - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> UDPSocket: - """ - Create a UDP socket. - - If ``port`` has been given, the socket will be bound to this port on the local - machine, making this socket suitable for providing UDP based services. - - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a UDP socket - - """ - if family is AddressFamily.AF_UNSPEC and not local_host: - raise ValueError('Either "family" or "local_host" must be given') - - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - elif family is AddressFamily.AF_INET6: - local_address = ("::", 0) - else: - local_address = ("0.0.0.0", 0) - - sock = await get_async_backend().create_udp_socket( - family, local_address, None, reuse_port - ) - return cast(UDPSocket, sock) - - -async def create_connected_udp_socket( - remote_host: IPAddressType, - remote_port: int, - *, - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> ConnectedUDPSocket: - """ - Create a connected UDP socket. - - Connected UDP sockets can only communicate with the specified remote host/port, an - any packets sent from other sources are dropped. - - :param remote_host: remote host to set as the default target - :param remote_port: port on the remote host to set as the default target - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` or ``remote_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a connected UDP socket - - """ - local_address = None - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - - gai_res = await getaddrinfo( - str(remote_host), remote_port, family=family, type=socket.SOCK_DGRAM - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - remote_address = gai_res[0][-1] - - sock = await get_async_backend().create_udp_socket( - family, local_address, remote_address, reuse_port - ) - return cast(ConnectedUDPSocket, sock) - - -async def create_unix_datagram_socket( - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> UNIXDatagramSocket: - """ - Create a UNIX datagram socket. - - Not available on Windows. - - If ``local_path`` has been given, the socket will be bound to this path, making this - socket suitable for receiving datagrams from other processes. Other processes can - send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a UNIX datagram socket - - """ - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket(raw_socket, None) - - -async def create_connected_unix_datagram_socket( - remote_path: str | bytes | PathLike[Any], - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> ConnectedUNIXDatagramSocket: - """ - Create a connected UNIX datagram socket. - - Connected datagram sockets can only communicate with the specified remote path. - - If ``local_path`` has been given, the socket will be bound to this path, making - this socket suitable for receiving datagrams from other processes. Other processes - can send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param remote_path: the path to set as the default target - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a connected UNIX datagram socket - - """ - remote_path = os.fspath(remote_path) - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket( - raw_socket, remote_path - ) - - -async def getaddrinfo( - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, -) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]]: - """ - Look up a numeric IP address given a host name. - - Internationalized domain names are translated according to the (non-transitional) - IDNA 2008 standard. - - .. note:: 4-tuple IPv6 socket addresses are automatically converted to 2-tuples of - (host, port), unlike what :func:`socket.getaddrinfo` does. - - :param host: host name - :param port: port number - :param family: socket family (`'AF_INET``, ...) - :param type: socket type (``SOCK_STREAM``, ...) - :param proto: protocol number - :param flags: flags to pass to upstream ``getaddrinfo()`` - :return: list of tuples containing (family, type, proto, canonname, sockaddr) - - .. seealso:: :func:`socket.getaddrinfo` - - """ - # Handle unicode hostnames - if isinstance(host, str): - try: - encoded_host: bytes | None = host.encode("ascii") - except UnicodeEncodeError: - import idna - - encoded_host = idna.encode(host, uts46=True) - else: - encoded_host = host - - gai_res = await get_async_backend().getaddrinfo( - encoded_host, port, family=family, type=type, proto=proto, flags=flags - ) - return [ - (family, type, proto, canonname, convert_ipv6_sockaddr(sockaddr)) - for family, type, proto, canonname, sockaddr in gai_res - ] - - -def getnameinfo(sockaddr: IPSockAddrType, flags: int = 0) -> Awaitable[tuple[str, str]]: - """ - Look up the host name of an IP address. - - :param sockaddr: socket address (e.g. (ipaddress, port) for IPv4) - :param flags: flags to pass to upstream ``getnameinfo()`` - :return: a tuple of (host name, service name) - - .. seealso:: :func:`socket.getnameinfo` - - """ - return get_async_backend().getnameinfo(sockaddr, flags) - - -@deprecated("This function is deprecated; use `wait_readable` instead") -def wait_socket_readable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_readable` instead. - - Wait until the given socket has data to be read. - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become readable - - """ - return get_async_backend().wait_readable(sock.fileno()) - - -@deprecated("This function is deprecated; use `wait_writable` instead") -def wait_socket_writable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_writable` instead. - - Wait until the given socket can be written to. - - This does **NOT** work on Windows when using the asyncio backend with a proactor - event loop (default on py3.8+). - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become writable - - """ - return get_async_backend().wait_writable(sock.fileno()) - - -def wait_readable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object has data to be read. - - On Unix systems, ``obj`` must either be an integer file descriptor, or else an - object with a ``.fileno()`` method which returns an integer file descriptor. Any - kind of file descriptor can be passed, though the exact semantics will depend on - your kernel. For example, this probably won't do anything useful for on-disk files. - - On Windows systems, ``obj`` must either be an integer ``SOCKET`` handle, or else an - object with a ``.fileno()`` method which returns an integer ``SOCKET`` handle. File - descriptors aren't supported, and neither are handles that refer to anything besides - a ``SOCKET``. - - On backends where this functionality is not natively provided (asyncio - ``ProactorEventLoop`` on Windows), it is provided using a separate selector thread - which is set to shut down when the interpreter shuts down. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become readable - - """ - return get_async_backend().wait_readable(obj) - - -def wait_writable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object can be written to. - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become writable - - .. seealso:: See the documentation of :func:`wait_readable` for the definition of - ``obj`` and notes on backend compatibility. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - """ - return get_async_backend().wait_writable(obj) - - -# -# Private API -# - - -def convert_ipv6_sockaddr( - sockaddr: tuple[str, int, int, int] | tuple[str, int], -) -> tuple[str, int]: - """ - Convert a 4-tuple IPv6 socket address to a 2-tuple (address, port) format. - - If the scope ID is nonzero, it is added to the address, separated with ``%``. - Otherwise the flow id and scope id are simply cut off from the tuple. - Any other kinds of socket addresses are returned as-is. - - :param sockaddr: the result of :meth:`~socket.socket.getsockname` - :return: the converted socket address - - """ - # This is more complicated than it should be because of MyPy - if isinstance(sockaddr, tuple) and len(sockaddr) == 4: - host, port, flowinfo, scope_id = sockaddr - if scope_id: - # PyPy (as of v7.3.11) leaves the interface name in the result, so - # we discard it and only get the scope ID from the end - # (https://foss.heptapod.net/pypy/pypy/-/issues/3938) - host = host.split("%")[0] - - # Add scope_id to the address - return f"{host}%{scope_id}", port - else: - return host, port - else: - return sockaddr - - -async def setup_unix_local_socket( - path: None | str | bytes | PathLike[Any], - mode: int | None, - socktype: int, -) -> socket.socket: - """ - Create a UNIX local socket object, deleting the socket at the given path if it - exists. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param socktype: socket.SOCK_STREAM or socket.SOCK_DGRAM - - """ - path_str: str | None - if path is not None: - path_str = os.fsdecode(path) - - # Linux abstract namespace sockets aren't backed by a concrete file so skip stat call - if not path_str.startswith("\0"): - # Copied from pathlib... - try: - stat_result = os.stat(path) - except OSError as e: - if e.errno not in ( - errno.ENOENT, - errno.ENOTDIR, - errno.EBADF, - errno.ELOOP, - ): - raise - else: - if stat.S_ISSOCK(stat_result.st_mode): - os.unlink(path) - else: - path_str = None - - raw_socket = socket.socket(socket.AF_UNIX, socktype) - raw_socket.setblocking(False) - - if path_str is not None: - try: - await to_thread.run_sync(raw_socket.bind, path_str, abandon_on_cancel=True) - if mode is not None: - await to_thread.run_sync(chmod, path_str, mode, abandon_on_cancel=True) - except BaseException: - raw_socket.close() - raise - - return raw_socket diff --git a/contrib/python/anyio/anyio/_core/_streams.py b/contrib/python/anyio/anyio/_core/_streams.py deleted file mode 100644 index 6a9814e5a91..00000000000 --- a/contrib/python/anyio/anyio/_core/_streams.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import math -from typing import TypeVar -from warnings import warn - -from ..streams.memory import ( - MemoryObjectReceiveStream, - MemoryObjectSendStream, - MemoryObjectStreamState, -) - -T_Item = TypeVar("T_Item") - - -class create_memory_object_stream( - tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], -): - """ - Create a memory object stream. - - The stream's item type can be annotated like - :func:`create_memory_object_stream[T_Item]`. - - :param max_buffer_size: number of items held in the buffer until ``send()`` starts - blocking - :param item_type: old way of marking the streams with the right generic type for - static typing (does nothing on AnyIO 4) - - .. deprecated:: 4.0 - Use ``create_memory_object_stream[YourItemType](...)`` instead. - :return: a tuple of (send stream, receive stream) - - """ - - def __new__( # type: ignore[misc] - cls, max_buffer_size: float = 0, item_type: object = None - ) -> tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]]: - if max_buffer_size != math.inf and not isinstance(max_buffer_size, int): - raise ValueError("max_buffer_size must be either an integer or math.inf") - if max_buffer_size < 0: - raise ValueError("max_buffer_size cannot be negative") - if item_type is not None: - warn( - "The item_type argument has been deprecated in AnyIO 4.0. " - "Use create_memory_object_stream[YourItemType](...) instead.", - DeprecationWarning, - stacklevel=2, - ) - - state = MemoryObjectStreamState[T_Item](max_buffer_size) - return (MemoryObjectSendStream(state), MemoryObjectReceiveStream(state)) diff --git a/contrib/python/anyio/anyio/_core/_subprocesses.py b/contrib/python/anyio/anyio/_core/_subprocesses.py deleted file mode 100644 index 7ba41a5b031..00000000000 --- a/contrib/python/anyio/anyio/_core/_subprocesses.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import AsyncIterable, Iterable, Mapping, Sequence -from io import BytesIO -from os import PathLike -from subprocess import DEVNULL, PIPE, CalledProcessError, CompletedProcess -from typing import IO, Any, Union, cast - -from ..abc import Process -from ._eventloop import get_async_backend -from ._tasks import create_task_group - -if sys.version_info >= (3, 10): - from typing import TypeAlias -else: - from typing_extensions import TypeAlias - -StrOrBytesPath: TypeAlias = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"] - - -async def run_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - input: bytes | None = None, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - check: bool = True, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> CompletedProcess[bytes]: - """ - Run an external command in a subprocess and wait until it completes. - - .. seealso:: :func:`subprocess.run` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param input: bytes passed to the standard input of the subprocess - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or `None` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or `None` - :param check: if ``True``, raise :exc:`~subprocess.CalledProcessError` if the - process terminates with a return code other than 0 - :param cwd: If not ``None``, change the working directory to this before running the - command - :param env: if not ``None``, this mapping replaces the inherited environment - variables from the parent process - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (Python >= 3.9, POSIX only) - :param group: effective group to run the process as (Python >= 3.9, POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9, - POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (Python >= 3.9, POSIX only) - :return: an object representing the completed process - :raises ~subprocess.CalledProcessError: if ``check`` is ``True`` and the process - exits with a nonzero return code - - """ - - async def drain_stream(stream: AsyncIterable[bytes], index: int) -> None: - buffer = BytesIO() - async for chunk in stream: - buffer.write(chunk) - - stream_contents[index] = buffer.getvalue() - - async with await open_process( - command, - stdin=PIPE if input else DEVNULL, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - user=user, - group=group, - extra_groups=extra_groups, - umask=umask, - ) as process: - stream_contents: list[bytes | None] = [None, None] - async with create_task_group() as tg: - if process.stdout: - tg.start_soon(drain_stream, process.stdout, 0) - - if process.stderr: - tg.start_soon(drain_stream, process.stderr, 1) - - if process.stdin and input: - await process.stdin.send(input) - await process.stdin.aclose() - - await process.wait() - - output, errors = stream_contents - if check and process.returncode != 0: - raise CalledProcessError(cast(int, process.returncode), command, output, errors) - - return CompletedProcess(command, cast(int, process.returncode), output, errors) - - -async def open_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None = PIPE, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> Process: - """ - Start an external command in a subprocess. - - .. seealso:: :class:`subprocess.Popen` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, a - file-like object, or ``None`` - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or ``None`` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or ``None`` - :param cwd: If not ``None``, the working directory is changed before executing - :param env: If env is not ``None``, it must be a mapping that defines the - environment variables for the new process - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (POSIX only) - :param group: effective group to run the process as (POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (POSIX only) - :return: an asynchronous process object - - """ - kwargs: dict[str, Any] = {} - if user is not None: - kwargs["user"] = user - - if group is not None: - kwargs["group"] = group - - if extra_groups is not None: - kwargs["extra_groups"] = group - - if umask >= 0: - kwargs["umask"] = umask - - return await get_async_backend().open_process( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - **kwargs, - ) diff --git a/contrib/python/anyio/anyio/_core/_synchronization.py b/contrib/python/anyio/anyio/_core/_synchronization.py deleted file mode 100644 index a6331328d4f..00000000000 --- a/contrib/python/anyio/anyio/_core/_synchronization.py +++ /dev/null @@ -1,732 +0,0 @@ -from __future__ import annotations - -import math -from collections import deque -from dataclasses import dataclass -from types import TracebackType - -from sniffio import AsyncLibraryNotFoundError - -from ..lowlevel import checkpoint -from ._eventloop import get_async_backend -from ._exceptions import BusyResourceError -from ._tasks import CancelScope -from ._testing import TaskInfo, get_current_task - - -@dataclass(frozen=True) -class EventStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Event.wait` - """ - - tasks_waiting: int - - -@dataclass(frozen=True) -class CapacityLimiterStatistics: - """ - :ivar int borrowed_tokens: number of tokens currently borrowed by tasks - :ivar float total_tokens: total number of available tokens - :ivar tuple borrowers: tasks or other objects currently holding tokens borrowed from - this limiter - :ivar int tasks_waiting: number of tasks waiting on - :meth:`~.CapacityLimiter.acquire` or - :meth:`~.CapacityLimiter.acquire_on_behalf_of` - """ - - borrowed_tokens: int - total_tokens: float - borrowers: tuple[object, ...] - tasks_waiting: int - - -@dataclass(frozen=True) -class LockStatistics: - """ - :ivar bool locked: flag indicating if this lock is locked or not - :ivar ~anyio.TaskInfo owner: task currently holding the lock (or ``None`` if the - lock is not held by any task) - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Lock.acquire` - """ - - locked: bool - owner: TaskInfo | None - tasks_waiting: int - - -@dataclass(frozen=True) -class ConditionStatistics: - """ - :ivar int tasks_waiting: number of tasks blocked on :meth:`~.Condition.wait` - :ivar ~anyio.LockStatistics lock_statistics: statistics of the underlying - :class:`~.Lock` - """ - - tasks_waiting: int - lock_statistics: LockStatistics - - -@dataclass(frozen=True) -class SemaphoreStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Semaphore.acquire` - - """ - - tasks_waiting: int - - -class Event: - def __new__(cls) -> Event: - try: - return get_async_backend().create_event() - except AsyncLibraryNotFoundError: - return EventAdapter() - - def set(self) -> None: - """Set the flag, notifying all listeners.""" - raise NotImplementedError - - def is_set(self) -> bool: - """Return ``True`` if the flag is set, ``False`` if not.""" - raise NotImplementedError - - async def wait(self) -> None: - """ - Wait until the flag has been set. - - If the flag has already been set when this method is called, it returns - immediately. - - """ - raise NotImplementedError - - def statistics(self) -> EventStatistics: - """Return statistics about the current state of this event.""" - raise NotImplementedError - - -class EventAdapter(Event): - _internal_event: Event | None = None - _is_set: bool = False - - def __new__(cls) -> EventAdapter: - return object.__new__(cls) - - @property - def _event(self) -> Event: - if self._internal_event is None: - self._internal_event = get_async_backend().create_event() - if self._is_set: - self._internal_event.set() - - return self._internal_event - - def set(self) -> None: - if self._internal_event is None: - self._is_set = True - else: - self._event.set() - - def is_set(self) -> bool: - if self._internal_event is None: - return self._is_set - - return self._internal_event.is_set() - - async def wait(self) -> None: - await self._event.wait() - - def statistics(self) -> EventStatistics: - if self._internal_event is None: - return EventStatistics(tasks_waiting=0) - - return self._internal_event.statistics() - - -class Lock: - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - try: - return get_async_backend().create_lock(fast_acquire=fast_acquire) - except AsyncLibraryNotFoundError: - return LockAdapter(fast_acquire=fast_acquire) - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Release the lock.""" - raise NotImplementedError - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - raise NotImplementedError - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class LockAdapter(Lock): - _internal_lock: Lock | None = None - - def __new__(cls, *, fast_acquire: bool = False) -> LockAdapter: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False): - self._fast_acquire = fast_acquire - - @property - def _lock(self) -> Lock: - if self._internal_lock is None: - self._internal_lock = get_async_backend().create_lock( - fast_acquire=self._fast_acquire - ) - - return self._internal_lock - - async def __aenter__(self) -> None: - await self._lock.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - if self._internal_lock is not None: - self._internal_lock.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - await self._lock.acquire() - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - - def release(self) -> None: - """Release the lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - return self._lock.locked() - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - - """ - if self._internal_lock is None: - return LockStatistics(False, None, 0) - - return self._internal_lock.statistics() - - -class Condition: - _owner_task: TaskInfo | None = None - - def __init__(self, lock: Lock | None = None): - self._lock = lock or Lock() - self._waiters: deque[Event] = deque() - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - def _check_acquired(self) -> None: - if self._owner_task != get_current_task(): - raise RuntimeError("The current task is not holding the underlying lock") - - async def acquire(self) -> None: - """Acquire the underlying lock.""" - await self._lock.acquire() - self._owner_task = get_current_task() - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - self._owner_task = get_current_task() - - def release(self) -> None: - """Release the underlying lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is set.""" - return self._lock.locked() - - def notify(self, n: int = 1) -> None: - """Notify exactly n listeners.""" - self._check_acquired() - for _ in range(n): - try: - event = self._waiters.popleft() - except IndexError: - break - - event.set() - - def notify_all(self) -> None: - """Notify all the listeners.""" - self._check_acquired() - for event in self._waiters: - event.set() - - self._waiters.clear() - - async def wait(self) -> None: - """Wait for a notification.""" - await checkpoint() - event = Event() - self._waiters.append(event) - self.release() - try: - await event.wait() - except BaseException: - if not event.is_set(): - self._waiters.remove(event) - - raise - finally: - with CancelScope(shield=True): - await self.acquire() - - def statistics(self) -> ConditionStatistics: - """ - Return statistics about the current state of this condition. - - .. versionadded:: 3.0 - """ - return ConditionStatistics(len(self._waiters), self._lock.statistics()) - - -class Semaphore: - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - try: - return get_async_backend().create_semaphore( - initial_value, max_value=max_value, fast_acquire=fast_acquire - ) - except AsyncLibraryNotFoundError: - return SemaphoreAdapter(initial_value, max_value=max_value) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - if not isinstance(initial_value, int): - raise TypeError("initial_value must be an integer") - if initial_value < 0: - raise ValueError("initial_value must be >= 0") - if max_value is not None: - if not isinstance(max_value, int): - raise TypeError("max_value must be an integer or None") - if max_value < initial_value: - raise ValueError( - "max_value must be equal to or higher than initial_value" - ) - - self._fast_acquire = fast_acquire - - async def __aenter__(self) -> Semaphore: - await self.acquire() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Decrement the semaphore value, blocking if necessary.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Increment the semaphore value.""" - raise NotImplementedError - - @property - def value(self) -> int: - """The current value of the semaphore.""" - raise NotImplementedError - - @property - def max_value(self) -> int | None: - """The maximum value of the semaphore.""" - raise NotImplementedError - - def statistics(self) -> SemaphoreStatistics: - """ - Return statistics about the current state of this semaphore. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class SemaphoreAdapter(Semaphore): - _internal_semaphore: Semaphore | None = None - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> SemaphoreAdapter: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self._initial_value = initial_value - self._max_value = max_value - - @property - def _semaphore(self) -> Semaphore: - if self._internal_semaphore is None: - self._internal_semaphore = get_async_backend().create_semaphore( - self._initial_value, max_value=self._max_value - ) - - return self._internal_semaphore - - async def acquire(self) -> None: - await self._semaphore.acquire() - - def acquire_nowait(self) -> None: - self._semaphore.acquire_nowait() - - def release(self) -> None: - self._semaphore.release() - - @property - def value(self) -> int: - if self._internal_semaphore is None: - return self._initial_value - - return self._semaphore.value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - if self._internal_semaphore is None: - return SemaphoreStatistics(tasks_waiting=0) - - return self._semaphore.statistics() - - -class CapacityLimiter: - def __new__(cls, total_tokens: float) -> CapacityLimiter: - try: - return get_async_backend().create_capacity_limiter(total_tokens) - except AsyncLibraryNotFoundError: - return CapacityLimiterAdapter(total_tokens) - - async def __aenter__(self) -> None: - raise NotImplementedError - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - raise NotImplementedError - - @property - def total_tokens(self) -> float: - """ - The total number of tokens available for borrowing. - - This is a read-write property. If the total number of tokens is increased, the - proportionate number of tasks waiting on this limiter will be granted their - tokens. - - .. versionchanged:: 3.0 - The property is now writable. - - """ - raise NotImplementedError - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - raise NotImplementedError - - @property - def borrowed_tokens(self) -> int: - """The number of tokens that have currently been borrowed.""" - raise NotImplementedError - - @property - def available_tokens(self) -> float: - """The number of tokens currently available to be borrowed""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire a token for the current task without waiting for one to become - available. - - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - """ - Acquire a token without waiting for one to become available. - - :param borrower: the entity borrowing a token - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - async def acquire(self) -> None: - """ - Acquire a token for the current task, waiting if necessary for one to become - available. - - """ - raise NotImplementedError - - async def acquire_on_behalf_of(self, borrower: object) -> None: - """ - Acquire a token, waiting if necessary for one to become available. - - :param borrower: the entity borrowing a token - - """ - raise NotImplementedError - - def release(self) -> None: - """ - Release the token held by the current task. - - :raises RuntimeError: if the current task has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def release_on_behalf_of(self, borrower: object) -> None: - """ - Release the token held by the given borrower. - - :raises RuntimeError: if the borrower has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def statistics(self) -> CapacityLimiterStatistics: - """ - Return statistics about the current state of this limiter. - - .. versionadded:: 3.0 - - """ - raise NotImplementedError - - -class CapacityLimiterAdapter(CapacityLimiter): - _internal_limiter: CapacityLimiter | None = None - - def __new__(cls, total_tokens: float) -> CapacityLimiterAdapter: - return object.__new__(cls) - - def __init__(self, total_tokens: float) -> None: - self.total_tokens = total_tokens - - @property - def _limiter(self) -> CapacityLimiter: - if self._internal_limiter is None: - self._internal_limiter = get_async_backend().create_capacity_limiter( - self._total_tokens - ) - - return self._internal_limiter - - async def __aenter__(self) -> None: - await self._limiter.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - return await self._limiter.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and value is not math.inf: - raise TypeError("total_tokens must be an int or math.inf") - elif value < 1: - raise ValueError("total_tokens must be >= 1") - - if self._internal_limiter is None: - self._total_tokens = value - return - - self._limiter.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - if self._internal_limiter is None: - return 0 - - return self._internal_limiter.borrowed_tokens - - @property - def available_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.available_tokens - - def acquire_nowait(self) -> None: - self._limiter.acquire_nowait() - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - self._limiter.acquire_on_behalf_of_nowait(borrower) - - async def acquire(self) -> None: - await self._limiter.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self._limiter.acquire_on_behalf_of(borrower) - - def release(self) -> None: - self._limiter.release() - - def release_on_behalf_of(self, borrower: object) -> None: - self._limiter.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - if self._internal_limiter is None: - return CapacityLimiterStatistics( - borrowed_tokens=0, - total_tokens=self.total_tokens, - borrowers=(), - tasks_waiting=0, - ) - - return self._internal_limiter.statistics() - - -class ResourceGuard: - """ - A context manager for ensuring that a resource is only used by a single task at a - time. - - Entering this context manager while the previous has not exited it yet will trigger - :exc:`BusyResourceError`. - - :param action: the action to guard against (visible in the :exc:`BusyResourceError` - when triggered, e.g. "Another task is already {action} this resource") - - .. versionadded:: 4.1 - """ - - __slots__ = "action", "_guarded" - - def __init__(self, action: str = "using"): - self.action: str = action - self._guarded = False - - def __enter__(self) -> None: - if self._guarded: - raise BusyResourceError(self.action) - - self._guarded = True - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._guarded = False diff --git a/contrib/python/anyio/anyio/_core/_tasks.py b/contrib/python/anyio/anyio/_core/_tasks.py deleted file mode 100644 index fe49015102b..00000000000 --- a/contrib/python/anyio/anyio/_core/_tasks.py +++ /dev/null @@ -1,158 +0,0 @@ -from __future__ import annotations - -import math -from collections.abc import Generator -from contextlib import contextmanager -from types import TracebackType - -from ..abc._tasks import TaskGroup, TaskStatus -from ._eventloop import get_async_backend - - -class _IgnoredTaskStatus(TaskStatus[object]): - def started(self, value: object = None) -> None: - pass - - -TASK_STATUS_IGNORED = _IgnoredTaskStatus() - - -class CancelScope: - """ - Wraps a unit of work that can be made separately cancellable. - - :param deadline: The time (clock value) when this scope is cancelled automatically - :param shield: ``True`` to shield the cancel scope from external cancellation - """ - - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return get_async_backend().create_cancel_scope(shield=shield, deadline=deadline) - - def cancel(self) -> None: - """Cancel this scope immediately.""" - raise NotImplementedError - - @property - def deadline(self) -> float: - """ - The time (clock value) when this scope is cancelled automatically. - - Will be ``float('inf')`` if no timeout has been set. - - """ - raise NotImplementedError - - @deadline.setter - def deadline(self, value: float) -> None: - raise NotImplementedError - - @property - def cancel_called(self) -> bool: - """``True`` if :meth:`cancel` has been called.""" - raise NotImplementedError - - @property - def cancelled_caught(self) -> bool: - """ - ``True`` if this scope suppressed a cancellation exception it itself raised. - - This is typically used to check if any work was interrupted, or to see if the - scope was cancelled due to its deadline being reached. The value will, however, - only be ``True`` if the cancellation was triggered by the scope itself (and not - an outer scope). - - """ - raise NotImplementedError - - @property - def shield(self) -> bool: - """ - ``True`` if this scope is shielded from external cancellation. - - While a scope is shielded, it will not receive cancellations from outside. - - """ - raise NotImplementedError - - @shield.setter - def shield(self, value: bool) -> None: - raise NotImplementedError - - def __enter__(self) -> CancelScope: - raise NotImplementedError - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - raise NotImplementedError - - -@contextmanager -def fail_after( - delay: float | None, shield: bool = False -) -> Generator[CancelScope, None, None]: - """ - Create a context manager which raises a :class:`TimeoutError` if does not finish in - time. - - :param delay: maximum allowed time (in seconds) before raising the exception, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a context manager that yields a cancel scope - :rtype: :class:`~typing.ContextManager`\\[:class:`~anyio.CancelScope`\\] - - """ - current_time = get_async_backend().current_time - deadline = (current_time() + delay) if delay is not None else math.inf - with get_async_backend().create_cancel_scope( - deadline=deadline, shield=shield - ) as cancel_scope: - yield cancel_scope - - if cancel_scope.cancelled_caught and current_time() >= cancel_scope.deadline: - raise TimeoutError - - -def move_on_after(delay: float | None, shield: bool = False) -> CancelScope: - """ - Create a cancel scope with a deadline that expires after the given delay. - - :param delay: maximum allowed time (in seconds) before exiting the context block, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a cancel scope - - """ - deadline = ( - (get_async_backend().current_time() + delay) if delay is not None else math.inf - ) - return get_async_backend().create_cancel_scope(deadline=deadline, shield=shield) - - -def current_effective_deadline() -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the current - task. - - :return: a clock value from the event loop's internal clock (or ``float('inf')`` if - there is no deadline in effect, or ``float('-inf')`` if the current scope has - been cancelled) - :rtype: float - - """ - return get_async_backend().current_effective_deadline() - - -def create_task_group() -> TaskGroup: - """ - Create a task group. - - :return: a task group - - """ - return get_async_backend().create_task_group() diff --git a/contrib/python/anyio/anyio/_core/_testing.py b/contrib/python/anyio/anyio/_core/_testing.py deleted file mode 100644 index 9e28b22766d..00000000000 --- a/contrib/python/anyio/anyio/_core/_testing.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, Generator -from typing import Any, cast - -from ._eventloop import get_async_backend - - -class TaskInfo: - """ - Represents an asynchronous task. - - :ivar int id: the unique identifier of the task - :ivar parent_id: the identifier of the parent task, if any - :vartype parent_id: Optional[int] - :ivar str name: the description of the task (if any) - :ivar ~collections.abc.Coroutine coro: the coroutine object of the task - """ - - __slots__ = "_name", "id", "parent_id", "name", "coro" - - def __init__( - self, - id: int, - parent_id: int | None, - name: str | None, - coro: Generator[Any, Any, Any] | Awaitable[Any], - ): - func = get_current_task - self._name = f"{func.__module__}.{func.__qualname__}" - self.id: int = id - self.parent_id: int | None = parent_id - self.name: str | None = name - self.coro: Generator[Any, Any, Any] | Awaitable[Any] = coro - - def __eq__(self, other: object) -> bool: - if isinstance(other, TaskInfo): - return self.id == other.id - - return NotImplemented - - def __hash__(self) -> int: - return hash(self.id) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(id={self.id!r}, name={self.name!r})" - - def has_pending_cancellation(self) -> bool: - """ - Return ``True`` if the task has a cancellation pending, ``False`` otherwise. - - """ - return False - - -def get_current_task() -> TaskInfo: - """ - Return the current task. - - :return: a representation of the current task - - """ - return get_async_backend().get_current_task() - - -def get_running_tasks() -> list[TaskInfo]: - """ - Return a list of running tasks in the current event loop. - - :return: a list of task info objects - - """ - return cast("list[TaskInfo]", get_async_backend().get_running_tasks()) - - -async def wait_all_tasks_blocked() -> None: - """Wait until all other tasks are waiting for something.""" - await get_async_backend().wait_all_tasks_blocked() diff --git a/contrib/python/anyio/anyio/_core/_typedattr.py b/contrib/python/anyio/anyio/_core/_typedattr.py deleted file mode 100644 index f358a448cb1..00000000000 --- a/contrib/python/anyio/anyio/_core/_typedattr.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping -from typing import Any, TypeVar, final, overload - -from ._exceptions import TypedAttributeLookupError - -T_Attr = TypeVar("T_Attr") -T_Default = TypeVar("T_Default") -undefined = object() - - -def typed_attribute() -> Any: - """Return a unique object, used to mark typed attributes.""" - return object() - - -class TypedAttributeSet: - """ - Superclass for typed attribute collections. - - Checks that every public attribute of every subclass has a type annotation. - """ - - def __init_subclass__(cls) -> None: - annotations: dict[str, Any] = getattr(cls, "__annotations__", {}) - for attrname in dir(cls): - if not attrname.startswith("_") and attrname not in annotations: - raise TypeError( - f"Attribute {attrname!r} is missing its type annotation" - ) - - super().__init_subclass__() - - -class TypedAttributeProvider: - """Base class for classes that wish to provide typed extra attributes.""" - - @property - def extra_attributes(self) -> Mapping[T_Attr, Callable[[], T_Attr]]: - """ - A mapping of the extra attributes to callables that return the corresponding - values. - - If the provider wraps another provider, the attributes from that wrapper should - also be included in the returned mapping (but the wrapper may override the - callables from the wrapped instance). - - """ - return {} - - @overload - def extra(self, attribute: T_Attr) -> T_Attr: ... - - @overload - def extra(self, attribute: T_Attr, default: T_Default) -> T_Attr | T_Default: ... - - @final - def extra(self, attribute: Any, default: object = undefined) -> object: - """ - extra(attribute, default=undefined) - - Return the value of the given typed extra attribute. - - :param attribute: the attribute (member of a :class:`~TypedAttributeSet`) to - look for - :param default: the value that should be returned if no value is found for the - attribute - :raises ~anyio.TypedAttributeLookupError: if the search failed and no default - value was given - - """ - try: - getter = self.extra_attributes[attribute] - except KeyError: - if default is undefined: - raise TypedAttributeLookupError("Attribute not found") from None - else: - return default - - return getter() diff --git a/contrib/python/anyio/anyio/abc/__init__.py b/contrib/python/anyio/anyio/abc/__init__.py deleted file mode 100644 index 3d3b61cc9a0..00000000000 --- a/contrib/python/anyio/anyio/abc/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -from ._eventloop import AsyncBackend as AsyncBackend -from ._resources import AsyncResource as AsyncResource -from ._sockets import ConnectedUDPSocket as ConnectedUDPSocket -from ._sockets import ConnectedUNIXDatagramSocket as ConnectedUNIXDatagramSocket -from ._sockets import IPAddressType as IPAddressType -from ._sockets import IPSockAddrType as IPSockAddrType -from ._sockets import SocketAttribute as SocketAttribute -from ._sockets import SocketListener as SocketListener -from ._sockets import SocketStream as SocketStream -from ._sockets import UDPPacketType as UDPPacketType -from ._sockets import UDPSocket as UDPSocket -from ._sockets import UNIXDatagramPacketType as UNIXDatagramPacketType -from ._sockets import UNIXDatagramSocket as UNIXDatagramSocket -from ._sockets import UNIXSocketStream as UNIXSocketStream -from ._streams import AnyByteReceiveStream as AnyByteReceiveStream -from ._streams import AnyByteSendStream as AnyByteSendStream -from ._streams import AnyByteStream as AnyByteStream -from ._streams import AnyUnreliableByteReceiveStream as AnyUnreliableByteReceiveStream -from ._streams import AnyUnreliableByteSendStream as AnyUnreliableByteSendStream -from ._streams import AnyUnreliableByteStream as AnyUnreliableByteStream -from ._streams import ByteReceiveStream as ByteReceiveStream -from ._streams import ByteSendStream as ByteSendStream -from ._streams import ByteStream as ByteStream -from ._streams import Listener as Listener -from ._streams import ObjectReceiveStream as ObjectReceiveStream -from ._streams import ObjectSendStream as ObjectSendStream -from ._streams import ObjectStream as ObjectStream -from ._streams import UnreliableObjectReceiveStream as UnreliableObjectReceiveStream -from ._streams import UnreliableObjectSendStream as UnreliableObjectSendStream -from ._streams import UnreliableObjectStream as UnreliableObjectStream -from ._subprocesses import Process as Process -from ._tasks import TaskGroup as TaskGroup -from ._tasks import TaskStatus as TaskStatus -from ._testing import TestRunner as TestRunner - -# Re-exported here, for backwards compatibility -# isort: off -from .._core._synchronization import ( - CapacityLimiter as CapacityLimiter, - Condition as Condition, - Event as Event, - Lock as Lock, - Semaphore as Semaphore, -) -from .._core._tasks import CancelScope as CancelScope -from ..from_thread import BlockingPortal as BlockingPortal - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio.abc."): - __value.__module__ = __name__ - -del __value diff --git a/contrib/python/anyio/anyio/abc/_eventloop.py b/contrib/python/anyio/anyio/abc/_eventloop.py deleted file mode 100644 index 2bfdf286357..00000000000 --- a/contrib/python/anyio/anyio/abc/_eventloop.py +++ /dev/null @@ -1,376 +0,0 @@ -from __future__ import annotations - -import math -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence -from contextlib import AbstractContextManager -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind, socket -from typing import ( - IO, - TYPE_CHECKING, - Any, - TypeVar, - Union, - overload, -) - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if sys.version_info >= (3, 10): - from typing import TypeAlias -else: - from typing_extensions import TypeAlias - -if TYPE_CHECKING: - from _typeshed import HasFileno - - from .._core._synchronization import CapacityLimiter, Event, Lock, Semaphore - from .._core._tasks import CancelScope - from .._core._testing import TaskInfo - from ..from_thread import BlockingPortal - from ._sockets import ( - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, - ) - from ._subprocesses import Process - from ._tasks import TaskGroup - from ._testing import TestRunner - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") -StrOrBytesPath: TypeAlias = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"] - - -class AsyncBackend(metaclass=ABCMeta): - @classmethod - @abstractmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param kwargs: positional arguments to ``func`` - :param options: keyword arguments to call the backend ``run()`` implementation - with - :return: the return value of the coroutine function - """ - - @classmethod - @abstractmethod - def current_token(cls) -> object: - """ - - :return: - """ - - @classmethod - @abstractmethod - def current_time(cls) -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - """ - - @classmethod - @abstractmethod - def cancelled_exception_class(cls) -> type[BaseException]: - """Return the exception class that is raised in a task if it's cancelled.""" - - @classmethod - @abstractmethod - async def checkpoint(cls) -> None: - """ - Check if the task has been cancelled, and allow rescheduling of other tasks. - - This is effectively the same as running :meth:`checkpoint_if_cancelled` and then - :meth:`cancel_shielded_checkpoint`. - """ - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - """ - Check if the current task group has been cancelled. - - This will check if the task has been cancelled, but will not allow other tasks - to be scheduled if not. - - """ - if cls.current_effective_deadline() == -math.inf: - await cls.checkpoint() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - """ - Allow the rescheduling of other tasks. - - This will give other tasks the opportunity to run, but without checking if the - current task group has been cancelled, unlike with :meth:`checkpoint`. - - """ - with cls.create_cancel_scope(shield=True): - await cls.sleep(0) - - @classmethod - @abstractmethod - async def sleep(cls, delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - """ - - @classmethod - @abstractmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - pass - - @classmethod - @abstractmethod - def current_effective_deadline(cls) -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the - current task. - - :return: - - a clock value from the event loop's internal clock - - ``inf`` if there is no deadline in effect - - ``-inf`` if the current scope has been cancelled - :rtype: float - """ - - @classmethod - @abstractmethod - def create_task_group(cls) -> TaskGroup: - pass - - @classmethod - @abstractmethod - def create_event(cls) -> Event: - pass - - @classmethod - @abstractmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - pass - - @classmethod - @abstractmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - pass - - @classmethod - @abstractmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: CapacityLimiter | None = None, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def check_cancelled(cls) -> None: - pass - - @classmethod - @abstractmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def create_blocking_portal(cls) -> BlockingPortal: - pass - - @classmethod - @abstractmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - pass - - @classmethod - @abstractmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[Process]) -> None: - pass - - @classmethod - @abstractmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - pass - - @classmethod - @abstractmethod - async def connect_unix(cls, path: str | bytes) -> UNIXSocketStream: - pass - - @classmethod - @abstractmethod - def create_tcp_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - def create_unix_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - pass - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: None - ) -> UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes - ) -> ConnectedUNIXDatagramSocket: ... - - @classmethod - @abstractmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes | None - ) -> UNIXDatagramSocket | ConnectedUNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> list[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int], - ] - ]: - pass - - @classmethod - @abstractmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - pass - - @classmethod - @abstractmethod - async def wait_readable(cls, obj: HasFileno | int) -> None: - pass - - @classmethod - @abstractmethod - async def wait_writable(cls, obj: HasFileno | int) -> None: - pass - - @classmethod - @abstractmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - pass - - @classmethod - @abstractmethod - def get_current_task(cls) -> TaskInfo: - pass - - @classmethod - @abstractmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - pass - - @classmethod - @abstractmethod - async def wait_all_tasks_blocked(cls) -> None: - pass - - @classmethod - @abstractmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - pass diff --git a/contrib/python/anyio/anyio/abc/_resources.py b/contrib/python/anyio/anyio/abc/_resources.py deleted file mode 100644 index 10df115a7b9..00000000000 --- a/contrib/python/anyio/anyio/abc/_resources.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from types import TracebackType -from typing import TypeVar - -T = TypeVar("T") - - -class AsyncResource(metaclass=ABCMeta): - """ - Abstract base class for all closeable asynchronous resources. - - Works as an asynchronous context manager which returns the instance itself on enter, - and calls :meth:`aclose` on exit. - """ - - __slots__ = () - - async def __aenter__(self: T) -> T: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.aclose() - - @abstractmethod - async def aclose(self) -> None: - """Close the resource.""" diff --git a/contrib/python/anyio/anyio/abc/_sockets.py b/contrib/python/anyio/anyio/abc/_sockets.py deleted file mode 100644 index 1c6a450cdcd..00000000000 --- a/contrib/python/anyio/anyio/abc/_sockets.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import annotations - -import socket -from abc import abstractmethod -from collections.abc import Callable, Collection, Mapping -from contextlib import AsyncExitStack -from io import IOBase -from ipaddress import IPv4Address, IPv6Address -from socket import AddressFamily -from types import TracebackType -from typing import Any, TypeVar, Union - -from .._core._typedattr import ( - TypedAttributeProvider, - TypedAttributeSet, - typed_attribute, -) -from ._streams import ByteStream, Listener, UnreliableObjectStream -from ._tasks import TaskGroup - -IPAddressType = Union[str, IPv4Address, IPv6Address] -IPSockAddrType = tuple[str, int] -SockAddrType = Union[IPSockAddrType, str] -UDPPacketType = tuple[bytes, IPSockAddrType] -UNIXDatagramPacketType = tuple[bytes, str] -T_Retval = TypeVar("T_Retval") - - -class _NullAsyncContextManager: - async def __aenter__(self) -> None: - pass - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - return None - - -class SocketAttribute(TypedAttributeSet): - #: the address family of the underlying socket - family: AddressFamily = typed_attribute() - #: the local socket address of the underlying socket - local_address: SockAddrType = typed_attribute() - #: for IP addresses, the local port the underlying socket is bound to - local_port: int = typed_attribute() - #: the underlying stdlib socket object - raw_socket: socket.socket = typed_attribute() - #: the remote address the underlying socket is connected to - remote_address: SockAddrType = typed_attribute() - #: for IP addresses, the remote port the underlying socket is connected to - remote_port: int = typed_attribute() - - -class _SocketProvider(TypedAttributeProvider): - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - from .._core._sockets import convert_ipv6_sockaddr as convert - - attributes: dict[Any, Callable[[], Any]] = { - SocketAttribute.family: lambda: self._raw_socket.family, - SocketAttribute.local_address: lambda: convert( - self._raw_socket.getsockname() - ), - SocketAttribute.raw_socket: lambda: self._raw_socket, - } - try: - peername: tuple[str, int] | None = convert(self._raw_socket.getpeername()) - except OSError: - peername = None - - # Provide the remote address for connected sockets - if peername is not None: - attributes[SocketAttribute.remote_address] = lambda: peername - - # Provide local and remote ports for IP based sockets - if self._raw_socket.family in (AddressFamily.AF_INET, AddressFamily.AF_INET6): - attributes[SocketAttribute.local_port] = ( - lambda: self._raw_socket.getsockname()[1] - ) - if peername is not None: - remote_port = peername[1] - attributes[SocketAttribute.remote_port] = lambda: remote_port - - return attributes - - @property - @abstractmethod - def _raw_socket(self) -> socket.socket: - pass - - -class SocketStream(ByteStream, _SocketProvider): - """ - Transports bytes over a socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - -class UNIXSocketStream(SocketStream): - @abstractmethod - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - """ - Send file descriptors along with a message to the peer. - - :param message: a non-empty bytestring - :param fds: a collection of files (either numeric file descriptors or open file - or socket objects) - """ - - @abstractmethod - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - """ - Receive file descriptors along with a message from the peer. - - :param msglen: length of the message to expect from the peer - :param maxfds: maximum number of file descriptors to expect from the peer - :return: a tuple of (message, file descriptors) - """ - - -class SocketListener(Listener[SocketStream], _SocketProvider): - """ - Listens to incoming socket connections. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @abstractmethod - async def accept(self) -> SocketStream: - """Accept an incoming connection.""" - - async def serve( - self, - handler: Callable[[SocketStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - from .. import create_task_group - - async with AsyncExitStack() as stack: - if task_group is None: - task_group = await stack.enter_async_context(create_task_group()) - - while True: - stream = await self.accept() - task_group.start_soon(handler, stream) - - -class UDPSocket(UnreliableObjectStream[UDPPacketType], _SocketProvider): - """ - Represents an unconnected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - async def sendto(self, data: bytes, host: str, port: int) -> None: - """ - Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, (host, port))). - - """ - return await self.send((data, (host, port))) - - -class ConnectedUDPSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents an connected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - -class UNIXDatagramSocket( - UnreliableObjectStream[UNIXDatagramPacketType], _SocketProvider -): - """ - Represents an unconnected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - async def sendto(self, data: bytes, path: str) -> None: - """Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, path)).""" - return await self.send((data, path)) - - -class ConnectedUNIXDatagramSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents a connected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ diff --git a/contrib/python/anyio/anyio/abc/_streams.py b/contrib/python/anyio/anyio/abc/_streams.py deleted file mode 100644 index 8c638683a49..00000000000 --- a/contrib/python/anyio/anyio/abc/_streams.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from collections.abc import Callable -from typing import Any, Generic, TypeVar, Union - -from .._core._exceptions import EndOfStream -from .._core._typedattr import TypedAttributeProvider -from ._resources import AsyncResource -from ._tasks import TaskGroup - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class UnreliableObjectReceiveStream( - Generic[T_co], AsyncResource, TypedAttributeProvider -): - """ - An interface for receiving objects. - - This interface makes no guarantees that the received messages arrive in the order in - which they were sent, or that no messages are missed. - - Asynchronously iterating over objects of this type will yield objects matching the - given type parameter. - """ - - def __aiter__(self) -> UnreliableObjectReceiveStream[T_co]: - return self - - async def __anext__(self) -> T_co: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration - - @abstractmethod - async def receive(self) -> T_co: - """ - Receive the next item. - - :raises ~anyio.ClosedResourceError: if the receive stream has been explicitly - closed - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectSendStream( - Generic[T_contra], AsyncResource, TypedAttributeProvider -): - """ - An interface for sending objects. - - This interface makes no guarantees that the messages sent will reach the - recipient(s) in the same order in which they were sent, or at all. - """ - - @abstractmethod - async def send(self, item: T_contra) -> None: - """ - Send an item to the peer(s). - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if the send stream has been explicitly - closed - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectStream( - UnreliableObjectReceiveStream[T_Item], UnreliableObjectSendStream[T_Item] -): - """ - A bidirectional message stream which does not guarantee the order or reliability of - message delivery. - """ - - -class ObjectReceiveStream(UnreliableObjectReceiveStream[T_co]): - """ - A receive message stream which guarantees that messages are received in the same - order in which they were sent, and that no messages are missed. - """ - - -class ObjectSendStream(UnreliableObjectSendStream[T_contra]): - """ - A send message stream which guarantees that messages are delivered in the same order - in which they were sent, without missing any messages in the middle. - """ - - -class ObjectStream( - ObjectReceiveStream[T_Item], - ObjectSendStream[T_Item], - UnreliableObjectStream[T_Item], -): - """ - A bidirectional message stream which guarantees the order and reliability of message - delivery. - """ - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -class ByteReceiveStream(AsyncResource, TypedAttributeProvider): - """ - An interface for receiving bytes from a single peer. - - Iterating this byte stream will yield a byte string of arbitrary length, but no more - than 65536 bytes. - """ - - def __aiter__(self) -> ByteReceiveStream: - return self - - async def __anext__(self) -> bytes: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration - - @abstractmethod - async def receive(self, max_bytes: int = 65536) -> bytes: - """ - Receive at most ``max_bytes`` bytes from the peer. - - .. note:: Implementors of this interface should not return an empty - :class:`bytes` object, and users should ignore them. - - :param max_bytes: maximum number of bytes to receive - :return: the received bytes - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - """ - - -class ByteSendStream(AsyncResource, TypedAttributeProvider): - """An interface for sending bytes to a single peer.""" - - @abstractmethod - async def send(self, item: bytes) -> None: - """ - Send the given bytes to the peer. - - :param item: the bytes to send - """ - - -class ByteStream(ByteReceiveStream, ByteSendStream): - """A bidirectional byte stream.""" - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -#: Type alias for all unreliable bytes-oriented receive streams. -AnyUnreliableByteReceiveStream = Union[ - UnreliableObjectReceiveStream[bytes], ByteReceiveStream -] -#: Type alias for all unreliable bytes-oriented send streams. -AnyUnreliableByteSendStream = Union[UnreliableObjectSendStream[bytes], ByteSendStream] -#: Type alias for all unreliable bytes-oriented streams. -AnyUnreliableByteStream = Union[UnreliableObjectStream[bytes], ByteStream] -#: Type alias for all bytes-oriented receive streams. -AnyByteReceiveStream = Union[ObjectReceiveStream[bytes], ByteReceiveStream] -#: Type alias for all bytes-oriented send streams. -AnyByteSendStream = Union[ObjectSendStream[bytes], ByteSendStream] -#: Type alias for all bytes-oriented streams. -AnyByteStream = Union[ObjectStream[bytes], ByteStream] - - -class Listener(Generic[T_co], AsyncResource, TypedAttributeProvider): - """An interface for objects that let you accept incoming connections.""" - - @abstractmethod - async def serve( - self, handler: Callable[[T_co], Any], task_group: TaskGroup | None = None - ) -> None: - """ - Accept incoming connections as they come in and start tasks to handle them. - - :param handler: a callable that will be used to handle each accepted connection - :param task_group: the task group that will be used to start tasks for handling - each accepted connection (if omitted, an ad-hoc task group will be created) - """ diff --git a/contrib/python/anyio/anyio/abc/_subprocesses.py b/contrib/python/anyio/anyio/abc/_subprocesses.py deleted file mode 100644 index ce0564ceac8..00000000000 --- a/contrib/python/anyio/anyio/abc/_subprocesses.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from signal import Signals - -from ._resources import AsyncResource -from ._streams import ByteReceiveStream, ByteSendStream - - -class Process(AsyncResource): - """An asynchronous version of :class:`subprocess.Popen`.""" - - @abstractmethod - async def wait(self) -> int: - """ - Wait until the process exits. - - :return: the exit code of the process - """ - - @abstractmethod - def terminate(self) -> None: - """ - Terminates the process, gracefully if possible. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGTERM`` to the process. - - .. seealso:: :meth:`subprocess.Popen.terminate` - """ - - @abstractmethod - def kill(self) -> None: - """ - Kills the process. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGKILL`` to the process. - - .. seealso:: :meth:`subprocess.Popen.kill` - """ - - @abstractmethod - def send_signal(self, signal: Signals) -> None: - """ - Send a signal to the subprocess. - - .. seealso:: :meth:`subprocess.Popen.send_signal` - - :param signal: the signal number (e.g. :data:`signal.SIGHUP`) - """ - - @property - @abstractmethod - def pid(self) -> int: - """The process ID of the process.""" - - @property - @abstractmethod - def returncode(self) -> int | None: - """ - The return code of the process. If the process has not yet terminated, this will - be ``None``. - """ - - @property - @abstractmethod - def stdin(self) -> ByteSendStream | None: - """The stream for the standard input of the process.""" - - @property - @abstractmethod - def stdout(self) -> ByteReceiveStream | None: - """The stream for the standard output of the process.""" - - @property - @abstractmethod - def stderr(self) -> ByteReceiveStream | None: - """The stream for the standard error output of the process.""" diff --git a/contrib/python/anyio/anyio/abc/_tasks.py b/contrib/python/anyio/anyio/abc/_tasks.py deleted file mode 100644 index f6e5c40c7ff..00000000000 --- a/contrib/python/anyio/anyio/abc/_tasks.py +++ /dev/null @@ -1,101 +0,0 @@ -from __future__ import annotations - -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import Awaitable, Callable -from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol, TypeVar, overload - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from .._core._tasks import CancelScope - -T_Retval = TypeVar("T_Retval") -T_contra = TypeVar("T_contra", contravariant=True) -PosArgsT = TypeVarTuple("PosArgsT") - - -class TaskStatus(Protocol[T_contra]): - @overload - def started(self: TaskStatus[None]) -> None: ... - - @overload - def started(self, value: T_contra) -> None: ... - - def started(self, value: T_contra | None = None) -> None: - """ - Signal that the task has started. - - :param value: object passed back to the starter of the task - """ - - -class TaskGroup(metaclass=ABCMeta): - """ - Groups several asynchronous tasks together. - - :ivar cancel_scope: the cancel scope inherited by all child tasks - :vartype cancel_scope: CancelScope - - .. note:: On asyncio, support for eager task factories is considered to be - **experimental**. In particular, they don't follow the usual semantics of new - tasks being scheduled on the next iteration of the event loop, and may thus - cause unexpected behavior in code that wasn't written with such semantics in - mind. - """ - - cancel_scope: CancelScope - - @abstractmethod - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - """ - Start a new task in this task group. - - :param func: a coroutine function - :param args: positional arguments to call the function with - :param name: name of the task, for the purposes of introspection and debugging - - .. versionadded:: 3.0 - """ - - @abstractmethod - async def start( - self, - func: Callable[..., Awaitable[Any]], - *args: object, - name: object = None, - ) -> Any: - """ - Start a new task and wait until it signals for readiness. - - :param func: a coroutine function - :param args: positional arguments to call the function with - :param name: name of the task, for the purposes of introspection and debugging - :return: the value passed to ``task_status.started()`` - :raises RuntimeError: if the task finishes without calling - ``task_status.started()`` - - .. versionadded:: 3.0 - """ - - @abstractmethod - async def __aenter__(self) -> TaskGroup: - """Enter the task group context and allow starting new tasks.""" - - @abstractmethod - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - """Exit the task group context waiting for all tasks to finish.""" diff --git a/contrib/python/anyio/anyio/abc/_testing.py b/contrib/python/anyio/anyio/abc/_testing.py deleted file mode 100644 index 7c50ed76dc4..00000000000 --- a/contrib/python/anyio/anyio/abc/_testing.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -import types -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable -from typing import Any, TypeVar - -_T = TypeVar("_T") - - -class TestRunner(metaclass=ABCMeta): - """ - Encapsulates a running event loop. Every call made through this object will use the - same event loop. - """ - - def __enter__(self) -> TestRunner: - return self - - @abstractmethod - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> bool | None: ... - - @abstractmethod - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[_T, Any]], - kwargs: dict[str, Any], - ) -> Iterable[_T]: - """ - Run an async generator fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: an iterator yielding the value yielded from the async generator - """ - - @abstractmethod - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, _T]], - kwargs: dict[str, Any], - ) -> _T: - """ - Run an async fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: the return value of the fixture function - """ - - @abstractmethod - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - """ - Run an async test function. - - :param test_func: the test function - :param kwargs: keyword arguments to call the test function with - """ diff --git a/contrib/python/anyio/anyio/from_thread.py b/contrib/python/anyio/anyio/from_thread.py deleted file mode 100644 index 93a4cfe8e49..00000000000 --- a/contrib/python/anyio/anyio/from_thread.py +++ /dev/null @@ -1,527 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Awaitable, Callable, Generator -from concurrent.futures import Future -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - contextmanager, -) -from dataclasses import dataclass, field -from inspect import isawaitable -from threading import Lock, Thread, get_ident -from types import TracebackType -from typing import ( - Any, - Generic, - TypeVar, - cast, - overload, -) - -from ._core import _eventloop -from ._core._eventloop import get_async_backend, get_cancelled_exc_class, threadlocals -from ._core._synchronization import Event -from ._core._tasks import CancelScope, create_task_group -from .abc import AsyncBackend -from .abc._tasks import TaskStatus - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -PosArgsT = TypeVarTuple("PosArgsT") - - -def run( - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], *args: Unpack[PosArgsT] -) -> T_Retval: - """ - Call a coroutine function from a worker thread. - - :param func: a coroutine function - :param args: positional arguments for the callable - :return: the return value of the coroutine function - - """ - try: - async_backend = threadlocals.current_async_backend - token = threadlocals.current_token - except AttributeError: - raise RuntimeError( - "This function can only be run from an AnyIO worker thread" - ) from None - - return async_backend.run_async_from_thread(func, args, token=token) - - -def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] -) -> T_Retval: - """ - Call a function in the event loop thread from a worker thread. - - :param func: a callable - :param args: positional arguments for the callable - :return: the return value of the callable - - """ - try: - async_backend = threadlocals.current_async_backend - token = threadlocals.current_token - except AttributeError: - raise RuntimeError( - "This function can only be run from an AnyIO worker thread" - ) from None - - return async_backend.run_sync_from_thread(func, args, token=token) - - -class _BlockingAsyncContextManager(Generic[T_co], AbstractContextManager): - _enter_future: Future[T_co] - _exit_future: Future[bool | None] - _exit_event: Event - _exit_exc_info: tuple[ - type[BaseException] | None, BaseException | None, TracebackType | None - ] = (None, None, None) - - def __init__( - self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal - ): - self._async_cm = async_cm - self._portal = portal - - async def run_async_cm(self) -> bool | None: - try: - self._exit_event = Event() - value = await self._async_cm.__aenter__() - except BaseException as exc: - self._enter_future.set_exception(exc) - raise - else: - self._enter_future.set_result(value) - - try: - # Wait for the sync context manager to exit. - # This next statement can raise `get_cancelled_exc_class()` if - # something went wrong in a task group in this async context - # manager. - await self._exit_event.wait() - finally: - # In case of cancellation, it could be that we end up here before - # `_BlockingAsyncContextManager.__exit__` is called, and an - # `_exit_exc_info` has been set. - result = await self._async_cm.__aexit__(*self._exit_exc_info) - return result - - def __enter__(self) -> T_co: - self._enter_future = Future() - self._exit_future = self._portal.start_task_soon(self.run_async_cm) - return self._enter_future.result() - - def __exit__( - self, - __exc_type: type[BaseException] | None, - __exc_value: BaseException | None, - __traceback: TracebackType | None, - ) -> bool | None: - self._exit_exc_info = __exc_type, __exc_value, __traceback - self._portal.call(self._exit_event.set) - return self._exit_future.result() - - -class _BlockingPortalTaskStatus(TaskStatus): - def __init__(self, future: Future): - self._future = future - - def started(self, value: object = None) -> None: - self._future.set_result(value) - - -class BlockingPortal: - """An object that lets external threads run code in an asynchronous event loop.""" - - def __new__(cls) -> BlockingPortal: - return get_async_backend().create_blocking_portal() - - def __init__(self) -> None: - self._event_loop_thread_id: int | None = get_ident() - self._stop_event = Event() - self._task_group = create_task_group() - self._cancelled_exc_class = get_cancelled_exc_class() - - async def __aenter__(self) -> BlockingPortal: - await self._task_group.__aenter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - await self.stop() - return await self._task_group.__aexit__(exc_type, exc_val, exc_tb) - - def _check_running(self) -> None: - if self._event_loop_thread_id is None: - raise RuntimeError("This portal is not running") - if self._event_loop_thread_id == get_ident(): - raise RuntimeError( - "This method cannot be called from the event loop thread" - ) - - async def sleep_until_stopped(self) -> None: - """Sleep until :meth:`stop` is called.""" - await self._stop_event.wait() - - async def stop(self, cancel_remaining: bool = False) -> None: - """ - Signal the portal to shut down. - - This marks the portal as no longer accepting new calls and exits from - :meth:`sleep_until_stopped`. - - :param cancel_remaining: ``True`` to cancel all the remaining tasks, ``False`` - to let them finish before returning - - """ - self._event_loop_thread_id = None - self._stop_event.set() - if cancel_remaining: - self._task_group.cancel_scope.cancel() - - async def _call_func( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - future: Future[T_Retval], - ) -> None: - def callback(f: Future[T_Retval]) -> None: - if f.cancelled() and self._event_loop_thread_id not in ( - None, - get_ident(), - ): - self.call(scope.cancel) - - try: - retval_or_awaitable = func(*args, **kwargs) - if isawaitable(retval_or_awaitable): - with CancelScope() as scope: - if future.cancelled(): - scope.cancel() - else: - future.add_done_callback(callback) - - retval = await retval_or_awaitable - else: - retval = retval_or_awaitable - except self._cancelled_exc_class: - future.cancel() - future.set_running_or_notify_cancel() - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - # Let base exceptions fall through - if not isinstance(exc, Exception): - raise - else: - if not future.cancelled(): - future.set_result(retval) - finally: - scope = None # type: ignore[assignment] - - def _spawn_task_from_thread( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - name: object, - future: Future[T_Retval], - ) -> None: - """ - Spawn a new task using the given callable. - - Implementors must ensure that the future is resolved when the task finishes. - - :param func: a callable - :param args: positional arguments to be passed to the callable - :param kwargs: keyword arguments to be passed to the callable - :param name: name of the task (will be coerced to a string if not ``None``) - :param future: a future that will resolve to the return value of the callable, - or the exception raised during its execution - - """ - raise NotImplementedError - - @overload - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - ) -> T_Retval: ... - - @overload - def call( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: ... - - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - ) -> T_Retval: - """ - Call the given function in the event loop thread. - - If the callable returns a coroutine object, it is awaited on. - - :param func: any callable - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - - """ - return cast(T_Retval, self.start_task_soon(func, *args).result()) - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: - """ - Start a task in the portal's task group. - - The task will be run inside a cancel scope which can be cancelled by cancelling - the returned future. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a future that resolves with the return value of the callable if the - task completes successfully, or with the exception raised in the task - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - :rtype: concurrent.futures.Future[T_Retval] - - .. versionadded:: 3.0 - - """ - self._check_running() - f: Future[T_Retval] = Future() - self._spawn_task_from_thread(func, args, {}, name, f) - return f - - def start_task( - self, - func: Callable[..., Awaitable[T_Retval]], - *args: object, - name: object = None, - ) -> tuple[Future[T_Retval], Any]: - """ - Start a task in the portal's task group and wait until it signals for readiness. - - This method works the same way as :meth:`.abc.TaskGroup.start`. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a tuple of (future, task_status_value) where the ``task_status_value`` - is the value passed to ``task_status.started()`` from within the target - function - :rtype: tuple[concurrent.futures.Future[T_Retval], Any] - - .. versionadded:: 3.0 - - """ - - def task_done(future: Future[T_Retval]) -> None: - if not task_status_future.done(): - if future.cancelled(): - task_status_future.cancel() - elif future.exception(): - task_status_future.set_exception(future.exception()) - else: - exc = RuntimeError( - "Task exited without calling task_status.started()" - ) - task_status_future.set_exception(exc) - - self._check_running() - task_status_future: Future = Future() - task_status = _BlockingPortalTaskStatus(task_status_future) - f: Future = Future() - f.add_done_callback(task_done) - self._spawn_task_from_thread(func, args, {"task_status": task_status}, name, f) - return f, task_status_future.result() - - def wrap_async_context_manager( - self, cm: AbstractAsyncContextManager[T_co] - ) -> AbstractContextManager[T_co]: - """ - Wrap an async context manager as a synchronous context manager via this portal. - - Spawns a task that will call both ``__aenter__()`` and ``__aexit__()``, stopping - in the middle until the synchronous context manager exits. - - :param cm: an asynchronous context manager - :return: a synchronous context manager - - .. versionadded:: 2.1 - - """ - return _BlockingAsyncContextManager(cm, self) - - -@dataclass -class BlockingPortalProvider: - """ - A manager for a blocking portal. Used as a context manager. The first thread to - enter this context manager causes a blocking portal to be started with the specific - parameters, and the last thread to exit causes the portal to be shut down. Thus, - there will be exactly one blocking portal running in this context as long as at - least one thread has entered this context manager. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - - .. versionadded:: 4.4 - """ - - backend: str = "asyncio" - backend_options: dict[str, Any] | None = None - _lock: Lock = field(init=False, default_factory=Lock) - _leases: int = field(init=False, default=0) - _portal: BlockingPortal = field(init=False) - _portal_cm: AbstractContextManager[BlockingPortal] | None = field( - init=False, default=None - ) - - def __enter__(self) -> BlockingPortal: - with self._lock: - if self._portal_cm is None: - self._portal_cm = start_blocking_portal( - self.backend, self.backend_options - ) - self._portal = self._portal_cm.__enter__() - - self._leases += 1 - return self._portal - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - portal_cm: AbstractContextManager[BlockingPortal] | None = None - with self._lock: - assert self._portal_cm - assert self._leases > 0 - self._leases -= 1 - if not self._leases: - portal_cm = self._portal_cm - self._portal_cm = None - del self._portal - - if portal_cm: - portal_cm.__exit__(None, None, None) - - -@contextmanager -def start_blocking_portal( - backend: str = "asyncio", backend_options: dict[str, Any] | None = None -) -> Generator[BlockingPortal, Any, None]: - """ - Start a new event loop in a new thread and run a blocking portal in its main task. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - :return: a context manager that yields a blocking portal - - .. versionchanged:: 3.0 - Usage as a context manager is now required. - - """ - - async def run_portal() -> None: - async with BlockingPortal() as portal_: - future.set_result(portal_) - await portal_.sleep_until_stopped() - - def run_blocking_portal() -> None: - if future.set_running_or_notify_cancel(): - try: - _eventloop.run( - run_portal, backend=backend, backend_options=backend_options - ) - except BaseException as exc: - if not future.done(): - future.set_exception(exc) - - future: Future[BlockingPortal] = Future() - thread = Thread(target=run_blocking_portal, daemon=True) - thread.start() - try: - cancel_remaining_tasks = False - portal = future.result() - try: - yield portal - except BaseException: - cancel_remaining_tasks = True - raise - finally: - try: - portal.call(portal.stop, cancel_remaining_tasks) - except RuntimeError: - pass - finally: - thread.join() - - -def check_cancelled() -> None: - """ - Check if the cancel scope of the host task's running the current worker thread has - been cancelled. - - If the host task's current cancel scope has indeed been cancelled, the - backend-specific cancellation exception will be raised. - - :raises RuntimeError: if the current thread was not spawned by - :func:`.to_thread.run_sync` - - """ - try: - async_backend: AsyncBackend = threadlocals.current_async_backend - except AttributeError: - raise RuntimeError( - "This function can only be run from an AnyIO worker thread" - ) from None - - async_backend.check_cancelled() diff --git a/contrib/python/anyio/anyio/lowlevel.py b/contrib/python/anyio/anyio/lowlevel.py deleted file mode 100644 index 14c7668cb3f..00000000000 --- a/contrib/python/anyio/anyio/lowlevel.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import annotations - -import enum -from dataclasses import dataclass -from typing import Any, Generic, Literal, TypeVar, overload -from weakref import WeakKeyDictionary - -from ._core._eventloop import get_async_backend - -T = TypeVar("T") -D = TypeVar("D") - - -async def checkpoint() -> None: - """ - Check for cancellation and allow the scheduler to switch to another task. - - Equivalent to (but more efficient than):: - - await checkpoint_if_cancelled() - await cancel_shielded_checkpoint() - - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint() - - -async def checkpoint_if_cancelled() -> None: - """ - Enter a checkpoint if the enclosing cancel scope has been cancelled. - - This does not allow the scheduler to switch to a different task. - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint_if_cancelled() - - -async def cancel_shielded_checkpoint() -> None: - """ - Allow the scheduler to switch to another task but without checking for cancellation. - - Equivalent to (but potentially more efficient than):: - - with CancelScope(shield=True): - await checkpoint() - - - .. versionadded:: 3.0 - - """ - await get_async_backend().cancel_shielded_checkpoint() - - -def current_token() -> object: - """ - Return a backend specific token object that can be used to get back to the event - loop. - - """ - return get_async_backend().current_token() - - -_run_vars: WeakKeyDictionary[Any, dict[str, Any]] = WeakKeyDictionary() -_token_wrappers: dict[Any, _TokenWrapper] = {} - - -@dataclass(frozen=True) -class _TokenWrapper: - __slots__ = "_token", "__weakref__" - _token: object - - -class _NoValueSet(enum.Enum): - NO_VALUE_SET = enum.auto() - - -class RunvarToken(Generic[T]): - __slots__ = "_var", "_value", "_redeemed" - - def __init__(self, var: RunVar[T], value: T | Literal[_NoValueSet.NO_VALUE_SET]): - self._var = var - self._value: T | Literal[_NoValueSet.NO_VALUE_SET] = value - self._redeemed = False - - -class RunVar(Generic[T]): - """ - Like a :class:`~contextvars.ContextVar`, except scoped to the running event loop. - """ - - __slots__ = "_name", "_default" - - NO_VALUE_SET: Literal[_NoValueSet.NO_VALUE_SET] = _NoValueSet.NO_VALUE_SET - - _token_wrappers: set[_TokenWrapper] = set() - - def __init__( - self, name: str, default: T | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ): - self._name = name - self._default = default - - @property - def _current_vars(self) -> dict[str, T]: - token = current_token() - try: - return _run_vars[token] - except KeyError: - run_vars = _run_vars[token] = {} - return run_vars - - @overload - def get(self, default: D) -> T | D: ... - - @overload - def get(self) -> T: ... - - def get( - self, default: D | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ) -> T | D: - try: - return self._current_vars[self._name] - except KeyError: - if default is not RunVar.NO_VALUE_SET: - return default - elif self._default is not RunVar.NO_VALUE_SET: - return self._default - - raise LookupError( - f'Run variable "{self._name}" has no value and no default set' - ) - - def set(self, value: T) -> RunvarToken[T]: - current_vars = self._current_vars - token = RunvarToken(self, current_vars.get(self._name, RunVar.NO_VALUE_SET)) - current_vars[self._name] = value - return token - - def reset(self, token: RunvarToken[T]) -> None: - if token._var is not self: - raise ValueError("This token does not belong to this RunVar") - - if token._redeemed: - raise ValueError("This token has already been used") - - if token._value is _NoValueSet.NO_VALUE_SET: - try: - del self._current_vars[self._name] - except KeyError: - pass - else: - self._current_vars[self._name] = token._value - - token._redeemed = True - - def __repr__(self) -> str: - return f"<RunVar name={self._name!r}>" diff --git a/contrib/python/anyio/anyio/py.typed b/contrib/python/anyio/anyio/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/anyio/anyio/py.typed +++ /dev/null diff --git a/contrib/python/anyio/anyio/pytest_plugin.py b/contrib/python/anyio/anyio/pytest_plugin.py deleted file mode 100644 index 4a0d59dd067..00000000000 --- a/contrib/python/anyio/anyio/pytest_plugin.py +++ /dev/null @@ -1,191 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Generator, Iterator -from contextlib import ExitStack, contextmanager -from inspect import isasyncgenfunction, iscoroutinefunction, ismethod -from typing import Any, cast - -import pytest -import sniffio -from _pytest.fixtures import SubRequest -from _pytest.outcomes import Exit - -from ._core._eventloop import get_all_backends, get_async_backend -from ._core._exceptions import iterate_exceptions -from .abc import TestRunner - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -_current_runner: TestRunner | None = None -_runner_stack: ExitStack | None = None -_runner_leases = 0 - - -def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: - if isinstance(backend, str): - return backend, {} - elif isinstance(backend, tuple) and len(backend) == 2: - if isinstance(backend[0], str) and isinstance(backend[1], dict): - return cast(tuple[str, dict[str, Any]], backend) - - raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") - - -@contextmanager -def get_runner( - backend_name: str, backend_options: dict[str, Any] -) -> Iterator[TestRunner]: - global _current_runner, _runner_leases, _runner_stack - if _current_runner is None: - asynclib = get_async_backend(backend_name) - _runner_stack = ExitStack() - if sniffio.current_async_library_cvar.get(None) is None: - # Since we're in control of the event loop, we can cache the name of the - # async library - token = sniffio.current_async_library_cvar.set(backend_name) - _runner_stack.callback(sniffio.current_async_library_cvar.reset, token) - - backend_options = backend_options or {} - _current_runner = _runner_stack.enter_context( - asynclib.create_test_runner(backend_options) - ) - - _runner_leases += 1 - try: - yield _current_runner - finally: - _runner_leases -= 1 - if not _runner_leases: - assert _runner_stack is not None - _runner_stack.close() - _runner_stack = _current_runner = None - - -def pytest_configure(config: Any) -> None: - config.addinivalue_line( - "markers", - "anyio: mark the (coroutine function) test to be run " - "asynchronously via anyio.", - ) - - [email protected](hookwrapper=True) -def pytest_fixture_setup(fixturedef: Any, request: Any) -> Generator[Any]: - def wrapper( - *args: Any, anyio_backend: Any, request: SubRequest, **kwargs: Any - ) -> Any: - # Rebind any fixture methods to the request instance - if ( - request.instance - and ismethod(func) - and type(func.__self__) is type(request.instance) - ): - local_func = func.__func__.__get__(request.instance) - else: - local_func = func - - backend_name, backend_options = extract_backend_and_options(anyio_backend) - if has_backend_arg: - kwargs["anyio_backend"] = anyio_backend - - if has_request_arg: - kwargs["request"] = request - - with get_runner(backend_name, backend_options) as runner: - if isasyncgenfunction(local_func): - yield from runner.run_asyncgen_fixture(local_func, kwargs) - else: - yield runner.run_fixture(local_func, kwargs) - - # Only apply this to coroutine functions and async generator functions in requests - # that involve the anyio_backend fixture - func = fixturedef.func - if isasyncgenfunction(func) or iscoroutinefunction(func): - if "anyio_backend" in request.fixturenames: - fixturedef.func = wrapper - original_argname = fixturedef.argnames - - if not (has_backend_arg := "anyio_backend" in fixturedef.argnames): - fixturedef.argnames += ("anyio_backend",) - - if not (has_request_arg := "request" in fixturedef.argnames): - fixturedef.argnames += ("request",) - - try: - return (yield) - finally: - fixturedef.func = func - fixturedef.argnames = original_argname - - return (yield) - - [email protected](tryfirst=True) -def pytest_pycollect_makeitem(collector: Any, name: Any, obj: Any) -> None: - if collector.istestfunction(obj, name): - inner_func = obj.hypothesis.inner_test if hasattr(obj, "hypothesis") else obj - if iscoroutinefunction(inner_func): - marker = collector.get_closest_marker("anyio") - own_markers = getattr(obj, "pytestmark", ()) - if marker or any(marker.name == "anyio" for marker in own_markers): - pytest.mark.usefixtures("anyio_backend")(obj) - - [email protected](tryfirst=True) -def pytest_pyfunc_call(pyfuncitem: Any) -> bool | None: - def run_with_hypothesis(**kwargs: Any) -> None: - with get_runner(backend_name, backend_options) as runner: - runner.run_test(original_func, kwargs) - - backend = pyfuncitem.funcargs.get("anyio_backend") - if backend: - backend_name, backend_options = extract_backend_and_options(backend) - - if hasattr(pyfuncitem.obj, "hypothesis"): - # Wrap the inner test function unless it's already wrapped - original_func = pyfuncitem.obj.hypothesis.inner_test - if original_func.__qualname__ != run_with_hypothesis.__qualname__: - if iscoroutinefunction(original_func): - pyfuncitem.obj.hypothesis.inner_test = run_with_hypothesis - - return None - - if iscoroutinefunction(pyfuncitem.obj): - funcargs = pyfuncitem.funcargs - testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} - with get_runner(backend_name, backend_options) as runner: - try: - runner.run_test(pyfuncitem.obj, testargs) - except ExceptionGroup as excgrp: - for exc in iterate_exceptions(excgrp): - if isinstance(exc, (Exit, KeyboardInterrupt, SystemExit)): - raise exc from excgrp - - raise - - return True - - return None - - [email protected](scope="module", params=get_all_backends()) -def anyio_backend(request: Any) -> Any: - return request.param - - -def anyio_backend_name(anyio_backend: Any) -> str: - if isinstance(anyio_backend, str): - return anyio_backend - else: - return anyio_backend[0] - - -def anyio_backend_options(anyio_backend: Any) -> dict[str, Any]: - if isinstance(anyio_backend, str): - return {} - else: - return anyio_backend[1] diff --git a/contrib/python/anyio/anyio/streams/__init__.py b/contrib/python/anyio/anyio/streams/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/anyio/anyio/streams/__init__.py +++ /dev/null diff --git a/contrib/python/anyio/anyio/streams/buffered.py b/contrib/python/anyio/anyio/streams/buffered.py deleted file mode 100644 index f5d5e836dd4..00000000000 --- a/contrib/python/anyio/anyio/streams/buffered.py +++ /dev/null @@ -1,119 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping -from dataclasses import dataclass, field -from typing import Any - -from .. import ClosedResourceError, DelimiterNotFound, EndOfStream, IncompleteRead -from ..abc import AnyByteReceiveStream, ByteReceiveStream - - -@dataclass(eq=False) -class BufferedByteReceiveStream(ByteReceiveStream): - """ - Wraps any bytes-based receive stream and uses a buffer to provide sophisticated - receiving capabilities in the form of a byte stream. - """ - - receive_stream: AnyByteReceiveStream - _buffer: bytearray = field(init=False, default_factory=bytearray) - _closed: bool = field(init=False, default=False) - - async def aclose(self) -> None: - await self.receive_stream.aclose() - self._closed = True - - @property - def buffer(self) -> bytes: - """The bytes currently in the buffer.""" - return bytes(self._buffer) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.receive_stream.extra_attributes - - async def receive(self, max_bytes: int = 65536) -> bytes: - if self._closed: - raise ClosedResourceError - - if self._buffer: - chunk = bytes(self._buffer[:max_bytes]) - del self._buffer[:max_bytes] - return chunk - elif isinstance(self.receive_stream, ByteReceiveStream): - return await self.receive_stream.receive(max_bytes) - else: - # With a bytes-oriented object stream, we need to handle any surplus bytes - # we get from the receive() call - chunk = await self.receive_stream.receive() - if len(chunk) > max_bytes: - # Save the surplus bytes in the buffer - self._buffer.extend(chunk[max_bytes:]) - return chunk[:max_bytes] - else: - return chunk - - async def receive_exactly(self, nbytes: int) -> bytes: - """ - Read exactly the given amount of bytes from the stream. - - :param nbytes: the number of bytes to read - :return: the bytes read - :raises ~anyio.IncompleteRead: if the stream was closed before the requested - amount of bytes could be read from the stream - - """ - while True: - remaining = nbytes - len(self._buffer) - if remaining <= 0: - retval = self._buffer[:nbytes] - del self._buffer[:nbytes] - return bytes(retval) - - try: - if isinstance(self.receive_stream, ByteReceiveStream): - chunk = await self.receive_stream.receive(remaining) - else: - chunk = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - self._buffer.extend(chunk) - - async def receive_until(self, delimiter: bytes, max_bytes: int) -> bytes: - """ - Read from the stream until the delimiter is found or max_bytes have been read. - - :param delimiter: the marker to look for in the stream - :param max_bytes: maximum number of bytes that will be read before raising - :exc:`~anyio.DelimiterNotFound` - :return: the bytes read (not including the delimiter) - :raises ~anyio.IncompleteRead: if the stream was closed before the delimiter - was found - :raises ~anyio.DelimiterNotFound: if the delimiter is not found within the - bytes read up to the maximum allowed - - """ - delimiter_size = len(delimiter) - offset = 0 - while True: - # Check if the delimiter can be found in the current buffer - index = self._buffer.find(delimiter, offset) - if index >= 0: - found = self._buffer[:index] - del self._buffer[: index + len(delimiter) :] - return bytes(found) - - # Check if the buffer is already at or over the limit - if len(self._buffer) >= max_bytes: - raise DelimiterNotFound(max_bytes) - - # Read more data into the buffer from the socket - try: - data = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - # Move the offset forward and add the new data to the buffer - offset = max(len(self._buffer) - delimiter_size + 1, 0) - self._buffer.extend(data) diff --git a/contrib/python/anyio/anyio/streams/file.py b/contrib/python/anyio/anyio/streams/file.py deleted file mode 100644 index f492464267a..00000000000 --- a/contrib/python/anyio/anyio/streams/file.py +++ /dev/null @@ -1,148 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping -from io import SEEK_SET, UnsupportedOperation -from os import PathLike -from pathlib import Path -from typing import Any, BinaryIO, cast - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - TypedAttributeSet, - to_thread, - typed_attribute, -) -from ..abc import ByteReceiveStream, ByteSendStream - - -class FileStreamAttribute(TypedAttributeSet): - #: the open file descriptor - file: BinaryIO = typed_attribute() - #: the path of the file on the file system, if available (file must be a real file) - path: Path = typed_attribute() - #: the file number, if available (file must be a real file or a TTY) - fileno: int = typed_attribute() - - -class _BaseFileStream: - def __init__(self, file: BinaryIO): - self._file = file - - async def aclose(self) -> None: - await to_thread.run_sync(self._file.close) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict[Any, Callable[[], Any]] = { - FileStreamAttribute.file: lambda: self._file, - } - - if hasattr(self._file, "name"): - attributes[FileStreamAttribute.path] = lambda: Path(self._file.name) - - try: - self._file.fileno() - except UnsupportedOperation: - pass - else: - attributes[FileStreamAttribute.fileno] = lambda: self._file.fileno() - - return attributes - - -class FileReadStream(_BaseFileStream, ByteReceiveStream): - """ - A byte stream that reads from a file in the file system. - - :param file: a file that has been opened for reading in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path(cls, path: str | PathLike[str]) -> FileReadStream: - """ - Create a file read stream by opening the given file. - - :param path: path of the file to read from - - """ - file = await to_thread.run_sync(Path(path).open, "rb") - return cls(cast(BinaryIO, file)) - - async def receive(self, max_bytes: int = 65536) -> bytes: - try: - data = await to_thread.run_sync(self._file.read, max_bytes) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc - - if data: - return data - else: - raise EndOfStream - - async def seek(self, position: int, whence: int = SEEK_SET) -> int: - """ - Seek the file to the given position. - - .. seealso:: :meth:`io.IOBase.seek` - - .. note:: Not all file descriptors are seekable. - - :param position: position to seek the file to - :param whence: controls how ``position`` is interpreted - :return: the new absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.seek, position, whence) - - async def tell(self) -> int: - """ - Return the current stream position. - - .. note:: Not all file descriptors are seekable. - - :return: the current absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.tell) - - -class FileWriteStream(_BaseFileStream, ByteSendStream): - """ - A byte stream that writes to a file in the file system. - - :param file: a file that has been opened for writing in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path( - cls, path: str | PathLike[str], append: bool = False - ) -> FileWriteStream: - """ - Create a file write stream by opening the given file for writing. - - :param path: path of the file to write to - :param append: if ``True``, open the file for appending; if ``False``, any - existing file at the given path will be truncated - - """ - mode = "ab" if append else "wb" - file = await to_thread.run_sync(Path(path).open, mode) - return cls(cast(BinaryIO, file)) - - async def send(self, item: bytes) -> None: - try: - await to_thread.run_sync(self._file.write, item) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc diff --git a/contrib/python/anyio/anyio/streams/memory.py b/contrib/python/anyio/anyio/streams/memory.py deleted file mode 100644 index b547aa6a48c..00000000000 --- a/contrib/python/anyio/anyio/streams/memory.py +++ /dev/null @@ -1,317 +0,0 @@ -from __future__ import annotations - -import warnings -from collections import OrderedDict, deque -from dataclasses import dataclass, field -from types import TracebackType -from typing import Generic, NamedTuple, TypeVar - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - WouldBlock, -) -from .._core._testing import TaskInfo, get_current_task -from ..abc import Event, ObjectReceiveStream, ObjectSendStream -from ..lowlevel import checkpoint - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class MemoryObjectStreamStatistics(NamedTuple): - current_buffer_used: int #: number of items stored in the buffer - #: maximum number of items that can be stored on this stream (or :data:`math.inf`) - max_buffer_size: float - open_send_streams: int #: number of unclosed clones of the send stream - open_receive_streams: int #: number of unclosed clones of the receive stream - #: number of tasks blocked on :meth:`MemoryObjectSendStream.send` - tasks_waiting_send: int - #: number of tasks blocked on :meth:`MemoryObjectReceiveStream.receive` - tasks_waiting_receive: int - - -@dataclass(eq=False) -class MemoryObjectItemReceiver(Generic[T_Item]): - task_info: TaskInfo = field(init=False, default_factory=get_current_task) - item: T_Item = field(init=False) - - def __repr__(self) -> str: - # When item is not defined, we get following error with default __repr__: - # AttributeError: 'MemoryObjectItemReceiver' object has no attribute 'item' - item = getattr(self, "item", None) - return f"{self.__class__.__name__}(task_info={self.task_info}, item={item!r})" - - -@dataclass(eq=False) -class MemoryObjectStreamState(Generic[T_Item]): - max_buffer_size: float = field() - buffer: deque[T_Item] = field(init=False, default_factory=deque) - open_send_channels: int = field(init=False, default=0) - open_receive_channels: int = field(init=False, default=0) - waiting_receivers: OrderedDict[Event, MemoryObjectItemReceiver[T_Item]] = field( - init=False, default_factory=OrderedDict - ) - waiting_senders: OrderedDict[Event, T_Item] = field( - init=False, default_factory=OrderedDict - ) - - def statistics(self) -> MemoryObjectStreamStatistics: - return MemoryObjectStreamStatistics( - len(self.buffer), - self.max_buffer_size, - self.open_send_channels, - self.open_receive_channels, - len(self.waiting_senders), - len(self.waiting_receivers), - ) - - -@dataclass(eq=False) -class MemoryObjectReceiveStream(Generic[T_co], ObjectReceiveStream[T_co]): - _state: MemoryObjectStreamState[T_co] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_receive_channels += 1 - - def receive_nowait(self) -> T_co: - """ - Receive the next item if it can be done without waiting. - - :return: the received item - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.EndOfStream: if the buffer is empty and this stream has been - closed from the sending end - :raises ~anyio.WouldBlock: if there are no items in the buffer and no tasks - waiting to send - - """ - if self._closed: - raise ClosedResourceError - - if self._state.waiting_senders: - # Get the item from the next sender - send_event, item = self._state.waiting_senders.popitem(last=False) - self._state.buffer.append(item) - send_event.set() - - if self._state.buffer: - return self._state.buffer.popleft() - elif not self._state.open_send_channels: - raise EndOfStream - - raise WouldBlock - - async def receive(self) -> T_co: - await checkpoint() - try: - return self.receive_nowait() - except WouldBlock: - # Add ourselves in the queue - receive_event = Event() - receiver = MemoryObjectItemReceiver[T_co]() - self._state.waiting_receivers[receive_event] = receiver - - try: - await receive_event.wait() - finally: - self._state.waiting_receivers.pop(receive_event, None) - - try: - return receiver.item - except AttributeError: - raise EndOfStream - - def clone(self) -> MemoryObjectReceiveStream[T_co]: - """ - Create a clone of this receive stream. - - Each clone can be closed separately. Only when all clones have been closed will - the receiving end of the memory stream be considered closed by the sending ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectReceiveStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_receive_channels -= 1 - if self._state.open_receive_channels == 0: - send_events = list(self._state.waiting_senders.keys()) - for event in send_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectReceiveStream[T_co]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - source=self, - ) - - -@dataclass(eq=False) -class MemoryObjectSendStream(Generic[T_contra], ObjectSendStream[T_contra]): - _state: MemoryObjectStreamState[T_contra] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_send_channels += 1 - - def send_nowait(self, item: T_contra) -> None: - """ - Send an item immediately if it can be done without waiting. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - :raises ~anyio.WouldBlock: if the buffer is full and there are no tasks waiting - to receive - - """ - if self._closed: - raise ClosedResourceError - if not self._state.open_receive_channels: - raise BrokenResourceError - - while self._state.waiting_receivers: - receive_event, receiver = self._state.waiting_receivers.popitem(last=False) - if not receiver.task_info.has_pending_cancellation(): - receiver.item = item - receive_event.set() - return - - if len(self._state.buffer) < self._state.max_buffer_size: - self._state.buffer.append(item) - else: - raise WouldBlock - - async def send(self, item: T_contra) -> None: - """ - Send an item to the stream. - - If the buffer is full, this method blocks until there is again room in the - buffer or the item can be sent directly to a receiver. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - - """ - await checkpoint() - try: - self.send_nowait(item) - except WouldBlock: - # Wait until there's someone on the receiving end - send_event = Event() - self._state.waiting_senders[send_event] = item - try: - await send_event.wait() - except BaseException: - self._state.waiting_senders.pop(send_event, None) - raise - - if send_event in self._state.waiting_senders: - del self._state.waiting_senders[send_event] - raise BrokenResourceError from None - - def clone(self) -> MemoryObjectSendStream[T_contra]: - """ - Create a clone of this send stream. - - Each clone can be closed separately. Only when all clones have been closed will - the sending end of the memory stream be considered closed by the receiving ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectSendStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_send_channels -= 1 - if self._state.open_send_channels == 0: - receive_events = list(self._state.waiting_receivers.keys()) - self._state.waiting_receivers.clear() - for event in receive_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectSendStream[T_contra]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - source=self, - ) diff --git a/contrib/python/anyio/anyio/streams/stapled.py b/contrib/python/anyio/anyio/streams/stapled.py deleted file mode 100644 index 80f64a2e8e5..00000000000 --- a/contrib/python/anyio/anyio/streams/stapled.py +++ /dev/null @@ -1,141 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from typing import Any, Generic, TypeVar - -from ..abc import ( - ByteReceiveStream, - ByteSendStream, - ByteStream, - Listener, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, - TaskGroup, -) - -T_Item = TypeVar("T_Item") -T_Stream = TypeVar("T_Stream") - - -@dataclass(eq=False) -class StapledByteStream(ByteStream): - """ - Combines two byte streams into a single, bidirectional byte stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ByteSendStream send_stream: the sending byte stream - :param ByteReceiveStream receive_stream: the receiving byte stream - """ - - send_stream: ByteSendStream - receive_stream: ByteReceiveStream - - async def receive(self, max_bytes: int = 65536) -> bytes: - return await self.receive_stream.receive(max_bytes) - - async def send(self, item: bytes) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class StapledObjectStream(Generic[T_Item], ObjectStream[T_Item]): - """ - Combines two object streams into a single, bidirectional object stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ObjectSendStream send_stream: the sending object stream - :param ObjectReceiveStream receive_stream: the receiving object stream - """ - - send_stream: ObjectSendStream[T_Item] - receive_stream: ObjectReceiveStream[T_Item] - - async def receive(self) -> T_Item: - return await self.receive_stream.receive() - - async def send(self, item: T_Item) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class MultiListener(Generic[T_Stream], Listener[T_Stream]): - """ - Combines multiple listeners into one, serving connections from all of them at once. - - Any MultiListeners in the given collection of listeners will have their listeners - moved into this one. - - Extra attributes are provided from each listener, with each successive listener - overriding any conflicting attributes from the previous one. - - :param listeners: listeners to serve - :type listeners: Sequence[Listener[T_Stream]] - """ - - listeners: Sequence[Listener[T_Stream]] - - def __post_init__(self) -> None: - listeners: list[Listener[T_Stream]] = [] - for listener in self.listeners: - if isinstance(listener, MultiListener): - listeners.extend(listener.listeners) - del listener.listeners[:] # type: ignore[attr-defined] - else: - listeners.append(listener) - - self.listeners = listeners - - async def serve( - self, handler: Callable[[T_Stream], Any], task_group: TaskGroup | None = None - ) -> None: - from .. import create_task_group - - async with create_task_group() as tg: - for listener in self.listeners: - tg.start_soon(listener.serve, handler, task_group) - - async def aclose(self) -> None: - for listener in self.listeners: - await listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict = {} - for listener in self.listeners: - attributes.update(listener.extra_attributes) - - return attributes diff --git a/contrib/python/anyio/anyio/streams/text.py b/contrib/python/anyio/anyio/streams/text.py deleted file mode 100644 index f1a11278e3d..00000000000 --- a/contrib/python/anyio/anyio/streams/text.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -import codecs -from collections.abc import Callable, Mapping -from dataclasses import InitVar, dataclass, field -from typing import Any - -from ..abc import ( - AnyByteReceiveStream, - AnyByteSendStream, - AnyByteStream, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, -) - - -@dataclass(eq=False) -class TextReceiveStream(ObjectReceiveStream[str]): - """ - Stream wrapper that decodes bytes to strings using the given encoding. - - Decoding is done using :class:`~codecs.IncrementalDecoder` which returns any - completely received unicode characters as soon as they come in. - - :param transport_stream: any bytes-based receive stream - :param encoding: character encoding to use for decoding bytes to strings (defaults - to ``utf-8``) - :param errors: handling scheme for decoding errors (defaults to ``strict``; see the - `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteReceiveStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _decoder: codecs.IncrementalDecoder = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - decoder_class = codecs.getincrementaldecoder(encoding) - self._decoder = decoder_class(errors=errors) - - async def receive(self) -> str: - while True: - chunk = await self.transport_stream.receive() - decoded = self._decoder.decode(chunk) - if decoded: - return decoded - - async def aclose(self) -> None: - await self.transport_stream.aclose() - self._decoder.reset() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextSendStream(ObjectSendStream[str]): - """ - Sends strings to the wrapped stream as bytes using the given encoding. - - :param AnyByteSendStream transport_stream: any bytes-based send stream - :param str encoding: character encoding to use for encoding strings to bytes - (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteSendStream - encoding: InitVar[str] = "utf-8" - errors: str = "strict" - _encoder: Callable[..., tuple[bytes, int]] = field(init=False) - - def __post_init__(self, encoding: str) -> None: - self._encoder = codecs.getencoder(encoding) - - async def send(self, item: str) -> None: - encoded = self._encoder(item, self.errors)[0] - await self.transport_stream.send(encoded) - - async def aclose(self) -> None: - await self.transport_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextStream(ObjectStream[str]): - """ - A bidirectional stream that decodes bytes to strings on receive and encodes strings - to bytes on send. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param AnyByteStream transport_stream: any bytes-based stream - :param str encoding: character encoding to use for encoding/decoding strings to/from - bytes (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _receive_stream: TextReceiveStream = field(init=False) - _send_stream: TextSendStream = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - self._receive_stream = TextReceiveStream( - self.transport_stream, encoding=encoding, errors=errors - ) - self._send_stream = TextSendStream( - self.transport_stream, encoding=encoding, errors=errors - ) - - async def receive(self) -> str: - return await self._receive_stream.receive() - - async def send(self, item: str) -> None: - await self._send_stream.send(item) - - async def send_eof(self) -> None: - await self.transport_stream.send_eof() - - async def aclose(self) -> None: - await self._send_stream.aclose() - await self._receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self._send_stream.extra_attributes, - **self._receive_stream.extra_attributes, - } diff --git a/contrib/python/anyio/anyio/streams/tls.py b/contrib/python/anyio/anyio/streams/tls.py deleted file mode 100644 index b6961bee169..00000000000 --- a/contrib/python/anyio/anyio/streams/tls.py +++ /dev/null @@ -1,337 +0,0 @@ -from __future__ import annotations - -import logging -import re -import ssl -import sys -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from functools import wraps -from typing import Any, TypeVar - -from .. import ( - BrokenResourceError, - EndOfStream, - aclose_forcefully, - get_cancelled_exc_class, -) -from .._core._typedattr import TypedAttributeSet, typed_attribute -from ..abc import AnyByteStream, ByteStream, Listener, TaskGroup - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") -_PCTRTT = tuple[tuple[str, str], ...] -_PCTRTTT = tuple[_PCTRTT, ...] - - -class TLSAttribute(TypedAttributeSet): - """Contains Transport Layer Security related attributes.""" - - #: the selected ALPN protocol - alpn_protocol: str | None = typed_attribute() - #: the channel binding for type ``tls-unique`` - channel_binding_tls_unique: bytes = typed_attribute() - #: the selected cipher - cipher: tuple[str, str, int] = typed_attribute() - #: the peer certificate in dictionary form (see :meth:`ssl.SSLSocket.getpeercert` - # for more information) - peer_certificate: None | (dict[str, str | _PCTRTTT | _PCTRTT]) = typed_attribute() - #: the peer certificate in binary form - peer_certificate_binary: bytes | None = typed_attribute() - #: ``True`` if this is the server side of the connection - server_side: bool = typed_attribute() - #: ciphers shared by the client during the TLS handshake (``None`` if this is the - #: client side) - shared_ciphers: list[tuple[str, str, int]] | None = typed_attribute() - #: the :class:`~ssl.SSLObject` used for encryption - ssl_object: ssl.SSLObject = typed_attribute() - #: ``True`` if this stream does (and expects) a closing TLS handshake when the - #: stream is being closed - standard_compatible: bool = typed_attribute() - #: the TLS protocol version (e.g. ``TLSv1.2``) - tls_version: str = typed_attribute() - - -@dataclass(eq=False) -class TLSStream(ByteStream): - """ - A stream wrapper that encrypts all sent data and decrypts received data. - - This class has no public initializer; use :meth:`wrap` instead. - All extra attributes from :class:`~TLSAttribute` are supported. - - :var AnyByteStream transport_stream: the wrapped stream - - """ - - transport_stream: AnyByteStream - standard_compatible: bool - _ssl_object: ssl.SSLObject - _read_bio: ssl.MemoryBIO - _write_bio: ssl.MemoryBIO - - @classmethod - async def wrap( - cls, - transport_stream: AnyByteStream, - *, - server_side: bool | None = None, - hostname: str | None = None, - ssl_context: ssl.SSLContext | None = None, - standard_compatible: bool = True, - ) -> TLSStream: - """ - Wrap an existing stream with Transport Layer Security. - - This performs a TLS handshake with the peer. - - :param transport_stream: a bytes-transporting stream to wrap - :param server_side: ``True`` if this is the server side of the connection, - ``False`` if this is the client side (if omitted, will be set to ``False`` - if ``hostname`` has been provided, ``False`` otherwise). Used only to create - a default context when an explicit context has not been provided. - :param hostname: host name of the peer (if host name checking is desired) - :param ssl_context: the SSLContext object to use (if not provided, a secure - default will be created) - :param standard_compatible: if ``False``, skip the closing handshake when - closing the connection, and don't raise an exception if the peer does the - same - :raises ~ssl.SSLError: if the TLS handshake fails - - """ - if server_side is None: - server_side = not hostname - - if not ssl_context: - purpose = ( - ssl.Purpose.CLIENT_AUTH if server_side else ssl.Purpose.SERVER_AUTH - ) - ssl_context = ssl.create_default_context(purpose) - - # Re-enable detection of unexpected EOFs if it was disabled by Python - if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - ssl_context.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - - bio_in = ssl.MemoryBIO() - bio_out = ssl.MemoryBIO() - ssl_object = ssl_context.wrap_bio( - bio_in, bio_out, server_side=server_side, server_hostname=hostname - ) - wrapper = cls( - transport_stream=transport_stream, - standard_compatible=standard_compatible, - _ssl_object=ssl_object, - _read_bio=bio_in, - _write_bio=bio_out, - ) - await wrapper._call_sslobject_method(ssl_object.do_handshake) - return wrapper - - async def _call_sslobject_method( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: - while True: - try: - result = func(*args) - except ssl.SSLWantReadError: - try: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - data = await self.transport_stream.receive() - except EndOfStream: - self._read_bio.write_eof() - except OSError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - else: - self._read_bio.write(data) - except ssl.SSLWantWriteError: - await self.transport_stream.send(self._write_bio.read()) - except ssl.SSLSyscallError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - except ssl.SSLError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - if isinstance(exc, ssl.SSLEOFError) or ( - exc.strerror and "UNEXPECTED_EOF_WHILE_READING" in exc.strerror - ): - if self.standard_compatible: - raise BrokenResourceError from exc - else: - raise EndOfStream from None - - raise - else: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - return result - - async def unwrap(self) -> tuple[AnyByteStream, bytes]: - """ - Does the TLS closing handshake. - - :return: a tuple of (wrapped byte stream, bytes left in the read buffer) - - """ - await self._call_sslobject_method(self._ssl_object.unwrap) - self._read_bio.write_eof() - self._write_bio.write_eof() - return self.transport_stream, self._read_bio.read() - - async def aclose(self) -> None: - if self.standard_compatible: - try: - await self.unwrap() - except BaseException: - await aclose_forcefully(self.transport_stream) - raise - - await self.transport_stream.aclose() - - async def receive(self, max_bytes: int = 65536) -> bytes: - data = await self._call_sslobject_method(self._ssl_object.read, max_bytes) - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - await self._call_sslobject_method(self._ssl_object.write, item) - - async def send_eof(self) -> None: - tls_version = self.extra(TLSAttribute.tls_version) - match = re.match(r"TLSv(\d+)(?:\.(\d+))?", tls_version) - if match: - major, minor = int(match.group(1)), int(match.group(2) or 0) - if (major, minor) < (1, 3): - raise NotImplementedError( - f"send_eof() requires at least TLSv1.3; current " - f"session uses {tls_version}" - ) - - raise NotImplementedError( - "send_eof() has not yet been implemented for TLS streams" - ) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.transport_stream.extra_attributes, - TLSAttribute.alpn_protocol: self._ssl_object.selected_alpn_protocol, - TLSAttribute.channel_binding_tls_unique: ( - self._ssl_object.get_channel_binding - ), - TLSAttribute.cipher: self._ssl_object.cipher, - TLSAttribute.peer_certificate: lambda: self._ssl_object.getpeercert(False), - TLSAttribute.peer_certificate_binary: lambda: self._ssl_object.getpeercert( - True - ), - TLSAttribute.server_side: lambda: self._ssl_object.server_side, - TLSAttribute.shared_ciphers: lambda: self._ssl_object.shared_ciphers() - if self._ssl_object.server_side - else None, - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - TLSAttribute.ssl_object: lambda: self._ssl_object, - TLSAttribute.tls_version: self._ssl_object.version, - } - - -@dataclass(eq=False) -class TLSListener(Listener[TLSStream]): - """ - A convenience listener that wraps another listener and auto-negotiates a TLS session - on every accepted connection. - - If the TLS handshake times out or raises an exception, - :meth:`handle_handshake_error` is called to do whatever post-mortem processing is - deemed necessary. - - Supports only the :attr:`~TLSAttribute.standard_compatible` extra attribute. - - :param Listener listener: the listener to wrap - :param ssl_context: the SSL context object - :param standard_compatible: a flag passed through to :meth:`TLSStream.wrap` - :param handshake_timeout: time limit for the TLS handshake - (passed to :func:`~anyio.fail_after`) - """ - - listener: Listener[Any] - ssl_context: ssl.SSLContext - standard_compatible: bool = True - handshake_timeout: float = 30 - - @staticmethod - async def handle_handshake_error(exc: BaseException, stream: AnyByteStream) -> None: - """ - Handle an exception raised during the TLS handshake. - - This method does 3 things: - - #. Forcefully closes the original stream - #. Logs the exception (unless it was a cancellation exception) using the - ``anyio.streams.tls`` logger - #. Reraises the exception if it was a base exception or a cancellation exception - - :param exc: the exception - :param stream: the original stream - - """ - await aclose_forcefully(stream) - - # Log all except cancellation exceptions - if not isinstance(exc, get_cancelled_exc_class()): - # CPython (as of 3.11.5) returns incorrect `sys.exc_info()` here when using - # any asyncio implementation, so we explicitly pass the exception to log - # (https://github.com/python/cpython/issues/108668). Trio does not have this - # issue because it works around the CPython bug. - logging.getLogger(__name__).exception( - "Error during TLS handshake", exc_info=exc - ) - - # Only reraise base exceptions and cancellation exceptions - if not isinstance(exc, Exception) or isinstance(exc, get_cancelled_exc_class()): - raise - - async def serve( - self, - handler: Callable[[TLSStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - @wraps(handler) - async def handler_wrapper(stream: AnyByteStream) -> None: - from .. import fail_after - - try: - with fail_after(self.handshake_timeout): - wrapped_stream = await TLSStream.wrap( - stream, - ssl_context=self.ssl_context, - standard_compatible=self.standard_compatible, - ) - except BaseException as exc: - await self.handle_handshake_error(exc, stream) - else: - await handler(wrapped_stream) - - await self.listener.serve(handler_wrapper, task_group) - - async def aclose(self) -> None: - await self.listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - } diff --git a/contrib/python/anyio/anyio/to_interpreter.py b/contrib/python/anyio/anyio/to_interpreter.py deleted file mode 100644 index bcde24d3d1d..00000000000 --- a/contrib/python/anyio/anyio/to_interpreter.py +++ /dev/null @@ -1,218 +0,0 @@ -from __future__ import annotations - -import atexit -import os -import pickle -import sys -from collections import deque -from collections.abc import Callable -from textwrap import dedent -from typing import Any, Final, TypeVar - -from . import current_time, to_thread -from ._core._exceptions import BrokenWorkerIntepreter -from ._core._synchronization import CapacityLimiter -from .lowlevel import RunVar - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -UNBOUND: Final = 2 # I have no clue how this works, but it was used in the stdlib -FMT_UNPICKLED: Final = 0 -FMT_PICKLED: Final = 1 -DEFAULT_CPU_COUNT: Final = 8 # this is just an arbitrarily selected value -MAX_WORKER_IDLE_TIME = ( - 30 # seconds a subinterpreter can be idle before becoming eligible for pruning -) - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_idle_workers = RunVar[deque["Worker"]]("_available_workers") -_default_interpreter_limiter = RunVar[CapacityLimiter]("_default_interpreter_limiter") - - -class Worker: - _run_func = compile( - dedent(""" - import _interpqueues as queues - import _interpreters as interpreters - from pickle import loads, dumps, HIGHEST_PROTOCOL - - item = queues.get(queue_id)[0] - try: - func, args = loads(item) - retval = func(*args) - except BaseException as exc: - is_exception = True - retval = exc - else: - is_exception = False - - try: - queues.put(queue_id, (retval, is_exception), FMT_UNPICKLED, UNBOUND) - except interpreters.NotShareableError: - retval = dumps(retval, HIGHEST_PROTOCOL) - queues.put(queue_id, (retval, is_exception), FMT_PICKLED, UNBOUND) - """), - "<string>", - "exec", - ) - - last_used: float = 0 - - _initialized: bool = False - _interpreter_id: int - _queue_id: int - - def initialize(self) -> None: - import _interpqueues as queues - import _interpreters as interpreters - - self._interpreter_id = interpreters.create() - self._queue_id = queues.create(2, FMT_UNPICKLED, UNBOUND) # type: ignore[call-arg] - self._initialized = True - interpreters.set___main___attrs( - self._interpreter_id, - { - "queue_id": self._queue_id, - "FMT_PICKLED": FMT_PICKLED, - "FMT_UNPICKLED": FMT_UNPICKLED, - "UNBOUND": UNBOUND, - }, - ) - - def destroy(self) -> None: - import _interpqueues as queues - import _interpreters as interpreters - - if self._initialized: - interpreters.destroy(self._interpreter_id) - queues.destroy(self._queue_id) - - def _call( - self, - func: Callable[..., T_Retval], - args: tuple[Any], - ) -> tuple[Any, bool]: - import _interpqueues as queues - import _interpreters as interpreters - - if not self._initialized: - self.initialize() - - payload = pickle.dumps((func, args), pickle.HIGHEST_PROTOCOL) - queues.put(self._queue_id, payload, FMT_PICKLED, UNBOUND) # type: ignore[call-arg] - - res: Any - is_exception: bool - if exc_info := interpreters.exec(self._interpreter_id, self._run_func): # type: ignore[func-returns-value,arg-type] - raise BrokenWorkerIntepreter(exc_info) - - (res, is_exception), fmt = queues.get(self._queue_id)[:2] - if fmt == FMT_PICKLED: - res = pickle.loads(res) - - return res, is_exception - - async def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any], - limiter: CapacityLimiter, - ) -> T_Retval: - result, is_exception = await to_thread.run_sync( - self._call, - func, - args, - limiter=limiter, - ) - if is_exception: - raise result - - return result - - -def _stop_workers(workers: deque[Worker]) -> None: - for worker in workers: - worker.destroy() - - workers.clear() - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a subinterpreter. - - If the ``cancellable`` option is enabled and the task waiting for its completion is - cancelled, the call will still run its course but its return value (or any raised - exception) will be ignored. - - .. warning:: This feature is **experimental**. The upstream interpreter API has not - yet been finalized or thoroughly tested, so don't rely on this for anything - mission critical. - - :param func: a callable - :param args: positional arguments for the callable - :param limiter: capacity limiter to use to limit the total amount of subinterpreters - running (if omitted, the default limiter is used) - :return: the result of the call - :raises BrokenWorkerIntepreter: if there's an internal error in a subinterpreter - - """ - if sys.version_info <= (3, 13): - raise RuntimeError("subinterpreters require at least Python 3.13") - - if limiter is None: - limiter = current_default_interpreter_limiter() - - try: - idle_workers = _idle_workers.get() - except LookupError: - idle_workers = deque() - _idle_workers.set(idle_workers) - atexit.register(_stop_workers, idle_workers) - - async with limiter: - try: - worker = idle_workers.pop() - except IndexError: - worker = Worker() - - try: - return await worker.call(func, args, limiter) - finally: - # Prune workers that have been idle for too long - now = current_time() - while idle_workers: - if now - idle_workers[0].last_used <= MAX_WORKER_IDLE_TIME: - break - - await to_thread.run_sync(idle_workers.popleft().destroy, limiter=limiter) - - worker.last_used = current_time() - idle_workers.append(worker) - - -def current_default_interpreter_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of - concurrently running subinterpreters. - - Defaults to the number of CPU cores. - - :return: a capacity limiter object - - """ - try: - return _default_interpreter_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or DEFAULT_CPU_COUNT) - _default_interpreter_limiter.set(limiter) - return limiter diff --git a/contrib/python/anyio/anyio/to_process.py b/contrib/python/anyio/anyio/to_process.py deleted file mode 100644 index 495de2ae711..00000000000 --- a/contrib/python/anyio/anyio/to_process.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import annotations - -import os -import pickle -import subprocess -import sys -from collections import deque -from collections.abc import Callable -from importlib.util import module_from_spec, spec_from_file_location -from typing import TypeVar, cast - -from ._core._eventloop import current_time, get_async_backend, get_cancelled_exc_class -from ._core._exceptions import BrokenWorkerProcess -from ._core._subprocesses import open_process -from ._core._synchronization import CapacityLimiter -from ._core._tasks import CancelScope, fail_after -from .abc import ByteReceiveStream, ByteSendStream, Process -from .lowlevel import RunVar, checkpoint_if_cancelled -from .streams.buffered import BufferedByteReceiveStream - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -WORKER_MAX_IDLE_TIME = 300 # 5 minutes - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_process_pool_workers: RunVar[set[Process]] = RunVar("_process_pool_workers") -_process_pool_idle_workers: RunVar[deque[tuple[Process, float]]] = RunVar( - "_process_pool_idle_workers" -) -_default_process_limiter: RunVar[CapacityLimiter] = RunVar("_default_process_limiter") - - -async def run_sync( # type: ignore[return] - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - cancellable: bool = False, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker process. - - If the ``cancellable`` option is enabled and the task waiting for its completion is - cancelled, the worker process running it will be abruptly terminated using SIGKILL - (or ``terminateProcess()`` on Windows). - - :param func: a callable - :param args: positional arguments for the callable - :param cancellable: ``True`` to allow cancellation of the operation while it's - running - :param limiter: capacity limiter to use to limit the total amount of processes - running (if omitted, the default limiter is used) - :return: an awaitable that yields the return value of the function. - - """ - - async def send_raw_command(pickled_cmd: bytes) -> object: - try: - await stdin.send(pickled_cmd) - response = await buffered.receive_until(b"\n", 50) - status, length = response.split(b" ") - if status not in (b"RETURN", b"EXCEPTION"): - raise RuntimeError( - f"Worker process returned unexpected response: {response!r}" - ) - - pickled_response = await buffered.receive_exactly(int(length)) - except BaseException as exc: - workers.discard(process) - try: - process.kill() - with CancelScope(shield=True): - await process.aclose() - except ProcessLookupError: - pass - - if isinstance(exc, get_cancelled_exc_class()): - raise - else: - raise BrokenWorkerProcess from exc - - retval = pickle.loads(pickled_response) - if status == b"EXCEPTION": - assert isinstance(retval, BaseException) - raise retval - else: - return retval - - # First pickle the request before trying to reserve a worker process - await checkpoint_if_cancelled() - request = pickle.dumps(("run", func, args), protocol=pickle.HIGHEST_PROTOCOL) - - # If this is the first run in this event loop thread, set up the necessary variables - try: - workers = _process_pool_workers.get() - idle_workers = _process_pool_idle_workers.get() - except LookupError: - workers = set() - idle_workers = deque() - _process_pool_workers.set(workers) - _process_pool_idle_workers.set(idle_workers) - get_async_backend().setup_process_pool_exit_at_shutdown(workers) - - async with limiter or current_default_process_limiter(): - # Pop processes from the pool (starting from the most recently used) until we - # find one that hasn't exited yet - process: Process - while idle_workers: - process, idle_since = idle_workers.pop() - if process.returncode is None: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - - # Prune any other workers that have been idle for WORKER_MAX_IDLE_TIME - # seconds or longer - now = current_time() - killed_processes: list[Process] = [] - while idle_workers: - if now - idle_workers[0][1] < WORKER_MAX_IDLE_TIME: - break - - process_to_kill, idle_since = idle_workers.popleft() - process_to_kill.kill() - workers.remove(process_to_kill) - killed_processes.append(process_to_kill) - - with CancelScope(shield=True): - for killed_process in killed_processes: - await killed_process.aclose() - - break - - workers.remove(process) - else: - command = [sys.executable, "-u", "-m", __name__] - process = await open_process( - command, stdin=subprocess.PIPE, stdout=subprocess.PIPE - ) - try: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - with fail_after(20): - message = await buffered.receive(6) - - if message != b"READY\n": - raise BrokenWorkerProcess( - f"Worker process returned unexpected response: {message!r}" - ) - - main_module_path = getattr(sys.modules["__main__"], "__file__", None) - pickled = pickle.dumps( - ("init", sys.path, main_module_path), - protocol=pickle.HIGHEST_PROTOCOL, - ) - await send_raw_command(pickled) - except (BrokenWorkerProcess, get_cancelled_exc_class()): - raise - except BaseException as exc: - process.kill() - raise BrokenWorkerProcess( - "Error during worker process initialization" - ) from exc - - workers.add(process) - - with CancelScope(shield=not cancellable): - try: - return cast(T_Retval, await send_raw_command(request)) - finally: - if process in workers: - idle_workers.append((process, current_time())) - - -def current_default_process_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of worker - processes. - - :return: a capacity limiter object - - """ - try: - return _default_process_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or 2) - _default_process_limiter.set(limiter) - return limiter - - -def process_worker() -> None: - # Redirect standard streams to os.devnull so that user code won't interfere with the - # parent-worker communication - stdin = sys.stdin - stdout = sys.stdout - sys.stdin = open(os.devnull) - sys.stdout = open(os.devnull, "w") - - stdout.buffer.write(b"READY\n") - while True: - retval = exception = None - try: - command, *args = pickle.load(stdin.buffer) - except EOFError: - return - except BaseException as exc: - exception = exc - else: - if command == "run": - func, args = args - try: - retval = func(*args) - except BaseException as exc: - exception = exc - elif command == "init": - main_module_path: str | None - sys.path, main_module_path = args - del sys.modules["__main__"] - if main_module_path and os.path.isfile(main_module_path): - # Load the parent's main module but as __mp_main__ instead of - # __main__ (like multiprocessing does) to avoid infinite recursion - try: - spec = spec_from_file_location("__mp_main__", main_module_path) - if spec and spec.loader: - main = module_from_spec(spec) - spec.loader.exec_module(main) - sys.modules["__main__"] = main - except BaseException as exc: - exception = exc - try: - if exception is not None: - status = b"EXCEPTION" - pickled = pickle.dumps(exception, pickle.HIGHEST_PROTOCOL) - else: - status = b"RETURN" - pickled = pickle.dumps(retval, pickle.HIGHEST_PROTOCOL) - except BaseException as exc: - exception = exc - status = b"EXCEPTION" - pickled = pickle.dumps(exc, pickle.HIGHEST_PROTOCOL) - - stdout.buffer.write(b"%s %d\n" % (status, len(pickled))) - stdout.buffer.write(pickled) - - # Respect SIGTERM - if isinstance(exception, SystemExit): - raise exception - - -if __name__ == "__main__": - process_worker() diff --git a/contrib/python/anyio/anyio/to_thread.py b/contrib/python/anyio/anyio/to_thread.py deleted file mode 100644 index 5070516eb56..00000000000 --- a/contrib/python/anyio/anyio/to_thread.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Callable -from typing import TypeVar -from warnings import warn - -from ._core._eventloop import get_async_backend -from .abc import CapacityLimiter - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - abandon_on_cancel: bool = False, - cancellable: bool | None = None, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker thread. - - If the ``cancellable`` option is enabled and the task waiting for its completion is - cancelled, the thread will still run its course but its return value (or any raised - exception) will be ignored. - - :param func: a callable - :param args: positional arguments for the callable - :param abandon_on_cancel: ``True`` to abandon the thread (leaving it to run - unchecked on own) if the host task is cancelled, ``False`` to ignore - cancellations in the host task until the operation has completed in the worker - thread - :param cancellable: deprecated alias of ``abandon_on_cancel``; will override - ``abandon_on_cancel`` if both parameters are passed - :param limiter: capacity limiter to use to limit the total amount of threads running - (if omitted, the default limiter is used) - :return: an awaitable that yields the return value of the function. - - """ - if cancellable is not None: - abandon_on_cancel = cancellable - warn( - "The `cancellable=` keyword argument to `anyio.to_thread.run_sync` is " - "deprecated since AnyIO 4.1.0; use `abandon_on_cancel=` instead", - DeprecationWarning, - stacklevel=2, - ) - - return await get_async_backend().run_sync_in_worker_thread( - func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter - ) - - -def current_default_thread_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of - concurrent threads. - - :return: a capacity limiter object - - """ - return get_async_backend().current_default_thread_limiter() diff --git a/contrib/python/anyio/ya.make b/contrib/python/anyio/ya.make deleted file mode 100644 index b676cfdd808..00000000000 --- a/contrib/python/anyio/ya.make +++ /dev/null @@ -1,73 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(4.8.0) - -LICENSE(MIT) - -PEERDIR( - contrib/python/idna - contrib/python/sniffio - contrib/python/typing-extensions -) - -NO_LINT() - -NO_CHECK_IMPORTS( - anyio._backends._trio - anyio.pytest_plugin -) - -PY_SRCS( - TOP_LEVEL - anyio/__init__.py - anyio/_backends/__init__.py - anyio/_backends/_asyncio.py - anyio/_backends/_trio.py - anyio/_core/__init__.py - anyio/_core/_asyncio_selector_thread.py - anyio/_core/_eventloop.py - anyio/_core/_exceptions.py - anyio/_core/_fileio.py - anyio/_core/_resources.py - anyio/_core/_signals.py - anyio/_core/_sockets.py - anyio/_core/_streams.py - anyio/_core/_subprocesses.py - anyio/_core/_synchronization.py - anyio/_core/_tasks.py - anyio/_core/_testing.py - anyio/_core/_typedattr.py - anyio/abc/__init__.py - anyio/abc/_eventloop.py - anyio/abc/_resources.py - anyio/abc/_sockets.py - anyio/abc/_streams.py - anyio/abc/_subprocesses.py - anyio/abc/_tasks.py - anyio/abc/_testing.py - anyio/from_thread.py - anyio/lowlevel.py - anyio/pytest_plugin.py - anyio/streams/__init__.py - anyio/streams/buffered.py - anyio/streams/file.py - anyio/streams/memory.py - anyio/streams/stapled.py - anyio/streams/text.py - anyio/streams/tls.py - anyio/to_interpreter.py - anyio/to_process.py - anyio/to_thread.py -) - -RESOURCE_FILES( - PREFIX contrib/python/anyio/ - .dist-info/METADATA - .dist-info/entry_points.txt - .dist-info/top_level.txt - anyio/py.typed -) - -END() diff --git a/contrib/python/cachetools/py3/.dist-info/METADATA b/contrib/python/cachetools/py3/.dist-info/METADATA index b3ff29d8995..386951ecb89 100644 --- a/contrib/python/cachetools/py3/.dist-info/METADATA +++ b/contrib/python/cachetools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: cachetools -Version: 5.5.0 +Version: 5.5.1 Summary: Extensible memoizing collections and decorators Home-page: https://github.com/tkem/cachetools/ Author: Thomas Kemmer @@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Topic :: Software Development :: Libraries :: Python Modules Requires-Python: >=3.7 License-File: LICENSE diff --git a/contrib/python/cachetools/py3/cachetools/__init__.py b/contrib/python/cachetools/py3/cachetools/__init__.py index 2d2e2cf4ae0..ad301758612 100644 --- a/contrib/python/cachetools/py3/cachetools/__init__.py +++ b/contrib/python/cachetools/py3/cachetools/__init__.py @@ -13,7 +13,7 @@ __all__ = ( "cachedmethod", ) -__version__ = "5.5.0" +__version__ = "5.5.1" import collections import collections.abc diff --git a/contrib/python/cachetools/py3/ya.make b/contrib/python/cachetools/py3/ya.make index 16c48a0f31e..1581ce124b3 100644 --- a/contrib/python/cachetools/py3/ya.make +++ b/contrib/python/cachetools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(5.5.0) +VERSION(5.5.1) LICENSE(MIT) diff --git a/contrib/python/fonttools/.dist-info/METADATA b/contrib/python/fonttools/.dist-info/METADATA index ca6f7ccb089..931191c2892 100644 --- a/contrib/python/fonttools/.dist-info/METADATA +++ b/contrib/python/fonttools/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: fonttools -Version: 4.55.3 +Version: 4.55.4 Summary: Tools to manipulate font files Home-page: http://github.com/fonttools/fonttools Author: Just van Rossum @@ -72,6 +72,18 @@ Requires-Dist: sympy; extra == "all" Requires-Dist: xattr; sys_platform == "darwin" and extra == "all" Requires-Dist: skia-pathops>=0.5.0; extra == "all" Requires-Dist: uharfbuzz>=0.23.0; extra == "all" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: home-page +Dynamic: license +Dynamic: maintainer +Dynamic: maintainer-email +Dynamic: platform +Dynamic: provides-extra +Dynamic: requires-python +Dynamic: summary |CI Build Status| |Coverage Status| |PyPI| |Gitter Chat| @@ -377,9 +389,15 @@ Have fun! Changelog ~~~~~~~~~ -4.55.3 (released 2024-12-10) +4.55.4 (released 2025-01-21) ---------------------------- +- [bezierTools] Fixed ``splitCubicAtT`` sometimes not returning identical start/end points as result of numerical precision (#3742, #3743). +- [feaLib/ast] Fixed docstring of ``AlternateSubstStatement`` (#3735). +- [transform] Typing fixes (#3734). + +4.55.3 (released 2024-12-10) +---------------------------- - [Docs] fill out ttLib table section [#3716] - [feaLib] More efficient inline format 4 lookups [#3726] @@ -403,7 +421,6 @@ Changelog 4.55.0 (released 2024-11-14) ---------------------------- - - [cffLib.specializer] Adjust stack use calculation (#3689) - [varLib] Lets not add mac names if the rest of name doesn't have them (#3688) - [ttLib.reorderGlyphs] Update CFF table charstrings and charset (#3682) diff --git a/contrib/python/fonttools/fontTools/__init__.py b/contrib/python/fonttools/fontTools/__init__.py index 0af1b7a5eca..d1af15d948f 100644 --- a/contrib/python/fonttools/fontTools/__init__.py +++ b/contrib/python/fonttools/fontTools/__init__.py @@ -3,6 +3,6 @@ from fontTools.misc.loggingTools import configLogger log = logging.getLogger(__name__) -version = __version__ = "4.55.3" +version = __version__ = "4.55.4" __all__ = ["version", "log", "configLogger"] diff --git a/contrib/python/fonttools/fontTools/feaLib/ast.py b/contrib/python/fonttools/fontTools/feaLib/ast.py index 17c6cc3fbe4..b9bab88ef23 100644 --- a/contrib/python/fonttools/fontTools/feaLib/ast.py +++ b/contrib/python/fonttools/fontTools/feaLib/ast.py @@ -595,8 +595,8 @@ class MarkClassDefinition(Statement): class AlternateSubstStatement(Statement): """A ``sub ... from ...`` statement. - ``prefix``, ``glyph``, ``suffix`` and ``replacement`` should be lists of - `glyph-containing objects`_. ``glyph`` should be a `one element list`.""" + ``glyph`` and ``replacement`` should be `glyph-containing objects`_. + ``prefix`` and ``suffix`` should be lists of `glyph-containing objects`_.""" def __init__(self, prefix, glyph, suffix, replacement, location=None): Statement.__init__(self, location) diff --git a/contrib/python/fonttools/fontTools/misc/bezierTools.py b/contrib/python/fonttools/fontTools/misc/bezierTools.py index 1b37ade8d67..2021f244370 100644 --- a/contrib/python/fonttools/fontTools/misc/bezierTools.py +++ b/contrib/python/fonttools/fontTools/misc/bezierTools.py @@ -631,7 +631,14 @@ def splitCubicAtT(pt1, pt2, pt3, pt4, *ts): ((77.3438, 56.25), (85.9375, 43.75), (93.75, 25), (100, 0)) """ a, b, c, d = calcCubicParameters(pt1, pt2, pt3, pt4) - return _splitCubicAtT(a, b, c, d, *ts) + split = _splitCubicAtT(a, b, c, d, *ts) + + # the split impl can introduce floating point errors; we know the first + # segment should always start at pt1 and the last segment should end at pt4, + # so we set those values directly before returning. + split[0] = (pt1, *split[0][1:]) + split[-1] = (*split[-1][:-1], pt4) + return split @cython.locals( diff --git a/contrib/python/fonttools/fontTools/misc/transform.py b/contrib/python/fonttools/fontTools/misc/transform.py index 2f4a216fa69..aeacc30fcb1 100644 --- a/contrib/python/fonttools/fontTools/misc/transform.py +++ b/contrib/python/fonttools/fontTools/misc/transform.py @@ -52,6 +52,8 @@ translate, rotation, scale, skew, and transformation-center components. >>> """ +from __future__ import annotations + import math from typing import NamedTuple from dataclasses import dataclass @@ -65,7 +67,7 @@ _ONE_EPSILON = 1 - _EPSILON _MINUS_ONE_EPSILON = -1 + _EPSILON -def _normSinCos(v): +def _normSinCos(v: float) -> float: if abs(v) < _EPSILON: v = 0 elif v > _ONE_EPSILON: @@ -214,7 +216,7 @@ class Transform(NamedTuple): xx, xy, yx, yy = self[:4] return [(xx * dx + yx * dy, xy * dx + yy * dy) for dx, dy in vectors] - def translate(self, x=0, y=0): + def translate(self, x: float = 0, y: float = 0): """Return a new transformation, translated (offset) by x, y. :Example: @@ -225,7 +227,7 @@ class Transform(NamedTuple): """ return self.transform((1, 0, 0, 1, x, y)) - def scale(self, x=1, y=None): + def scale(self, x: float = 1, y: float | None = None): """Return a new transformation, scaled by x, y. The 'y' argument may be None, which implies to use the x value for y as well. @@ -241,7 +243,7 @@ class Transform(NamedTuple): y = x return self.transform((x, 0, 0, y, 0, 0)) - def rotate(self, angle): + def rotate(self, angle: float): """Return a new transformation, rotated by 'angle' (radians). :Example: @@ -251,13 +253,11 @@ class Transform(NamedTuple): <Transform [0 1 -1 0 0 0]> >>> """ - import math - c = _normSinCos(math.cos(angle)) s = _normSinCos(math.sin(angle)) return self.transform((c, s, -s, c, 0, 0)) - def skew(self, x=0, y=0): + def skew(self, x: float = 0, y: float = 0): """Return a new transformation, skewed by x and y. :Example: @@ -267,8 +267,6 @@ class Transform(NamedTuple): <Transform [1 0 1 1 0 0]> >>> """ - import math - return self.transform((1, math.tan(y), math.tan(x), 1, 0, 0)) def transform(self, other): @@ -336,7 +334,7 @@ class Transform(NamedTuple): dx, dy = -xx * dx - yx * dy, -xy * dx - yy * dy return self.__class__(xx, xy, yx, yy, dx, dy) - def toPS(self): + def toPS(self) -> str: """Return a PostScript representation :Example: @@ -352,7 +350,7 @@ class Transform(NamedTuple): """Decompose into a DecomposedTransform.""" return DecomposedTransform.fromTransform(self) - def __bool__(self): + def __bool__(self) -> bool: """Returns True if transform is not identity, False otherwise. :Example: @@ -374,14 +372,14 @@ class Transform(NamedTuple): """ return self != Identity - def __repr__(self): + def __repr__(self) -> str: return "<%s [%g %g %g %g %g %g]>" % ((self.__class__.__name__,) + self) Identity = Transform() -def Offset(x=0, y=0): +def Offset(x: float = 0, y: float = 0) -> Transform: """Return the identity transformation offset by x, y. :Example: @@ -392,7 +390,7 @@ def Offset(x=0, y=0): return Transform(1, 0, 0, 1, x, y) -def Scale(x, y=None): +def Scale(x: float, y: float | None = None) -> Transform: """Return the identity transformation scaled by x, y. The 'y' argument may be None, which implies to use the x value for y as well. @@ -492,7 +490,7 @@ class DecomposedTransform: 0, ) - def toTransform(self): + def toTransform(self) -> Transform: """Return the Transform() equivalent of this transformation. :Example: diff --git a/contrib/python/fonttools/fontTools/pens/statisticsPen.py b/contrib/python/fonttools/fontTools/pens/statisticsPen.py index b91d93b6eb6..874a3c5b8d9 100644 --- a/contrib/python/fonttools/fontTools/pens/statisticsPen.py +++ b/contrib/python/fonttools/fontTools/pens/statisticsPen.py @@ -106,6 +106,7 @@ class StatisticsControlPen(StatisticsBase, BasePen): def _moveTo(self, pt): self._nodes.append(complex(*pt)) + self._startPoint = pt def _lineTo(self, pt): self._nodes.append(complex(*pt)) @@ -119,12 +120,16 @@ class StatisticsControlPen(StatisticsBase, BasePen): self._nodes.append(complex(*pt)) def _closePath(self): + p0 = self._getCurrentPoint() + if p0 != self._startPoint: + self._lineTo(self._startPoint) self._update() def _endPath(self): p0 = self._getCurrentPoint() if p0 != self._startPoint: raise OpenContourError("Glyph statistics not defined on open contours.") + self._update() def _update(self): nodes = self._nodes diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_n_a_m_e.py b/contrib/python/fonttools/fontTools/ttLib/tables/_n_a_m_e.py index 3fd1cdb4834..31653f08eca 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_n_a_m_e.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_n_a_m_e.py @@ -48,6 +48,10 @@ class table__n_a_m_e(DefaultTable.DefaultTable): dependencies = ["ltag"] + def __init__(self, tag=None): + super().__init__(tag) + self.names = [] + def decompile(self, data, ttFont): format, n, stringOffset = struct.unpack(b">HHH", data[:6]) expectedStringOffset = 6 + n * nameRecordSize @@ -78,10 +82,6 @@ class table__n_a_m_e(DefaultTable.DefaultTable): self.names.append(name) def compile(self, ttFont): - if not hasattr(self, "names"): - # only happens when there are NO name table entries read - # from the TTX file - self.names = [] names = self.names names.sort() # sort according to the spec; see NameRecord.__lt__() stringData = b"" @@ -108,8 +108,6 @@ class table__n_a_m_e(DefaultTable.DefaultTable): def fromXML(self, name, attrs, content, ttFont): if name != "namerecord": return # ignore unknown tags - if not hasattr(self, "names"): - self.names = [] name = NameRecord() self.names.append(name) name.fromXML(name, attrs, content, ttFont) @@ -194,8 +192,6 @@ class table__n_a_m_e(DefaultTable.DefaultTable): identified by the (platformID, platEncID, langID) triplet. A warning is issued to prevent unexpected results. """ - if not hasattr(self, "names"): - self.names = [] if not isinstance(string, str): if isinstance(string, bytes): log.warning( @@ -262,7 +258,7 @@ class table__n_a_m_e(DefaultTable.DefaultTable): The nameID is assigned in the range between 'minNameID' and 32767 (inclusive), following the last nameID in the name table. """ - names = getattr(self, "names", []) + names = self.names nameID = 1 + max([n.nameID for n in names] + [minNameID - 1]) if nameID > 32767: raise ValueError("nameID must be less than 32768") @@ -359,8 +355,6 @@ class table__n_a_m_e(DefaultTable.DefaultTable): If the 'nameID' argument is None, the created nameID will not be less than the 'minNameID' argument. """ - if not hasattr(self, "names"): - self.names = [] if nameID is None: # Reuse nameID if possible nameID = self.findMultilingualName( @@ -404,8 +398,6 @@ class table__n_a_m_e(DefaultTable.DefaultTable): assert ( len(platforms) > 0 ), "'platforms' must contain at least one (platformID, platEncID, langID) tuple" - if not hasattr(self, "names"): - self.names = [] if not isinstance(string, str): raise TypeError( "expected str, found %s: %r" % (type(string).__name__, string) diff --git a/contrib/python/fonttools/ya.make b/contrib/python/fonttools/ya.make index a5f677fa7df..461b5646645 100644 --- a/contrib/python/fonttools/ya.make +++ b/contrib/python/fonttools/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.55.3) +VERSION(4.55.4) LICENSE(MIT) diff --git a/contrib/python/h11/.dist-info/METADATA b/contrib/python/h11/.dist-info/METADATA deleted file mode 100644 index cf12a82f193..00000000000 --- a/contrib/python/h11/.dist-info/METADATA +++ /dev/null @@ -1,193 +0,0 @@ -Metadata-Version: 2.1 -Name: h11 -Version: 0.14.0 -Summary: A pure-Python, bring-your-own-I/O implementation of HTTP/1.1 -Home-page: https://github.com/python-hyper/h11 -Author: Nathaniel J. Smith -Author-email: [email protected] -License: MIT -Classifier: Development Status :: 3 - Alpha -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Topic :: Internet :: WWW/HTTP -Classifier: Topic :: System :: Networking -Requires-Python: >=3.7 -License-File: LICENSE.txt -Requires-Dist: typing-extensions ; python_version < "3.8" - -h11 -=== - -.. image:: https://travis-ci.org/python-hyper/h11.svg?branch=master - :target: https://travis-ci.org/python-hyper/h11 - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-hyper/h11/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-hyper/h11 - :alt: Test coverage - -.. image:: https://readthedocs.org/projects/h11/badge/?version=latest - :target: http://h11.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -This is a little HTTP/1.1 library written from scratch in Python, -heavily inspired by `hyper-h2 <https://hyper-h2.readthedocs.io/>`_. - -It's a "bring-your-own-I/O" library; h11 contains no IO code -whatsoever. This means you can hook h11 up to your favorite network -API, and that could be anything you want: synchronous, threaded, -asynchronous, or your own implementation of `RFC 6214 -<https://tools.ietf.org/html/rfc6214>`_ -- h11 won't judge you. -(Compare this to the current state of the art, where every time a `new -network API <https://trio.readthedocs.io/>`_ comes along then someone -gets to start over reimplementing the entire HTTP protocol from -scratch.) Cory Benfield made an `excellent blog post describing the -benefits of this approach -<https://lukasa.co.uk/2015/10/The_New_Hyper/>`_, or if you like video -then here's his `PyCon 2016 talk on the same theme -<https://www.youtube.com/watch?v=7cC3_jGwl_U>`_. - -This also means that h11 is not immediately useful out of the box: -it's a toolkit for building programs that speak HTTP, not something -that could directly replace ``requests`` or ``twisted.web`` or -whatever. But h11 makes it much easier to implement something like -``requests`` or ``twisted.web``. - -At a high level, working with h11 goes like this: - -1) First, create an ``h11.Connection`` object to track the state of a - single HTTP/1.1 connection. - -2) When you read data off the network, pass it to - ``conn.receive_data(...)``; you'll get back a list of objects - representing high-level HTTP "events". - -3) When you want to send a high-level HTTP event, create the - corresponding "event" object and pass it to ``conn.send(...)``; - this will give you back some bytes that you can then push out - through the network. - -For example, a client might instantiate and then send a -``h11.Request`` object, then zero or more ``h11.Data`` objects for the -request body (e.g., if this is a POST), and then a -``h11.EndOfMessage`` to indicate the end of the message. Then the -server would then send back a ``h11.Response``, some ``h11.Data``, and -its own ``h11.EndOfMessage``. If either side violates the protocol, -you'll get a ``h11.ProtocolError`` exception. - -h11 is suitable for implementing both servers and clients, and has a -pleasantly symmetric API: the events you send as a client are exactly -the ones that you receive as a server and vice-versa. - -`Here's an example of a tiny HTTP client -<https://github.com/python-hyper/h11/blob/master/examples/basic-client.py>`_ - -It also has `a fine manual <https://h11.readthedocs.io/>`_. - -FAQ ---- - -*Whyyyyy?* - -I wanted to play with HTTP in `Curio -<https://curio.readthedocs.io/en/latest/tutorial.html>`__ and `Trio -<https://trio.readthedocs.io>`__, which at the time didn't have any -HTTP libraries. So I thought, no big deal, Python has, like, a dozen -different implementations of HTTP, surely I can find one that's -reusable. I didn't find one, but I did find Cory's call-to-arms -blog-post. So I figured, well, fine, if I have to implement HTTP from -scratch, at least I can make sure no-one *else* has to ever again. - -*Should I use it?* - -Maybe. You should be aware that it's a very young project. But, it's -feature complete and has an exhaustive test-suite and complete docs, -so the next step is for people to try using it and see how it goes -:-). If you do then please let us know -- if nothing else we'll want -to talk to you before making any incompatible changes! - -*What are the features/limitations?* - -Roughly speaking, it's trying to be a robust, complete, and non-hacky -implementation of the first "chapter" of the HTTP/1.1 spec: `RFC 7230: -HTTP/1.1 Message Syntax and Routing -<https://tools.ietf.org/html/rfc7230>`_. That is, it mostly focuses on -implementing HTTP at the level of taking bytes on and off the wire, -and the headers related to that, and tries to be anal about spec -conformance. It doesn't know about higher-level concerns like URL -routing, conditional GETs, cross-origin cookie policies, or content -negotiation. But it does know how to take care of framing, -cross-version differences in keep-alive handling, and the "obsolete -line folding" rule, so you can focus your energies on the hard / -interesting parts for your application, and it tries to support the -full specification in the sense that any useful HTTP/1.1 conformant -application should be able to use h11. - -It's pure Python, and has no dependencies outside of the standard -library. - -It has a test suite with 100.0% coverage for both statements and -branches. - -Currently it supports Python 3 (testing on 3.7-3.10) and PyPy 3. -The last Python 2-compatible version was h11 0.11.x. -(Originally it had a Cython wrapper for `http-parser -<https://github.com/nodejs/http-parser>`_ and a beautiful nested state -machine implemented with ``yield from`` to postprocess the output. But -I had to take these out -- the new *parser* needs fewer lines-of-code -than the old *parser wrapper*, is written in pure Python, uses no -exotic language syntax, and has more features. It's sad, really; that -old state machine was really slick. I just need a few sentences here -to mourn that.) - -I don't know how fast it is. I haven't benchmarked or profiled it yet, -so it's probably got a few pointless hot spots, and I've been trying -to err on the side of simplicity and robustness instead of -micro-optimization. But at the architectural level I tried hard to -avoid fundamentally bad decisions, e.g., I believe that all the -parsing algorithms remain linear-time even in the face of pathological -input like slowloris, and there are no byte-by-byte loops. (I also -believe that it maintains bounded memory usage in the face of -arbitrary/pathological input.) - -The whole library is ~800 lines-of-code. You can read and understand -the whole thing in less than an hour. Most of the energy invested in -this so far has been spent on trying to keep things simple by -minimizing special-cases and ad hoc state manipulation; even though it -is now quite small and simple, I'm still annoyed that I haven't -figured out how to make it even smaller and simpler. (Unfortunately, -HTTP does not lend itself to simplicity.) - -The API is ~feature complete and I don't expect the general outlines -to change much, but you can't judge an API's ergonomics until you -actually document and use it, so I'd expect some changes in the -details. - -*How do I try it?* - -.. code-block:: sh - - $ pip install h11 - $ git clone [email protected]:python-hyper/h11 - $ cd h11/examples - $ python basic-client.py - -and go from there. - -*License?* - -MIT - -*Code of conduct?* - -Contributors are requested to follow our `code of conduct -<https://github.com/python-hyper/h11/blob/master/CODE_OF_CONDUCT.md>`_ in -all project spaces. diff --git a/contrib/python/h11/.dist-info/top_level.txt b/contrib/python/h11/.dist-info/top_level.txt deleted file mode 100644 index 0d24def7113..00000000000 --- a/contrib/python/h11/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -h11 diff --git a/contrib/python/h11/LICENSE.txt b/contrib/python/h11/LICENSE.txt deleted file mode 100644 index 8f080eae848..00000000000 --- a/contrib/python/h11/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Nathaniel J. Smith <[email protected]> and other contributors - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/h11/README.rst b/contrib/python/h11/README.rst deleted file mode 100644 index 56e277e3d18..00000000000 --- a/contrib/python/h11/README.rst +++ /dev/null @@ -1,168 +0,0 @@ -h11 -=== - -.. image:: https://travis-ci.org/python-hyper/h11.svg?branch=master - :target: https://travis-ci.org/python-hyper/h11 - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-hyper/h11/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-hyper/h11 - :alt: Test coverage - -.. image:: https://readthedocs.org/projects/h11/badge/?version=latest - :target: http://h11.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -This is a little HTTP/1.1 library written from scratch in Python, -heavily inspired by `hyper-h2 <https://hyper-h2.readthedocs.io/>`_. - -It's a "bring-your-own-I/O" library; h11 contains no IO code -whatsoever. This means you can hook h11 up to your favorite network -API, and that could be anything you want: synchronous, threaded, -asynchronous, or your own implementation of `RFC 6214 -<https://tools.ietf.org/html/rfc6214>`_ -- h11 won't judge you. -(Compare this to the current state of the art, where every time a `new -network API <https://trio.readthedocs.io/>`_ comes along then someone -gets to start over reimplementing the entire HTTP protocol from -scratch.) Cory Benfield made an `excellent blog post describing the -benefits of this approach -<https://lukasa.co.uk/2015/10/The_New_Hyper/>`_, or if you like video -then here's his `PyCon 2016 talk on the same theme -<https://www.youtube.com/watch?v=7cC3_jGwl_U>`_. - -This also means that h11 is not immediately useful out of the box: -it's a toolkit for building programs that speak HTTP, not something -that could directly replace ``requests`` or ``twisted.web`` or -whatever. But h11 makes it much easier to implement something like -``requests`` or ``twisted.web``. - -At a high level, working with h11 goes like this: - -1) First, create an ``h11.Connection`` object to track the state of a - single HTTP/1.1 connection. - -2) When you read data off the network, pass it to - ``conn.receive_data(...)``; you'll get back a list of objects - representing high-level HTTP "events". - -3) When you want to send a high-level HTTP event, create the - corresponding "event" object and pass it to ``conn.send(...)``; - this will give you back some bytes that you can then push out - through the network. - -For example, a client might instantiate and then send a -``h11.Request`` object, then zero or more ``h11.Data`` objects for the -request body (e.g., if this is a POST), and then a -``h11.EndOfMessage`` to indicate the end of the message. Then the -server would then send back a ``h11.Response``, some ``h11.Data``, and -its own ``h11.EndOfMessage``. If either side violates the protocol, -you'll get a ``h11.ProtocolError`` exception. - -h11 is suitable for implementing both servers and clients, and has a -pleasantly symmetric API: the events you send as a client are exactly -the ones that you receive as a server and vice-versa. - -`Here's an example of a tiny HTTP client -<https://github.com/python-hyper/h11/blob/master/examples/basic-client.py>`_ - -It also has `a fine manual <https://h11.readthedocs.io/>`_. - -FAQ ---- - -*Whyyyyy?* - -I wanted to play with HTTP in `Curio -<https://curio.readthedocs.io/en/latest/tutorial.html>`__ and `Trio -<https://trio.readthedocs.io>`__, which at the time didn't have any -HTTP libraries. So I thought, no big deal, Python has, like, a dozen -different implementations of HTTP, surely I can find one that's -reusable. I didn't find one, but I did find Cory's call-to-arms -blog-post. So I figured, well, fine, if I have to implement HTTP from -scratch, at least I can make sure no-one *else* has to ever again. - -*Should I use it?* - -Maybe. You should be aware that it's a very young project. But, it's -feature complete and has an exhaustive test-suite and complete docs, -so the next step is for people to try using it and see how it goes -:-). If you do then please let us know -- if nothing else we'll want -to talk to you before making any incompatible changes! - -*What are the features/limitations?* - -Roughly speaking, it's trying to be a robust, complete, and non-hacky -implementation of the first "chapter" of the HTTP/1.1 spec: `RFC 7230: -HTTP/1.1 Message Syntax and Routing -<https://tools.ietf.org/html/rfc7230>`_. That is, it mostly focuses on -implementing HTTP at the level of taking bytes on and off the wire, -and the headers related to that, and tries to be anal about spec -conformance. It doesn't know about higher-level concerns like URL -routing, conditional GETs, cross-origin cookie policies, or content -negotiation. But it does know how to take care of framing, -cross-version differences in keep-alive handling, and the "obsolete -line folding" rule, so you can focus your energies on the hard / -interesting parts for your application, and it tries to support the -full specification in the sense that any useful HTTP/1.1 conformant -application should be able to use h11. - -It's pure Python, and has no dependencies outside of the standard -library. - -It has a test suite with 100.0% coverage for both statements and -branches. - -Currently it supports Python 3 (testing on 3.7-3.10) and PyPy 3. -The last Python 2-compatible version was h11 0.11.x. -(Originally it had a Cython wrapper for `http-parser -<https://github.com/nodejs/http-parser>`_ and a beautiful nested state -machine implemented with ``yield from`` to postprocess the output. But -I had to take these out -- the new *parser* needs fewer lines-of-code -than the old *parser wrapper*, is written in pure Python, uses no -exotic language syntax, and has more features. It's sad, really; that -old state machine was really slick. I just need a few sentences here -to mourn that.) - -I don't know how fast it is. I haven't benchmarked or profiled it yet, -so it's probably got a few pointless hot spots, and I've been trying -to err on the side of simplicity and robustness instead of -micro-optimization. But at the architectural level I tried hard to -avoid fundamentally bad decisions, e.g., I believe that all the -parsing algorithms remain linear-time even in the face of pathological -input like slowloris, and there are no byte-by-byte loops. (I also -believe that it maintains bounded memory usage in the face of -arbitrary/pathological input.) - -The whole library is ~800 lines-of-code. You can read and understand -the whole thing in less than an hour. Most of the energy invested in -this so far has been spent on trying to keep things simple by -minimizing special-cases and ad hoc state manipulation; even though it -is now quite small and simple, I'm still annoyed that I haven't -figured out how to make it even smaller and simpler. (Unfortunately, -HTTP does not lend itself to simplicity.) - -The API is ~feature complete and I don't expect the general outlines -to change much, but you can't judge an API's ergonomics until you -actually document and use it, so I'd expect some changes in the -details. - -*How do I try it?* - -.. code-block:: sh - - $ pip install h11 - $ git clone [email protected]:python-hyper/h11 - $ cd h11/examples - $ python basic-client.py - -and go from there. - -*License?* - -MIT - -*Code of conduct?* - -Contributors are requested to follow our `code of conduct -<https://github.com/python-hyper/h11/blob/master/CODE_OF_CONDUCT.md>`_ in -all project spaces. diff --git a/contrib/python/h11/h11/__init__.py b/contrib/python/h11/h11/__init__.py deleted file mode 100644 index 989e92c3458..00000000000 --- a/contrib/python/h11/h11/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# A highish-level implementation of the HTTP/1.1 wire protocol (RFC 7230), -# containing no networking code at all, loosely modelled on hyper-h2's generic -# implementation of HTTP/2 (and in particular the h2.connection.H2Connection -# class). There's still a bunch of subtle details you need to get right if you -# want to make this actually useful, because it doesn't implement all the -# semantics to check that what you're asking to write to the wire is sensible, -# but at least it gets you out of dealing with the wire itself. - -from h11._connection import Connection, NEED_DATA, PAUSED -from h11._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from h11._state import ( - CLIENT, - CLOSED, - DONE, - ERROR, - IDLE, - MIGHT_SWITCH_PROTOCOL, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, - SWITCHED_PROTOCOL, -) -from h11._util import LocalProtocolError, ProtocolError, RemoteProtocolError -from h11._version import __version__ - -PRODUCT_ID = "python-h11/" + __version__ - - -__all__ = ( - "Connection", - "NEED_DATA", - "PAUSED", - "ConnectionClosed", - "Data", - "EndOfMessage", - "Event", - "InformationalResponse", - "Request", - "Response", - "CLIENT", - "CLOSED", - "DONE", - "ERROR", - "IDLE", - "MUST_CLOSE", - "SEND_BODY", - "SEND_RESPONSE", - "SERVER", - "SWITCHED_PROTOCOL", - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", -) diff --git a/contrib/python/h11/h11/_abnf.py b/contrib/python/h11/h11/_abnf.py deleted file mode 100644 index 933587fba22..00000000000 --- a/contrib/python/h11/h11/_abnf.py +++ /dev/null @@ -1,132 +0,0 @@ -# We use native strings for all the re patterns, to take advantage of string -# formatting, and then convert to bytestrings when compiling the final re -# objects. - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#whitespace -# OWS = *( SP / HTAB ) -# ; optional whitespace -OWS = r"[ \t]*" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.token.separators -# token = 1*tchar -# -# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" -# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" -# / DIGIT / ALPHA -# ; any VCHAR, except delimiters -token = r"[-!#$%&'*+.^_`|~0-9a-zA-Z]+" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#header.fields -# field-name = token -field_name = token - -# The standard says: -# -# field-value = *( field-content / obs-fold ) -# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -# field-vchar = VCHAR / obs-text -# obs-fold = CRLF 1*( SP / HTAB ) -# ; obsolete line folding -# ; see Section 3.2.4 -# -# https://tools.ietf.org/html/rfc5234#appendix-B.1 -# -# VCHAR = %x21-7E -# ; visible (printing) characters -# -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.quoted-string -# obs-text = %x80-FF -# -# However, the standard definition of field-content is WRONG! It disallows -# fields containing a single visible character surrounded by whitespace, -# e.g. "foo a bar". -# -# See: https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 -# -# So our definition of field_content attempts to fix it up... -# -# Also, we allow lots of control characters, because apparently people assume -# that they're legal in practice (e.g., google analytics makes cookies with -# \x01 in them!): -# https://github.com/python-hyper/h11/issues/57 -# We still don't allow NUL or whitespace, because those are often treated as -# meta-characters and letting them through can lead to nasty issues like SSRF. -vchar = r"[\x21-\x7e]" -vchar_or_obs_text = r"[^\x00\s]" -field_vchar = vchar_or_obs_text -field_content = r"{field_vchar}+(?:[ \t]+{field_vchar}+)*".format(**globals()) - -# We handle obs-fold at a different level, and our fixed-up field_content -# already grows to swallow the whole value, so ? instead of * -field_value = r"({field_content})?".format(**globals()) - -# header-field = field-name ":" OWS field-value OWS -header_field = ( - r"(?P<field_name>{field_name})" - r":" - r"{OWS}" - r"(?P<field_value>{field_value})" - r"{OWS}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#request.line -# -# request-line = method SP request-target SP HTTP-version CRLF -# method = token -# HTTP-version = HTTP-name "/" DIGIT "." DIGIT -# HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive -# -# request-target is complicated (see RFC 7230 sec 5.3) -- could be path, full -# URL, host+port (for connect), or even "*", but in any case we are guaranteed -# that it contists of the visible printing characters. -method = token -request_target = r"{vchar}+".format(**globals()) -http_version = r"HTTP/(?P<http_version>[0-9]\.[0-9])" -request_line = ( - r"(?P<method>{method})" - r" " - r"(?P<target>{request_target})" - r" " - r"{http_version}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#status.line -# -# status-line = HTTP-version SP status-code SP reason-phrase CRLF -# status-code = 3DIGIT -# reason-phrase = *( HTAB / SP / VCHAR / obs-text ) -status_code = r"[0-9]{3}" -reason_phrase = r"([ \t]|{vchar_or_obs_text})*".format(**globals()) -status_line = ( - r"{http_version}" - r" " - r"(?P<status_code>{status_code})" - # However, there are apparently a few too many servers out there that just - # leave out the reason phrase: - # https://github.com/scrapy/scrapy/issues/345#issuecomment-281756036 - # https://github.com/seanmonstar/httparse/issues/29 - # so make it optional. ?: is a non-capturing group. - r"(?: (?P<reason>{reason_phrase}))?".format(**globals()) -) - -HEXDIG = r"[0-9A-Fa-f]" -# Actually -# -# chunk-size = 1*HEXDIG -# -# but we impose an upper-limit to avoid ridiculosity. len(str(2**64)) == 20 -chunk_size = r"({HEXDIG}){{1,20}}".format(**globals()) -# Actually -# -# chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) -# -# but we aren't parsing the things so we don't really care. -chunk_ext = r";.*" -chunk_header = ( - r"(?P<chunk_size>{chunk_size})" - r"(?P<chunk_ext>{chunk_ext})?" - r"{OWS}\r\n".format( - **globals() - ) # Even though the specification does not allow for extra whitespaces, - # we are lenient with trailing whitespaces because some servers on the wild use it. -) diff --git a/contrib/python/h11/h11/_connection.py b/contrib/python/h11/h11/_connection.py deleted file mode 100644 index d1752707598..00000000000 --- a/contrib/python/h11/h11/_connection.py +++ /dev/null @@ -1,633 +0,0 @@ -# This contains the main Connection class. Everything in h11 revolves around -# this. -from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Type, Union - -from ._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from ._headers import get_comma_header, has_expect_100_continue, set_comma_header -from ._readers import READERS, ReadersType -from ._receivebuffer import ReceiveBuffer -from ._state import ( - _SWITCH_CONNECT, - _SWITCH_UPGRADE, - CLIENT, - ConnectionState, - DONE, - ERROR, - MIGHT_SWITCH_PROTOCOL, - SEND_BODY, - SERVER, - SWITCHED_PROTOCOL, -) -from ._util import ( # Import the internal things we need - LocalProtocolError, - RemoteProtocolError, - Sentinel, -) -from ._writers import WRITERS, WritersType - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = ["Connection", "NEED_DATA", "PAUSED"] - - -class NEED_DATA(Sentinel, metaclass=Sentinel): - pass - - -class PAUSED(Sentinel, metaclass=Sentinel): - pass - - -# If we ever have this much buffered without it making a complete parseable -# event, we error out. The only time we really buffer is when reading the -# request/response line + headers together, so this is effectively the limit on -# the size of that. -# -# Some precedents for defaults: -# - node.js: 80 * 1024 -# - tomcat: 8 * 1024 -# - IIS: 16 * 1024 -# - Apache: <8 KiB per line> -DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 1024 - -# RFC 7230's rules for connection lifecycles: -# - If either side says they want to close the connection, then the connection -# must close. -# - HTTP/1.1 defaults to keep-alive unless someone says Connection: close -# - HTTP/1.0 defaults to close unless both sides say Connection: keep-alive -# (and even this is a mess -- e.g. if you're implementing a proxy then -# sending Connection: keep-alive is forbidden). -# -# We simplify life by simply not supporting keep-alive with HTTP/1.0 peers. So -# our rule is: -# - If someone says Connection: close, we will close -# - If someone uses HTTP/1.0, we will close. -def _keep_alive(event: Union[Request, Response]) -> bool: - connection = get_comma_header(event.headers, b"connection") - if b"close" in connection: - return False - if getattr(event, "http_version", b"1.1") < b"1.1": - return False - return True - - -def _body_framing( - request_method: bytes, event: Union[Request, Response] -) -> Tuple[str, Union[Tuple[()], Tuple[int]]]: - # Called when we enter SEND_BODY to figure out framing information for - # this body. - # - # These are the only two events that can trigger a SEND_BODY state: - assert type(event) in (Request, Response) - # Returns one of: - # - # ("content-length", count) - # ("chunked", ()) - # ("http/1.0", ()) - # - # which are (lookup key, *args) for constructing body reader/writer - # objects. - # - # Reference: https://tools.ietf.org/html/rfc7230#section-3.3.3 - # - # Step 1: some responses always have an empty body, regardless of what the - # headers say. - if type(event) is Response: - if ( - event.status_code in (204, 304) - or request_method == b"HEAD" - or (request_method == b"CONNECT" and 200 <= event.status_code < 300) - ): - return ("content-length", (0,)) - # Section 3.3.3 also lists another case -- responses with status_code - # < 200. For us these are InformationalResponses, not Responses, so - # they can't get into this function in the first place. - assert event.status_code >= 200 - - # Step 2: check for Transfer-Encoding (T-E beats C-L): - transfer_encodings = get_comma_header(event.headers, b"transfer-encoding") - if transfer_encodings: - assert transfer_encodings == [b"chunked"] - return ("chunked", ()) - - # Step 3: check for Content-Length - content_lengths = get_comma_header(event.headers, b"content-length") - if content_lengths: - return ("content-length", (int(content_lengths[0]),)) - - # Step 4: no applicable headers; fallback/default depends on type - if type(event) is Request: - return ("content-length", (0,)) - else: - return ("http/1.0", ()) - - -################################################################ -# -# The main Connection class -# -################################################################ - - -class Connection: - """An object encapsulating the state of an HTTP connection. - - Args: - our_role: If you're implementing a client, pass :data:`h11.CLIENT`. If - you're implementing a server, pass :data:`h11.SERVER`. - - max_incomplete_event_size (int): - The maximum number of bytes we're willing to buffer of an - incomplete event. In practice this mostly sets a limit on the - maximum size of the request/response line + headers. If this is - exceeded, then :meth:`next_event` will raise - :exc:`RemoteProtocolError`. - - """ - - def __init__( - self, - our_role: Type[Sentinel], - max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE, - ) -> None: - self._max_incomplete_event_size = max_incomplete_event_size - # State and role tracking - if our_role not in (CLIENT, SERVER): - raise ValueError("expected CLIENT or SERVER, not {!r}".format(our_role)) - self.our_role = our_role - self.their_role: Type[Sentinel] - if our_role is CLIENT: - self.their_role = SERVER - else: - self.their_role = CLIENT - self._cstate = ConnectionState() - - # Callables for converting data->events or vice-versa given the - # current state - self._writer = self._get_io_object(self.our_role, None, WRITERS) - self._reader = self._get_io_object(self.their_role, None, READERS) - - # Holds any unprocessed received data - self._receive_buffer = ReceiveBuffer() - # If this is true, then it indicates that the incoming connection was - # closed *after* the end of whatever's in self._receive_buffer: - self._receive_buffer_closed = False - - # Extra bits of state that don't fit into the state machine. - # - # These two are only used to interpret framing headers for figuring - # out how to read/write response bodies. their_http_version is also - # made available as a convenient public API. - self.their_http_version: Optional[bytes] = None - self._request_method: Optional[bytes] = None - # This is pure flow-control and doesn't at all affect the set of legal - # transitions, so no need to bother ConnectionState with it: - self.client_is_waiting_for_100_continue = False - - @property - def states(self) -> Dict[Type[Sentinel], Type[Sentinel]]: - """A dictionary like:: - - {CLIENT: <client state>, SERVER: <server state>} - - See :ref:`state-machine` for details. - - """ - return dict(self._cstate.states) - - @property - def our_state(self) -> Type[Sentinel]: - """The current state of whichever role we are playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.our_role] - - @property - def their_state(self) -> Type[Sentinel]: - """The current state of whichever role we are NOT playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.their_role] - - @property - def they_are_waiting_for_100_continue(self) -> bool: - return self.their_role is CLIENT and self.client_is_waiting_for_100_continue - - def start_next_cycle(self) -> None: - """Attempt to reset our connection state for a new request/response - cycle. - - If both client and server are in :data:`DONE` state, then resets them - both to :data:`IDLE` state in preparation for a new request/response - cycle on this same connection. Otherwise, raises a - :exc:`LocalProtocolError`. - - See :ref:`keepalive-and-pipelining`. - - """ - old_states = dict(self._cstate.states) - self._cstate.start_next_cycle() - self._request_method = None - # self.their_http_version gets left alone, since it presumably lasts - # beyond a single request/response cycle - assert not self.client_is_waiting_for_100_continue - self._respond_to_state_changes(old_states) - - def _process_error(self, role: Type[Sentinel]) -> None: - old_states = dict(self._cstate.states) - self._cstate.process_error(role) - self._respond_to_state_changes(old_states) - - def _server_switch_event(self, event: Event) -> Optional[Type[Sentinel]]: - if type(event) is InformationalResponse and event.status_code == 101: - return _SWITCH_UPGRADE - if type(event) is Response: - if ( - _SWITCH_CONNECT in self._cstate.pending_switch_proposals - and 200 <= event.status_code < 300 - ): - return _SWITCH_CONNECT - return None - - # All events go through here - def _process_event(self, role: Type[Sentinel], event: Event) -> None: - # First, pass the event through the state machine to make sure it - # succeeds. - old_states = dict(self._cstate.states) - if role is CLIENT and type(event) is Request: - if event.method == b"CONNECT": - self._cstate.process_client_switch_proposal(_SWITCH_CONNECT) - if get_comma_header(event.headers, b"upgrade"): - self._cstate.process_client_switch_proposal(_SWITCH_UPGRADE) - server_switch_event = None - if role is SERVER: - server_switch_event = self._server_switch_event(event) - self._cstate.process_event(role, type(event), server_switch_event) - - # Then perform the updates triggered by it. - - if type(event) is Request: - self._request_method = event.method - - if role is self.their_role and type(event) in ( - Request, - Response, - InformationalResponse, - ): - event = cast(Union[Request, Response, InformationalResponse], event) - self.their_http_version = event.http_version - - # Keep alive handling - # - # RFC 7230 doesn't really say what one should do if Connection: close - # shows up on a 1xx InformationalResponse. I think the idea is that - # this is not supposed to happen. In any case, if it does happen, we - # ignore it. - if type(event) in (Request, Response) and not _keep_alive( - cast(Union[Request, Response], event) - ): - self._cstate.process_keep_alive_disabled() - - # 100-continue - if type(event) is Request and has_expect_100_continue(event): - self.client_is_waiting_for_100_continue = True - if type(event) in (InformationalResponse, Response): - self.client_is_waiting_for_100_continue = False - if role is CLIENT and type(event) in (Data, EndOfMessage): - self.client_is_waiting_for_100_continue = False - - self._respond_to_state_changes(old_states, event) - - def _get_io_object( - self, - role: Type[Sentinel], - event: Optional[Event], - io_dict: Union[ReadersType, WritersType], - ) -> Optional[Callable[..., Any]]: - # event may be None; it's only used when entering SEND_BODY - state = self._cstate.states[role] - if state is SEND_BODY: - # Special case: the io_dict has a dict of reader/writer factories - # that depend on the request/response framing. - framing_type, args = _body_framing( - cast(bytes, self._request_method), cast(Union[Request, Response], event) - ) - return io_dict[SEND_BODY][framing_type](*args) # type: ignore[index] - else: - # General case: the io_dict just has the appropriate reader/writer - # for this state - return io_dict.get((role, state)) # type: ignore[return-value] - - # This must be called after any action that might have caused - # self._cstate.states to change. - def _respond_to_state_changes( - self, - old_states: Dict[Type[Sentinel], Type[Sentinel]], - event: Optional[Event] = None, - ) -> None: - # Update reader/writer - if self.our_state != old_states[self.our_role]: - self._writer = self._get_io_object(self.our_role, event, WRITERS) - if self.their_state != old_states[self.their_role]: - self._reader = self._get_io_object(self.their_role, event, READERS) - - @property - def trailing_data(self) -> Tuple[bytes, bool]: - """Data that has been received, but not yet processed, represented as - a tuple with two elements, where the first is a byte-string containing - the unprocessed data itself, and the second is a bool that is True if - the receive connection was closed. - - See :ref:`switching-protocols` for discussion of why you'd want this. - """ - return (bytes(self._receive_buffer), self._receive_buffer_closed) - - def receive_data(self, data: bytes) -> None: - """Add data to our internal receive buffer. - - This does not actually do any processing on the data, just stores - it. To trigger processing, you have to call :meth:`next_event`. - - Args: - data (:term:`bytes-like object`): - The new data that was just received. - - Special case: If *data* is an empty byte-string like ``b""``, - then this indicates that the remote side has closed the - connection (end of file). Normally this is convenient, because - standard Python APIs like :meth:`file.read` or - :meth:`socket.recv` use ``b""`` to indicate end-of-file, while - other failures to read are indicated using other mechanisms - like raising :exc:`TimeoutError`. When using such an API you - can just blindly pass through whatever you get from ``read`` - to :meth:`receive_data`, and everything will work. - - But, if you have an API where reading an empty string is a - valid non-EOF condition, then you need to be aware of this and - make sure to check for such strings and avoid passing them to - :meth:`receive_data`. - - Returns: - Nothing, but after calling this you should call :meth:`next_event` - to parse the newly received data. - - Raises: - RuntimeError: - Raised if you pass an empty *data*, indicating EOF, and then - pass a non-empty *data*, indicating more data that somehow - arrived after the EOF. - - (Calling ``receive_data(b"")`` multiple times is fine, - and equivalent to calling it once.) - - """ - if data: - if self._receive_buffer_closed: - raise RuntimeError("received close, then received more data?") - self._receive_buffer += data - else: - self._receive_buffer_closed = True - - def _extract_next_receive_event( - self, - ) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - state = self.their_state - # We don't pause immediately when they enter DONE, because even in - # DONE state we can still process a ConnectionClosed() event. But - # if we have data in our buffer, then we definitely aren't getting - # a ConnectionClosed() immediately and we need to pause. - if state is DONE and self._receive_buffer: - return PAUSED - if state is MIGHT_SWITCH_PROTOCOL or state is SWITCHED_PROTOCOL: - return PAUSED - assert self._reader is not None - event = self._reader(self._receive_buffer) - if event is None: - if not self._receive_buffer and self._receive_buffer_closed: - # In some unusual cases (basically just HTTP/1.0 bodies), EOF - # triggers an actual protocol event; in that case, we want to - # return that event, and then the state will change and we'll - # get called again to generate the actual ConnectionClosed(). - if hasattr(self._reader, "read_eof"): - event = self._reader.read_eof() # type: ignore[attr-defined] - else: - event = ConnectionClosed() - if event is None: - event = NEED_DATA - return event # type: ignore[no-any-return] - - def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - """Parse the next event out of our receive buffer, update our internal - state, and return it. - - This is a mutating operation -- think of it like calling :func:`next` - on an iterator. - - Returns: - : One of three things: - - 1) An event object -- see :ref:`events`. - - 2) The special constant :data:`NEED_DATA`, which indicates that - you need to read more data from your socket and pass it to - :meth:`receive_data` before this method will be able to return - any more events. - - 3) The special constant :data:`PAUSED`, which indicates that we - are not in a state where we can process incoming data (usually - because the peer has finished their part of the current - request/response cycle, and you have not yet called - :meth:`start_next_cycle`). See :ref:`flow-control` for details. - - Raises: - RemoteProtocolError: - The peer has misbehaved. You should close the connection - (possibly after sending some kind of 4xx response). - - Once this method returns :class:`ConnectionClosed` once, then all - subsequent calls will also return :class:`ConnectionClosed`. - - If this method raises any exception besides :exc:`RemoteProtocolError` - then that's a bug -- if it happens please file a bug report! - - If this method raises any exception then it also sets - :attr:`Connection.their_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - - if self.their_state is ERROR: - raise RemoteProtocolError("Can't receive data when peer state is ERROR") - try: - event = self._extract_next_receive_event() - if event not in [NEED_DATA, PAUSED]: - self._process_event(self.their_role, cast(Event, event)) - if event is NEED_DATA: - if len(self._receive_buffer) > self._max_incomplete_event_size: - # 431 is "Request header fields too large" which is pretty - # much the only situation where we can get here - raise RemoteProtocolError( - "Receive buffer too long", error_status_hint=431 - ) - if self._receive_buffer_closed: - # We're still trying to complete some event, but that's - # never going to happen because no more data is coming - raise RemoteProtocolError("peer unexpectedly closed connection") - return event - except BaseException as exc: - self._process_error(self.their_role) - if isinstance(exc, LocalProtocolError): - exc._reraise_as_remote_protocol_error() - else: - raise - - def send(self, event: Event) -> Optional[bytes]: - """Convert a high-level event into bytes that can be sent to the peer, - while updating our internal state machine. - - Args: - event: The :ref:`event <events>` to send. - - Returns: - If ``type(event) is ConnectionClosed``, then returns - ``None``. Otherwise, returns a :term:`bytes-like object`. - - Raises: - LocalProtocolError: - Sending this event at this time would violate our - understanding of the HTTP/1.1 protocol. - - If this method raises any exception then it also sets - :attr:`Connection.our_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - data_list = self.send_with_data_passthrough(event) - if data_list is None: - return None - else: - return b"".join(data_list) - - def send_with_data_passthrough(self, event: Event) -> Optional[List[bytes]]: - """Identical to :meth:`send`, except that in situations where - :meth:`send` returns a single :term:`bytes-like object`, this instead - returns a list of them -- and when sending a :class:`Data` event, this - list is guaranteed to contain the exact object you passed in as - :attr:`Data.data`. See :ref:`sendfile` for discussion. - - """ - if self.our_state is ERROR: - raise LocalProtocolError("Can't send data when our state is ERROR") - try: - if type(event) is Response: - event = self._clean_up_response_headers_for_sending(event) - # We want to call _process_event before calling the writer, - # because if someone tries to do something invalid then this will - # give a sensible error message, while our writers all just assume - # they will only receive valid events. But, _process_event might - # change self._writer. So we have to do a little dance: - writer = self._writer - self._process_event(self.our_role, event) - if type(event) is ConnectionClosed: - return None - else: - # In any situation where writer is None, process_event should - # have raised ProtocolError - assert writer is not None - data_list: List[bytes] = [] - writer(event, data_list.append) - return data_list - except: - self._process_error(self.our_role) - raise - - def send_failed(self) -> None: - """Notify the state machine that we failed to send the data it gave - us. - - This causes :attr:`Connection.our_state` to immediately become - :data:`ERROR` -- see :ref:`error-handling` for discussion. - - """ - self._process_error(self.our_role) - - # When sending a Response, we take responsibility for a few things: - # - # - Sometimes you MUST set Connection: close. We take care of those - # times. (You can also set it yourself if you want, and if you do then - # we'll respect that and close the connection at the right time. But you - # don't have to worry about that unless you want to.) - # - # - The user has to set Content-Length if they want it. Otherwise, for - # responses that have bodies (e.g. not HEAD), then we will automatically - # select the right mechanism for streaming a body of unknown length, - # which depends on depending on the peer's HTTP version. - # - # This function's *only* responsibility is making sure headers are set up - # right -- everything downstream just looks at the headers. There are no - # side channels. - def _clean_up_response_headers_for_sending(self, response: Response) -> Response: - assert type(response) is Response - - headers = response.headers - need_close = False - - # HEAD requests need some special handling: they always act like they - # have Content-Length: 0, and that's how _body_framing treats - # them. But their headers are supposed to match what we would send if - # the request was a GET. (Technically there is one deviation allowed: - # we're allowed to leave out the framing headers -- see - # https://tools.ietf.org/html/rfc7231#section-4.3.2 . But it's just as - # easy to get them right.) - method_for_choosing_headers = cast(bytes, self._request_method) - if method_for_choosing_headers == b"HEAD": - method_for_choosing_headers = b"GET" - framing_type, _ = _body_framing(method_for_choosing_headers, response) - if framing_type in ("chunked", "http/1.0"): - # This response has a body of unknown length. - # If our peer is HTTP/1.1, we use Transfer-Encoding: chunked - # If our peer is HTTP/1.0, we use no framing headers, and close the - # connection afterwards. - # - # Make sure to clear Content-Length (in principle user could have - # set both and then we ignored Content-Length b/c - # Transfer-Encoding overwrote it -- this would be naughty of them, - # but the HTTP spec says that if our peer does this then we have - # to fix it instead of erroring out, so we'll accord the user the - # same respect). - headers = set_comma_header(headers, b"content-length", []) - if self.their_http_version is None or self.their_http_version < b"1.1": - # Either we never got a valid request and are sending back an - # error (their_http_version is None), so we assume the worst; - # or else we did get a valid HTTP/1.0 request, so we know that - # they don't understand chunked encoding. - headers = set_comma_header(headers, b"transfer-encoding", []) - # This is actually redundant ATM, since currently we - # unconditionally disable keep-alive when talking to HTTP/1.0 - # peers. But let's be defensive just in case we add - # Connection: keep-alive support later: - if self._request_method != b"HEAD": - need_close = True - else: - headers = set_comma_header(headers, b"transfer-encoding", [b"chunked"]) - - if not self._cstate.keep_alive or need_close: - # Make sure Connection: close is set - connection = set(get_comma_header(headers, b"connection")) - connection.discard(b"keep-alive") - connection.add(b"close") - headers = set_comma_header(headers, b"connection", sorted(connection)) - - return Response( - headers=headers, - status_code=response.status_code, - http_version=response.http_version, - reason=response.reason, - ) diff --git a/contrib/python/h11/h11/_events.py b/contrib/python/h11/h11/_events.py deleted file mode 100644 index 075bf8a469d..00000000000 --- a/contrib/python/h11/h11/_events.py +++ /dev/null @@ -1,369 +0,0 @@ -# High level events that make up HTTP/1.1 conversations. Loosely inspired by -# the corresponding events in hyper-h2: -# -# http://python-hyper.org/h2/en/stable/api.html#events -# -# Don't subclass these. Stuff will break. - -import re -from abc import ABC -from dataclasses import dataclass, field -from typing import Any, cast, Dict, List, Tuple, Union - -from ._abnf import method, request_target -from ._headers import Headers, normalize_and_validate -from ._util import bytesify, LocalProtocolError, validate - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "Event", - "Request", - "InformationalResponse", - "Response", - "Data", - "EndOfMessage", - "ConnectionClosed", -] - -method_re = re.compile(method.encode("ascii")) -request_target_re = re.compile(request_target.encode("ascii")) - - -class Event(ABC): - """ - Base class for h11 events. - """ - - __slots__ = () - - -@dataclass(init=False, frozen=True) -class Request(Event): - """The beginning of an HTTP request. - - Fields: - - .. attribute:: method - - An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte - string. :term:`Bytes-like objects <bytes-like object>` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: target - - The target of an HTTP request, e.g. ``b"/index.html"``, or one of the - more exotic formats described in `RFC 7320, section 5.3 - <https://tools.ietf.org/html/rfc7230#section-5.3>`_. Always a byte - string. :term:`Bytes-like objects <bytes-like object>` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules <headers-format>` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - <http_version-format>` for details. - - """ - - __slots__ = ("method", "headers", "target", "http_version") - - method: bytes - headers: Headers - target: bytes - http_version: bytes - - def __init__( - self, - *, - method: Union[bytes, str], - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - target: Union[bytes, str], - http_version: Union[bytes, str] = b"1.1", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "method", bytesify(method)) - object.__setattr__(self, "target", bytesify(target)) - object.__setattr__(self, "http_version", bytesify(http_version)) - else: - object.__setattr__(self, "method", method) - object.__setattr__(self, "target", target) - object.__setattr__(self, "http_version", http_version) - - # "A server MUST respond with a 400 (Bad Request) status code to any - # HTTP/1.1 request message that lacks a Host header field and to any - # request message that contains more than one Host header field or a - # Host header field with an invalid field-value." - # -- https://tools.ietf.org/html/rfc7230#section-5.4 - host_count = 0 - for name, value in self.headers: - if name == b"host": - host_count += 1 - if self.http_version == b"1.1" and host_count == 0: - raise LocalProtocolError("Missing mandatory Host: header") - if host_count > 1: - raise LocalProtocolError("Found multiple Host: headers") - - validate(method_re, self.method, "Illegal method characters") - validate(request_target_re, self.target, "Illegal target characters") - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class _ResponseBase(Event): - __slots__ = ("headers", "http_version", "reason", "status_code") - - headers: Headers - http_version: bytes - reason: bytes - status_code: int - - def __init__( - self, - *, - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - status_code: int, - http_version: Union[bytes, str] = b"1.1", - reason: Union[bytes, str] = b"", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "reason", bytesify(reason)) - object.__setattr__(self, "http_version", bytesify(http_version)) - if not isinstance(status_code, int): - raise LocalProtocolError("status code must be integer") - # Because IntEnum objects are instances of int, but aren't - # duck-compatible (sigh), see gh-72. - object.__setattr__(self, "status_code", int(status_code)) - else: - object.__setattr__(self, "reason", reason) - object.__setattr__(self, "http_version", http_version) - object.__setattr__(self, "status_code", status_code) - - self.__post_init__() - - def __post_init__(self) -> None: - pass - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class InformationalResponse(_ResponseBase): - """An HTTP informational response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`InformationalResponse`, this is always in the range [100, - 200). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules <headers-format>` for - details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - <http_version-format>` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (100 <= self.status_code < 200): - raise LocalProtocolError( - "InformationalResponse status_code should be in range " - "[100, 200), not {}".format(self.status_code) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Response(_ResponseBase): - """The beginning of an HTTP response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`Response`, this is always in the range [200, - 1000). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules <headers-format>` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - <http_version-format>` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (200 <= self.status_code < 1000): - raise LocalProtocolError( - "Response status_code should be in range [200, 1000), not {}".format( - self.status_code - ) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Data(Event): - """Part of an HTTP message body. - - Fields: - - .. attribute:: data - - A :term:`bytes-like object` containing part of a message body. Or, if - using the ``combine=False`` argument to :meth:`Connection.send`, then - any object that your socket writing code knows what to do with, and for - which calling :func:`len` returns the number of bytes that will be - written -- see :ref:`sendfile` for details. - - .. attribute:: chunk_start - - A marker that indicates whether this data object is from the start of a - chunked transfer encoding chunk. This field is ignored when when a Data - event is provided to :meth:`Connection.send`: it is only valid on - events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - .. attribute:: chunk_end - - A marker that indicates whether this data object is the last for a - given chunked transfer encoding chunk. This field is ignored when when - a Data event is provided to :meth:`Connection.send`: it is only valid - on events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - """ - - __slots__ = ("data", "chunk_start", "chunk_end") - - data: bytes - chunk_start: bool - chunk_end: bool - - def __init__( - self, data: bytes, chunk_start: bool = False, chunk_end: bool = False - ) -> None: - object.__setattr__(self, "data", data) - object.__setattr__(self, "chunk_start", chunk_start) - object.__setattr__(self, "chunk_end", chunk_end) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that -# are forbidden to be sent in a trailer, since processing them as if they were -# present in the header section might bypass external security filters." -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part -# Unfortunately, the list of forbidden fields is long and vague :-/ -@dataclass(init=False, frozen=True) -class EndOfMessage(Event): - """The end of an HTTP message. - - Fields: - - .. attribute:: headers - - Default value: ``[]`` - - Any trailing headers attached to this message, represented as a list of - (name, value) pairs. See :ref:`the header normalization rules - <headers-format>` for details. - - Must be empty unless ``Transfer-Encoding: chunked`` is in use. - - """ - - __slots__ = ("headers",) - - headers: Headers - - def __init__( - self, - *, - headers: Union[ - Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]], None - ] = None, - _parsed: bool = False, - ) -> None: - super().__init__() - if headers is None: - headers = Headers([]) - elif not isinstance(headers, Headers): - headers = normalize_and_validate(headers, _parsed=_parsed) - - object.__setattr__(self, "headers", headers) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(frozen=True) -class ConnectionClosed(Event): - """This event indicates that the sender has closed their outgoing - connection. - - Note that this does not necessarily mean that they can't *receive* further - data, because TCP connections are composed to two one-way channels which - can be closed independently. See :ref:`closing` for details. - - No fields. - """ - - pass diff --git a/contrib/python/h11/h11/_headers.py b/contrib/python/h11/h11/_headers.py deleted file mode 100644 index b97d020b634..00000000000 --- a/contrib/python/h11/h11/_headers.py +++ /dev/null @@ -1,278 +0,0 @@ -import re -from typing import AnyStr, cast, List, overload, Sequence, Tuple, TYPE_CHECKING, Union - -from ._abnf import field_name, field_value -from ._util import bytesify, LocalProtocolError, validate - -if TYPE_CHECKING: - from ._events import Request - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal # type: ignore - - -# Facts -# ----- -# -# Headers are: -# keys: case-insensitive ascii -# values: mixture of ascii and raw bytes -# -# "Historically, HTTP has allowed field content with text in the ISO-8859-1 -# charset [ISO-8859-1], supporting other charsets only through use of -# [RFC2047] encoding. In practice, most HTTP header field values use only a -# subset of the US-ASCII charset [USASCII]. Newly defined header fields SHOULD -# limit their field values to US-ASCII octets. A recipient SHOULD treat other -# octets in field content (obs-text) as opaque data." -# And it deprecates all non-ascii values -# -# Leading/trailing whitespace in header names is forbidden -# -# Values get leading/trailing whitespace stripped -# -# Content-Disposition actually needs to contain unicode semantically; to -# accomplish this it has a terrifically weird way of encoding the filename -# itself as ascii (and even this still has lots of cross-browser -# incompatibilities) -# -# Order is important: -# "a proxy MUST NOT change the order of these field values when forwarding a -# message" -# (and there are several headers where the order indicates a preference) -# -# Multiple occurences of the same header: -# "A sender MUST NOT generate multiple header fields with the same field name -# in a message unless either the entire field value for that header field is -# defined as a comma-separated list [or the header is Set-Cookie which gets a -# special exception]" - RFC 7230. (cookies are in RFC 6265) -# -# So every header aside from Set-Cookie can be merged by b", ".join if it -# occurs repeatedly. But, of course, they can't necessarily be split by -# .split(b","), because quoting. -# -# Given all this mess (case insensitive, duplicates allowed, order is -# important, ...), there doesn't appear to be any standard way to handle -# headers in Python -- they're almost like dicts, but... actually just -# aren't. For now we punt and just use a super simple representation: headers -# are a list of pairs -# -# [(name1, value1), (name2, value2), ...] -# -# where all entries are bytestrings, names are lowercase and have no -# leading/trailing whitespace, and values are bytestrings with no -# leading/trailing whitespace. Searching and updating are done via naive O(n) -# methods. -# -# Maybe a dict-of-lists would be better? - -_content_length_re = re.compile(rb"[0-9]+") -_field_name_re = re.compile(field_name.encode("ascii")) -_field_value_re = re.compile(field_value.encode("ascii")) - - -class Headers(Sequence[Tuple[bytes, bytes]]): - """ - A list-like interface that allows iterating over headers as byte-pairs - of (lowercased-name, value). - - Internally we actually store the representation as three-tuples, - including both the raw original casing, in order to preserve casing - over-the-wire, and the lowercased name, for case-insensitive comparisions. - - r = Request( - method="GET", - target="/", - headers=[("Host", "example.org"), ("Connection", "keep-alive")], - http_version="1.1", - ) - assert r.headers == [ - (b"host", b"example.org"), - (b"connection", b"keep-alive") - ] - assert r.headers.raw_items() == [ - (b"Host", b"example.org"), - (b"Connection", b"keep-alive") - ] - """ - - __slots__ = "_full_items" - - def __init__(self, full_items: List[Tuple[bytes, bytes, bytes]]) -> None: - self._full_items = full_items - - def __bool__(self) -> bool: - return bool(self._full_items) - - def __eq__(self, other: object) -> bool: - return list(self) == list(other) # type: ignore - - def __len__(self) -> int: - return len(self._full_items) - - def __repr__(self) -> str: - return "<Headers(%s)>" % repr(list(self)) - - def __getitem__(self, idx: int) -> Tuple[bytes, bytes]: # type: ignore[override] - _, name, value = self._full_items[idx] - return (name, value) - - def raw_items(self) -> List[Tuple[bytes, bytes]]: - return [(raw_name, value) for raw_name, _, value in self._full_items] - - -HeaderTypes = Union[ - List[Tuple[bytes, bytes]], - List[Tuple[bytes, str]], - List[Tuple[str, bytes]], - List[Tuple[str, str]], -] - - -@overload -def normalize_and_validate(headers: Headers, _parsed: Literal[True]) -> Headers: - ... - - -@overload -def normalize_and_validate(headers: HeaderTypes, _parsed: Literal[False]) -> Headers: - ... - - -@overload -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - ... - - -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - new_headers = [] - seen_content_length = None - saw_transfer_encoding = False - for name, value in headers: - # For headers coming out of the parser, we can safely skip some steps, - # because it always returns bytes and has already run these regexes - # over the data: - if not _parsed: - name = bytesify(name) - value = bytesify(value) - validate(_field_name_re, name, "Illegal header name {!r}", name) - validate(_field_value_re, value, "Illegal header value {!r}", value) - assert isinstance(name, bytes) - assert isinstance(value, bytes) - - raw_name = name - name = name.lower() - if name == b"content-length": - lengths = {length.strip() for length in value.split(b",")} - if len(lengths) != 1: - raise LocalProtocolError("conflicting Content-Length headers") - value = lengths.pop() - validate(_content_length_re, value, "bad Content-Length") - if seen_content_length is None: - seen_content_length = value - new_headers.append((raw_name, name, value)) - elif seen_content_length != value: - raise LocalProtocolError("conflicting Content-Length headers") - elif name == b"transfer-encoding": - # "A server that receives a request message with a transfer coding - # it does not understand SHOULD respond with 501 (Not - # Implemented)." - # https://tools.ietf.org/html/rfc7230#section-3.3.1 - if saw_transfer_encoding: - raise LocalProtocolError( - "multiple Transfer-Encoding headers", error_status_hint=501 - ) - # "All transfer-coding names are case-insensitive" - # -- https://tools.ietf.org/html/rfc7230#section-4 - value = value.lower() - if value != b"chunked": - raise LocalProtocolError( - "Only Transfer-Encoding: chunked is supported", - error_status_hint=501, - ) - saw_transfer_encoding = True - new_headers.append((raw_name, name, value)) - else: - new_headers.append((raw_name, name, value)) - return Headers(new_headers) - - -def get_comma_header(headers: Headers, name: bytes) -> List[bytes]: - # Should only be used for headers whose value is a list of - # comma-separated, case-insensitive values. - # - # The header name `name` is expected to be lower-case bytes. - # - # Connection: meets these criteria (including cast insensitivity). - # - # Content-Length: technically is just a single value (1*DIGIT), but the - # standard makes reference to implementations that do multiple values, and - # using this doesn't hurt. Ditto, case insensitivity doesn't things either - # way. - # - # Transfer-Encoding: is more complex (allows for quoted strings), so - # splitting on , is actually wrong. For example, this is legal: - # - # Transfer-Encoding: foo; options="1,2", chunked - # - # and should be parsed as - # - # foo; options="1,2" - # chunked - # - # but this naive function will parse it as - # - # foo; options="1 - # 2" - # chunked - # - # However, this is okay because the only thing we are going to do with - # any Transfer-Encoding is reject ones that aren't just "chunked", so - # both of these will be treated the same anyway. - # - # Expect: the only legal value is the literal string - # "100-continue". Splitting on commas is harmless. Case insensitive. - # - out: List[bytes] = [] - for _, found_name, found_raw_value in headers._full_items: - if found_name == name: - found_raw_value = found_raw_value.lower() - for found_split_value in found_raw_value.split(b","): - found_split_value = found_split_value.strip() - if found_split_value: - out.append(found_split_value) - return out - - -def set_comma_header(headers: Headers, name: bytes, new_values: List[bytes]) -> Headers: - # The header name `name` is expected to be lower-case bytes. - # - # Note that when we store the header we use title casing for the header - # names, in order to match the conventional HTTP header style. - # - # Simply calling `.title()` is a blunt approach, but it's correct - # here given the cases where we're using `set_comma_header`... - # - # Connection, Content-Length, Transfer-Encoding. - new_headers: List[Tuple[bytes, bytes]] = [] - for found_raw_name, found_name, found_raw_value in headers._full_items: - if found_name != name: - new_headers.append((found_raw_name, found_raw_value)) - for new_value in new_values: - new_headers.append((name.title(), new_value)) - return normalize_and_validate(new_headers) - - -def has_expect_100_continue(request: "Request") -> bool: - # https://tools.ietf.org/html/rfc7231#section-5.1.1 - # "A server that receives a 100-continue expectation in an HTTP/1.0 request - # MUST ignore that expectation." - if request.http_version < b"1.1": - return False - expect = get_comma_header(request.headers, b"expect") - return b"100-continue" in expect diff --git a/contrib/python/h11/h11/_readers.py b/contrib/python/h11/h11/_readers.py deleted file mode 100644 index 08a9574da4a..00000000000 --- a/contrib/python/h11/h11/_readers.py +++ /dev/null @@ -1,247 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each reader is a callable which takes a ReceiveBuffer object, and -# either: -# 1) consumes some of it and returns an Event -# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate() -# and it might raise a LocalProtocolError, so simpler just to always use -# this) -# 3) returns None, meaning "I need more data" -# -# If they have a .read_eof attribute, then this will be called if an EOF is -# received -- but this is optional. Either way, the actual ConnectionClosed -# event will be generated afterwards. -# -# READERS is a dict describing how to pick a reader. It maps states to either: -# - a reader -# - or, for body readers, a dict of per-framing reader factories - -import re -from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union - -from ._abnf import chunk_header, header_field, request_line, status_line -from ._events import Data, EndOfMessage, InformationalResponse, Request, Response -from ._receivebuffer import ReceiveBuffer -from ._state import ( - CLIENT, - CLOSED, - DONE, - IDLE, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, -) -from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate - -__all__ = ["READERS"] - -header_field_re = re.compile(header_field.encode("ascii")) -obs_fold_re = re.compile(rb"[ \t]+") - - -def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: - it = iter(lines) - last: Optional[bytes] = None - for line in it: - match = obs_fold_re.match(line) - if match: - if last is None: - raise LocalProtocolError("continuation line at start of headers") - if not isinstance(last, bytearray): - # Cast to a mutable type, avoiding copy on append to ensure O(n) time - last = bytearray(last) - last += b" " - last += line[match.end() :] - else: - if last is not None: - yield last - last = line - if last is not None: - yield last - - -def _decode_header_lines( - lines: Iterable[bytes], -) -> Iterable[Tuple[bytes, bytes]]: - for line in _obsolete_line_fold(lines): - matches = validate(header_field_re, line, "illegal header line: {!r}", line) - yield (matches["field_name"], matches["field_value"]) - - -request_line_re = re.compile(request_line.encode("ascii")) - - -def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no request line received") - matches = validate( - request_line_re, lines[0], "illegal request line: {!r}", lines[0] - ) - return Request( - headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches - ) - - -status_line_re = re.compile(status_line.encode("ascii")) - - -def maybe_read_from_SEND_RESPONSE_server( - buf: ReceiveBuffer, -) -> Union[InformationalResponse, Response, None]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no response line received") - matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0]) - http_version = ( - b"1.1" if matches["http_version"] is None else matches["http_version"] - ) - reason = b"" if matches["reason"] is None else matches["reason"] - status_code = int(matches["status_code"]) - class_: Union[Type[InformationalResponse], Type[Response]] = ( - InformationalResponse if status_code < 200 else Response - ) - return class_( - headers=list(_decode_header_lines(lines[1:])), - _parsed=True, - status_code=status_code, - reason=reason, - http_version=http_version, - ) - - -class ContentLengthReader: - def __init__(self, length: int) -> None: - self._length = length - self._remaining = length - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._remaining == 0: - return EndOfMessage() - data = buf.maybe_extract_at_most(self._remaining) - if data is None: - return None - self._remaining -= len(data) - return Data(data=data) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(received {} bytes, expected {})".format( - self._length - self._remaining, self._length - ) - ) - - -chunk_header_re = re.compile(chunk_header.encode("ascii")) - - -class ChunkedReader: - def __init__(self) -> None: - self._bytes_in_chunk = 0 - # After reading a chunk, we have to throw away the trailing \r\n; if - # this is >0 then we discard that many bytes before resuming regular - # de-chunkification. - self._bytes_to_discard = 0 - self._reading_trailer = False - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._reading_trailer: - lines = buf.maybe_extract_lines() - if lines is None: - return None - return EndOfMessage(headers=list(_decode_header_lines(lines))) - if self._bytes_to_discard > 0: - data = buf.maybe_extract_at_most(self._bytes_to_discard) - if data is None: - return None - self._bytes_to_discard -= len(data) - if self._bytes_to_discard > 0: - return None - # else, fall through and read some more - assert self._bytes_to_discard == 0 - if self._bytes_in_chunk == 0: - # We need to refill our chunk count - chunk_header = buf.maybe_extract_next_line() - if chunk_header is None: - return None - matches = validate( - chunk_header_re, - chunk_header, - "illegal chunk header: {!r}", - chunk_header, - ) - # XX FIXME: we discard chunk extensions. Does anyone care? - self._bytes_in_chunk = int(matches["chunk_size"], base=16) - if self._bytes_in_chunk == 0: - self._reading_trailer = True - return self(buf) - chunk_start = True - else: - chunk_start = False - assert self._bytes_in_chunk > 0 - data = buf.maybe_extract_at_most(self._bytes_in_chunk) - if data is None: - return None - self._bytes_in_chunk -= len(data) - if self._bytes_in_chunk == 0: - self._bytes_to_discard = 2 - chunk_end = True - else: - chunk_end = False - return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(incomplete chunked read)" - ) - - -class Http10Reader: - def __call__(self, buf: ReceiveBuffer) -> Optional[Data]: - data = buf.maybe_extract_at_most(999999999) - if data is None: - return None - return Data(data=data) - - def read_eof(self) -> EndOfMessage: - return EndOfMessage() - - -def expect_nothing(buf: ReceiveBuffer) -> None: - if buf: - raise LocalProtocolError("Got data when expecting EOF") - return None - - -ReadersType = Dict[ - Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]], - Union[Callable[..., Any], Dict[str, Callable[..., Any]]], -] - -READERS: ReadersType = { - (CLIENT, IDLE): maybe_read_from_IDLE_client, - (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server, - (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server, - (CLIENT, DONE): expect_nothing, - (CLIENT, MUST_CLOSE): expect_nothing, - (CLIENT, CLOSED): expect_nothing, - (SERVER, DONE): expect_nothing, - (SERVER, MUST_CLOSE): expect_nothing, - (SERVER, CLOSED): expect_nothing, - SEND_BODY: { - "chunked": ChunkedReader, - "content-length": ContentLengthReader, - "http/1.0": Http10Reader, - }, -} diff --git a/contrib/python/h11/h11/_receivebuffer.py b/contrib/python/h11/h11/_receivebuffer.py deleted file mode 100644 index e5c4e08a56f..00000000000 --- a/contrib/python/h11/h11/_receivebuffer.py +++ /dev/null @@ -1,153 +0,0 @@ -import re -import sys -from typing import List, Optional, Union - -__all__ = ["ReceiveBuffer"] - - -# Operations we want to support: -# - find next \r\n or \r\n\r\n (\n or \n\n are also acceptable), -# or wait until there is one -# - read at-most-N bytes -# Goals: -# - on average, do this fast -# - worst case, do this in O(n) where n is the number of bytes processed -# Plan: -# - store bytearray, offset, how far we've searched for a separator token -# - use the how-far-we've-searched data to avoid rescanning -# - while doing a stream of uninterrupted processing, advance offset instead -# of constantly copying -# WARNING: -# - I haven't benchmarked or profiled any of this yet. -# -# Note that starting in Python 3.4, deleting the initial n bytes from a -# bytearray is amortized O(n), thanks to some excellent work by Antoine -# Martin: -# -# https://bugs.python.org/issue19087 -# -# This means that if we only supported 3.4+, we could get rid of the code here -# involving self._start and self.compress, because it's doing exactly the same -# thing that bytearray now does internally. -# -# BUT unfortunately, we still support 2.7, and reading short segments out of a -# long buffer MUST be O(bytes read) to avoid DoS issues, so we can't actually -# delete this code. Yet: -# -# https://pythonclock.org/ -# -# (Two things to double-check first though: make sure PyPy also has the -# optimization, and benchmark to make sure it's a win, since we do have a -# slightly clever thing where we delay calling compress() until we've -# processed a whole event, which could in theory be slightly more efficient -# than the internal bytearray support.) -blank_line_regex = re.compile(b"\n\r?\n", re.MULTILINE) - - -class ReceiveBuffer: - def __init__(self) -> None: - self._data = bytearray() - self._next_line_search = 0 - self._multiple_lines_search = 0 - - def __iadd__(self, byteslike: Union[bytes, bytearray]) -> "ReceiveBuffer": - self._data += byteslike - return self - - def __bool__(self) -> bool: - return bool(len(self)) - - def __len__(self) -> int: - return len(self._data) - - # for @property unprocessed_data - def __bytes__(self) -> bytes: - return bytes(self._data) - - def _extract(self, count: int) -> bytearray: - # extracting an initial slice of the data buffer and return it - out = self._data[:count] - del self._data[:count] - - self._next_line_search = 0 - self._multiple_lines_search = 0 - - return out - - def maybe_extract_at_most(self, count: int) -> Optional[bytearray]: - """ - Extract a fixed number of bytes from the buffer. - """ - out = self._data[:count] - if not out: - return None - - return self._extract(count) - - def maybe_extract_next_line(self) -> Optional[bytearray]: - """ - Extract the first line, if it is completed in the buffer. - """ - # Only search in buffer space that we've not already looked at. - search_start_index = max(0, self._next_line_search - 1) - partial_idx = self._data.find(b"\r\n", search_start_index) - - if partial_idx == -1: - self._next_line_search = len(self._data) - return None - - # + 2 is to compensate len(b"\r\n") - idx = partial_idx + 2 - - return self._extract(idx) - - def maybe_extract_lines(self) -> Optional[List[bytearray]]: - """ - Extract everything up to the first blank line, and return a list of lines. - """ - # Handle the case where we have an immediate empty line. - if self._data[:1] == b"\n": - self._extract(1) - return [] - - if self._data[:2] == b"\r\n": - self._extract(2) - return [] - - # Only search in buffer space that we've not already looked at. - match = blank_line_regex.search(self._data, self._multiple_lines_search) - if match is None: - self._multiple_lines_search = max(0, len(self._data) - 2) - return None - - # Truncate the buffer and return it. - idx = match.span(0)[-1] - out = self._extract(idx) - lines = out.split(b"\n") - - for line in lines: - if line.endswith(b"\r"): - del line[-1] - - assert lines[-2] == lines[-1] == b"" - - del lines[-2:] - - return lines - - # In theory we should wait until `\r\n` before starting to validate - # incoming data. However it's interesting to detect (very) invalid data - # early given they might not even contain `\r\n` at all (hence only - # timeout will get rid of them). - # This is not a 100% effective detection but more of a cheap sanity check - # allowing for early abort in some useful cases. - # This is especially interesting when peer is messing up with HTTPS and - # sent us a TLS stream where we were expecting plain HTTP given all - # versions of TLS so far start handshake with a 0x16 message type code. - def is_next_line_obviously_invalid_request_line(self) -> bool: - try: - # HTTP header line must not contain non-printable characters - # and should not start with a space - return self._data[0] < 0x21 - except IndexError: - return False diff --git a/contrib/python/h11/h11/_state.py b/contrib/python/h11/h11/_state.py deleted file mode 100644 index 3593430a74f..00000000000 --- a/contrib/python/h11/h11/_state.py +++ /dev/null @@ -1,367 +0,0 @@ -################################################################ -# The core state machine -################################################################ -# -# Rule 1: everything that affects the state machine and state transitions must -# live here in this file. As much as possible goes into the table-based -# representation, but for the bits that don't quite fit, the actual code and -# state must nonetheless live here. -# -# Rule 2: this file does not know about what role we're playing; it only knows -# about HTTP request/response cycles in the abstract. This ensures that we -# don't cheat and apply different rules to local and remote parties. -# -# -# Theory of operation -# =================== -# -# Possibly the simplest way to think about this is that we actually have 5 -# different state machines here. Yes, 5. These are: -# -# 1) The client state, with its complicated automaton (see the docs) -# 2) The server state, with its complicated automaton (see the docs) -# 3) The keep-alive state, with possible states {True, False} -# 4) The SWITCH_CONNECT state, with possible states {False, True} -# 5) The SWITCH_UPGRADE state, with possible states {False, True} -# -# For (3)-(5), the first state listed is the initial state. -# -# (1)-(3) are stored explicitly in member variables. The last -# two are stored implicitly in the pending_switch_proposals set as: -# (state of 4) == (_SWITCH_CONNECT in pending_switch_proposals) -# (state of 5) == (_SWITCH_UPGRADE in pending_switch_proposals) -# -# And each of these machines has two different kinds of transitions: -# -# a) Event-triggered -# b) State-triggered -# -# Event triggered is the obvious thing that you'd think it is: some event -# happens, and if it's the right event at the right time then a transition -# happens. But there are somewhat complicated rules for which machines can -# "see" which events. (As a rule of thumb, if a machine "sees" an event, this -# means two things: the event can affect the machine, and if the machine is -# not in a state where it expects that event then it's an error.) These rules -# are: -# -# 1) The client machine sees all h11.events objects emitted by the client. -# -# 2) The server machine sees all h11.events objects emitted by the server. -# -# It also sees the client's Request event. -# -# And sometimes, server events are annotated with a _SWITCH_* event. For -# example, we can have a (Response, _SWITCH_CONNECT) event, which is -# different from a regular Response event. -# -# 3) The keep-alive machine sees the process_keep_alive_disabled() event -# (which is derived from Request/Response events), and this event -# transitions it from True -> False, or from False -> False. There's no way -# to transition back. -# -# 4&5) The _SWITCH_* machines transition from False->True when we get a -# Request that proposes the relevant type of switch (via -# process_client_switch_proposals), and they go from True->False when we -# get a Response that has no _SWITCH_* annotation. -# -# So that's event-triggered transitions. -# -# State-triggered transitions are less standard. What they do here is couple -# the machines together. The way this works is, when certain *joint* -# configurations of states are achieved, then we automatically transition to a -# new *joint* state. So, for example, if we're ever in a joint state with -# -# client: DONE -# keep-alive: False -# -# then the client state immediately transitions to: -# -# client: MUST_CLOSE -# -# This is fundamentally different from an event-based transition, because it -# doesn't matter how we arrived at the {client: DONE, keep-alive: False} state -# -- maybe the client transitioned SEND_BODY -> DONE, or keep-alive -# transitioned True -> False. Either way, once this precondition is satisfied, -# this transition is immediately triggered. -# -# What if two conflicting state-based transitions get enabled at the same -# time? In practice there's only one case where this arises (client DONE -> -# MIGHT_SWITCH_PROTOCOL versus DONE -> MUST_CLOSE), and we resolve it by -# explicitly prioritizing the DONE -> MIGHT_SWITCH_PROTOCOL transition. -# -# Implementation -# -------------- -# -# The event-triggered transitions for the server and client machines are all -# stored explicitly in a table. Ditto for the state-triggered transitions that -# involve just the server and client state. -# -# The transitions for the other machines, and the state-triggered transitions -# that involve the other machines, are written out as explicit Python code. -# -# It'd be nice if there were some cleaner way to do all this. This isn't -# *too* terrible, but I feel like it could probably be better. -# -# WARNING -# ------- -# -# The script that generates the state machine diagrams for the docs knows how -# to read out the EVENT_TRIGGERED_TRANSITIONS and STATE_TRIGGERED_TRANSITIONS -# tables. But it can't automatically read the transitions that are written -# directly in Python code. So if you touch those, you need to also update the -# script to keep it in sync! -from typing import cast, Dict, Optional, Set, Tuple, Type, Union - -from ._events import * -from ._util import LocalProtocolError, Sentinel - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "CLIENT", - "SERVER", - "IDLE", - "SEND_RESPONSE", - "SEND_BODY", - "DONE", - "MUST_CLOSE", - "CLOSED", - "MIGHT_SWITCH_PROTOCOL", - "SWITCHED_PROTOCOL", - "ERROR", -] - - -class CLIENT(Sentinel, metaclass=Sentinel): - pass - - -class SERVER(Sentinel, metaclass=Sentinel): - pass - - -# States -class IDLE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_RESPONSE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_BODY(Sentinel, metaclass=Sentinel): - pass - - -class DONE(Sentinel, metaclass=Sentinel): - pass - - -class MUST_CLOSE(Sentinel, metaclass=Sentinel): - pass - - -class CLOSED(Sentinel, metaclass=Sentinel): - pass - - -class ERROR(Sentinel, metaclass=Sentinel): - pass - - -# Switch types -class MIGHT_SWITCH_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class SWITCHED_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_UPGRADE(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_CONNECT(Sentinel, metaclass=Sentinel): - pass - - -EventTransitionType = Dict[ - Type[Sentinel], - Dict[ - Type[Sentinel], - Dict[Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], Type[Sentinel]], - ], -] - -EVENT_TRIGGERED_TRANSITIONS: EventTransitionType = { - CLIENT: { - IDLE: {Request: SEND_BODY, ConnectionClosed: CLOSED}, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - MIGHT_SWITCH_PROTOCOL: {}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, - SERVER: { - IDLE: { - ConnectionClosed: CLOSED, - Response: SEND_BODY, - # Special case: server sees client Request events, in this form - (Request, CLIENT): SEND_RESPONSE, - }, - SEND_RESPONSE: { - InformationalResponse: SEND_RESPONSE, - Response: SEND_BODY, - (InformationalResponse, _SWITCH_UPGRADE): SWITCHED_PROTOCOL, - (Response, _SWITCH_CONNECT): SWITCHED_PROTOCOL, - }, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, -} - -StateTransitionType = Dict[ - Tuple[Type[Sentinel], Type[Sentinel]], Dict[Type[Sentinel], Type[Sentinel]] -] - -# NB: there are also some special-case state-triggered transitions hard-coded -# into _fire_state_triggered_transitions below. -STATE_TRIGGERED_TRANSITIONS: StateTransitionType = { - # (Client state, Server state) -> new states - # Protocol negotiation - (MIGHT_SWITCH_PROTOCOL, SWITCHED_PROTOCOL): {CLIENT: SWITCHED_PROTOCOL}, - # Socket shutdown - (CLOSED, DONE): {SERVER: MUST_CLOSE}, - (CLOSED, IDLE): {SERVER: MUST_CLOSE}, - (ERROR, DONE): {SERVER: MUST_CLOSE}, - (DONE, CLOSED): {CLIENT: MUST_CLOSE}, - (IDLE, CLOSED): {CLIENT: MUST_CLOSE}, - (DONE, ERROR): {CLIENT: MUST_CLOSE}, -} - - -class ConnectionState: - def __init__(self) -> None: - # Extra bits of state that don't quite fit into the state model. - - # If this is False then it enables the automatic DONE -> MUST_CLOSE - # transition. Don't set this directly; call .keep_alive_disabled() - self.keep_alive = True - - # This is a subset of {UPGRADE, CONNECT}, containing the proposals - # made by the client for switching protocols. - self.pending_switch_proposals: Set[Type[Sentinel]] = set() - - self.states: Dict[Type[Sentinel], Type[Sentinel]] = {CLIENT: IDLE, SERVER: IDLE} - - def process_error(self, role: Type[Sentinel]) -> None: - self.states[role] = ERROR - self._fire_state_triggered_transitions() - - def process_keep_alive_disabled(self) -> None: - self.keep_alive = False - self._fire_state_triggered_transitions() - - def process_client_switch_proposal(self, switch_event: Type[Sentinel]) -> None: - self.pending_switch_proposals.add(switch_event) - self._fire_state_triggered_transitions() - - def process_event( - self, - role: Type[Sentinel], - event_type: Type[Event], - server_switch_event: Optional[Type[Sentinel]] = None, - ) -> None: - _event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]] = event_type - if server_switch_event is not None: - assert role is SERVER - if server_switch_event not in self.pending_switch_proposals: - raise LocalProtocolError( - "Received server {} event without a pending proposal".format( - server_switch_event - ) - ) - _event_type = (event_type, server_switch_event) - if server_switch_event is None and _event_type is Response: - self.pending_switch_proposals = set() - self._fire_event_triggered_transitions(role, _event_type) - # Special case: the server state does get to see Request - # events. - if _event_type is Request: - assert role is CLIENT - self._fire_event_triggered_transitions(SERVER, (Request, CLIENT)) - self._fire_state_triggered_transitions() - - def _fire_event_triggered_transitions( - self, - role: Type[Sentinel], - event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], - ) -> None: - state = self.states[role] - try: - new_state = EVENT_TRIGGERED_TRANSITIONS[role][state][event_type] - except KeyError: - event_type = cast(Type[Event], event_type) - raise LocalProtocolError( - "can't handle event type {} when role={} and state={}".format( - event_type.__name__, role, self.states[role] - ) - ) from None - self.states[role] = new_state - - def _fire_state_triggered_transitions(self) -> None: - # We apply these rules repeatedly until converging on a fixed point - while True: - start_states = dict(self.states) - - # It could happen that both these special-case transitions are - # enabled at the same time: - # - # DONE -> MIGHT_SWITCH_PROTOCOL - # DONE -> MUST_CLOSE - # - # For example, this will always be true of a HTTP/1.0 client - # requesting CONNECT. If this happens, the protocol switch takes - # priority. From there the client will either go to - # SWITCHED_PROTOCOL, in which case it's none of our business when - # they close the connection, or else the server will deny the - # request, in which case the client will go back to DONE and then - # from there to MUST_CLOSE. - if self.pending_switch_proposals: - if self.states[CLIENT] is DONE: - self.states[CLIENT] = MIGHT_SWITCH_PROTOCOL - - if not self.pending_switch_proposals: - if self.states[CLIENT] is MIGHT_SWITCH_PROTOCOL: - self.states[CLIENT] = DONE - - if not self.keep_alive: - for role in (CLIENT, SERVER): - if self.states[role] is DONE: - self.states[role] = MUST_CLOSE - - # Tabular state-triggered transitions - joint_state = (self.states[CLIENT], self.states[SERVER]) - changes = STATE_TRIGGERED_TRANSITIONS.get(joint_state, {}) - self.states.update(changes) - - if self.states == start_states: - # Fixed point reached - return - - def start_next_cycle(self) -> None: - if self.states != {CLIENT: DONE, SERVER: DONE}: - raise LocalProtocolError( - "not in a reusable state. self.states={}".format(self.states) - ) - # Can't reach DONE/DONE with any of these active, but still, let's be - # sure. - assert self.keep_alive - assert not self.pending_switch_proposals - self.states = {CLIENT: IDLE, SERVER: IDLE} diff --git a/contrib/python/h11/h11/_util.py b/contrib/python/h11/h11/_util.py deleted file mode 100644 index 67184452907..00000000000 --- a/contrib/python/h11/h11/_util.py +++ /dev/null @@ -1,135 +0,0 @@ -from typing import Any, Dict, NoReturn, Pattern, Tuple, Type, TypeVar, Union - -__all__ = [ - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", - "validate", - "bytesify", -] - - -class ProtocolError(Exception): - """Exception indicating a violation of the HTTP/1.1 protocol. - - This as an abstract base class, with two concrete base classes: - :exc:`LocalProtocolError`, which indicates that you tried to do something - that HTTP/1.1 says is illegal, and :exc:`RemoteProtocolError`, which - indicates that the remote peer tried to do something that HTTP/1.1 says is - illegal. See :ref:`error-handling` for details. - - In addition to the normal :exc:`Exception` features, it has one attribute: - - .. attribute:: error_status_hint - - This gives a suggestion as to what status code a server might use if - this error occurred as part of a request. - - For a :exc:`RemoteProtocolError`, this is useful as a suggestion for - how you might want to respond to a misbehaving peer, if you're - implementing a server. - - For a :exc:`LocalProtocolError`, this can be taken as a suggestion for - how your peer might have responded to *you* if h11 had allowed you to - continue. - - The default is 400 Bad Request, a generic catch-all for protocol - violations. - - """ - - def __init__(self, msg: str, error_status_hint: int = 400) -> None: - if type(self) is ProtocolError: - raise TypeError("tried to directly instantiate ProtocolError") - Exception.__init__(self, msg) - self.error_status_hint = error_status_hint - - -# Strategy: there are a number of public APIs where a LocalProtocolError can -# be raised (send(), all the different event constructors, ...), and only one -# public API where RemoteProtocolError can be raised -# (receive_data()). Therefore we always raise LocalProtocolError internally, -# and then receive_data will translate this into a RemoteProtocolError. -# -# Internally: -# LocalProtocolError is the generic "ProtocolError". -# Externally: -# LocalProtocolError is for local errors and RemoteProtocolError is for -# remote errors. -class LocalProtocolError(ProtocolError): - def _reraise_as_remote_protocol_error(self) -> NoReturn: - # After catching a LocalProtocolError, use this method to re-raise it - # as a RemoteProtocolError. This method must be called from inside an - # except: block. - # - # An easy way to get an equivalent RemoteProtocolError is just to - # modify 'self' in place. - self.__class__ = RemoteProtocolError # type: ignore - # But the re-raising is somewhat non-trivial -- you might think that - # now that we've modified the in-flight exception object, that just - # doing 'raise' to re-raise it would be enough. But it turns out that - # this doesn't work, because Python tracks the exception type - # (exc_info[0]) separately from the exception object (exc_info[1]), - # and we only modified the latter. So we really do need to re-raise - # the new type explicitly. - # On py3, the traceback is part of the exception object, so our - # in-place modification preserved it and we can just re-raise: - raise self - - -class RemoteProtocolError(ProtocolError): - pass - - -def validate( - regex: Pattern[bytes], data: bytes, msg: str = "malformed data", *format_args: Any -) -> Dict[str, bytes]: - match = regex.fullmatch(data) - if not match: - if format_args: - msg = msg.format(*format_args) - raise LocalProtocolError(msg) - return match.groupdict() - - -# Sentinel values -# -# - Inherit identity-based comparison and hashing from object -# - Have a nice repr -# - Have a *bonus property*: type(sentinel) is sentinel -# -# The bonus property is useful if you want to take the return value from -# next_event() and do some sort of dispatch based on type(event). - -_T_Sentinel = TypeVar("_T_Sentinel", bound="Sentinel") - - -class Sentinel(type): - def __new__( - cls: Type[_T_Sentinel], - name: str, - bases: Tuple[type, ...], - namespace: Dict[str, Any], - **kwds: Any - ) -> _T_Sentinel: - assert bases == (Sentinel,) - v = super().__new__(cls, name, bases, namespace, **kwds) - v.__class__ = v # type: ignore - return v - - def __repr__(self) -> str: - return self.__name__ - - -# Used for methods, request targets, HTTP versions, header names, and header -# values. Accepts ascii-strings, or bytes/bytearray/memoryview/..., and always -# returns bytes. -def bytesify(s: Union[bytes, bytearray, memoryview, int, str]) -> bytes: - # Fast-path: - if type(s) is bytes: - return s - if isinstance(s, str): - s = s.encode("ascii") - if isinstance(s, int): - raise TypeError("expected bytes-like object, not int") - return bytes(s) diff --git a/contrib/python/h11/h11/_version.py b/contrib/python/h11/h11/_version.py deleted file mode 100644 index 4c891130568..00000000000 --- a/contrib/python/h11/h11/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file must be kept very simple, because it is consumed from several -# places -- it is imported by h11/__init__.py, execfile'd by setup.py, etc. - -# We use a simple scheme: -# 1.0.0 -> 1.0.0+dev -> 1.1.0 -> 1.1.0+dev -# where the +dev versions are never released into the wild, they're just what -# we stick into the VCS in between releases. -# -# This is compatible with PEP 440: -# http://legacy.python.org/dev/peps/pep-0440/ -# via the use of the "local suffix" "+dev", which is disallowed on index -# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we -# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before* -# 1.0.0.) - -__version__ = "0.14.0" diff --git a/contrib/python/h11/h11/_writers.py b/contrib/python/h11/h11/_writers.py deleted file mode 100644 index 939cdb912a9..00000000000 --- a/contrib/python/h11/h11/_writers.py +++ /dev/null @@ -1,145 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each writer takes an event + a write-some-bytes function, which is -# calls. -# -# WRITERS is a dict describing how to pick a reader. It maps states to either: -# - a writer -# - or, for body writers, a dict of framin-dependent writer factories - -from typing import Any, Callable, Dict, List, Tuple, Type, Union - -from ._events import Data, EndOfMessage, Event, InformationalResponse, Request, Response -from ._headers import Headers -from ._state import CLIENT, IDLE, SEND_BODY, SEND_RESPONSE, SERVER -from ._util import LocalProtocolError, Sentinel - -__all__ = ["WRITERS"] - -Writer = Callable[[bytes], Any] - - -def write_headers(headers: Headers, write: Writer) -> None: - # "Since the Host field-value is critical information for handling a - # request, a user agent SHOULD generate Host as the first header field - # following the request-line." - RFC 7230 - raw_items = headers._full_items - for raw_name, name, value in raw_items: - if name == b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - for raw_name, name, value in raw_items: - if name != b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - write(b"\r\n") - - -def write_request(request: Request, write: Writer) -> None: - if request.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - write(b"%s %s HTTP/1.1\r\n" % (request.method, request.target)) - write_headers(request.headers, write) - - -# Shared between InformationalResponse and Response -def write_any_response( - response: Union[InformationalResponse, Response], write: Writer -) -> None: - if response.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - status_bytes = str(response.status_code).encode("ascii") - # We don't bother sending ascii status messages like "OK"; they're - # optional and ignored by the protocol. (But the space after the numeric - # status code is mandatory.) - # - # XX FIXME: could at least make an effort to pull out the status message - # from stdlib's http.HTTPStatus table. Or maybe just steal their enums - # (either by import or copy/paste). We already accept them as status codes - # since they're of type IntEnum < int. - write(b"HTTP/1.1 %s %s\r\n" % (status_bytes, response.reason)) - write_headers(response.headers, write) - - -class BodyWriter: - def __call__(self, event: Event, write: Writer) -> None: - if type(event) is Data: - self.send_data(event.data, write) - elif type(event) is EndOfMessage: - self.send_eom(event.headers, write) - else: # pragma: no cover - assert False - - def send_data(self, data: bytes, write: Writer) -> None: - pass - - def send_eom(self, headers: Headers, write: Writer) -> None: - pass - - -# -# These are all careful not to do anything to 'data' except call len(data) and -# write(data). This allows us to transparently pass-through funny objects, -# like placeholder objects referring to files on disk that will be sent via -# sendfile(2). -# -class ContentLengthWriter(BodyWriter): - def __init__(self, length: int) -> None: - self._length = length - - def send_data(self, data: bytes, write: Writer) -> None: - self._length -= len(data) - if self._length < 0: - raise LocalProtocolError("Too much data for declared Content-Length") - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if self._length != 0: - raise LocalProtocolError("Too little data for declared Content-Length") - if headers: - raise LocalProtocolError("Content-Length and trailers don't mix") - - -class ChunkedWriter(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - # if we encoded 0-length data in the naive way, it would look like an - # end-of-message. - if not data: - return - write(b"%x\r\n" % len(data)) - write(data) - write(b"\r\n") - - def send_eom(self, headers: Headers, write: Writer) -> None: - write(b"0\r\n") - write_headers(headers, write) - - -class Http10Writer(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if headers: - raise LocalProtocolError("can't send trailers to HTTP/1.0 client") - # no need to close the socket ourselves, that will be taken care of by - # Connection: close machinery - - -WritersType = Dict[ - Union[Tuple[Type[Sentinel], Type[Sentinel]], Type[Sentinel]], - Union[ - Dict[str, Type[BodyWriter]], - Callable[[Union[InformationalResponse, Response], Writer], None], - Callable[[Request, Writer], None], - ], -] - -WRITERS: WritersType = { - (CLIENT, IDLE): write_request, - (SERVER, IDLE): write_any_response, - (SERVER, SEND_RESPONSE): write_any_response, - SEND_BODY: { - "chunked": ChunkedWriter, - "content-length": ContentLengthWriter, - "http/1.0": Http10Writer, - }, -} diff --git a/contrib/python/h11/h11/py.typed b/contrib/python/h11/h11/py.typed deleted file mode 100644 index f5642f79f21..00000000000 --- a/contrib/python/h11/h11/py.typed +++ /dev/null @@ -1 +0,0 @@ -Marker diff --git a/contrib/python/h11/ya.make b/contrib/python/h11/ya.make deleted file mode 100644 index 48fcc1a6541..00000000000 --- a/contrib/python/h11/ya.make +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(0.14.0) - -LICENSE(MIT) - -NO_LINT() - -PY_SRCS( - TOP_LEVEL - h11/__init__.py - h11/_abnf.py - h11/_connection.py - h11/_events.py - h11/_headers.py - h11/_readers.py - h11/_receivebuffer.py - h11/_state.py - h11/_util.py - h11/_version.py - h11/_writers.py -) - -RESOURCE_FILES( - PREFIX contrib/python/h11/ - .dist-info/METADATA - .dist-info/top_level.txt - h11/py.typed -) - -END() diff --git a/contrib/python/httpcore/.dist-info/METADATA b/contrib/python/httpcore/.dist-info/METADATA deleted file mode 100644 index 99be2236cdd..00000000000 --- a/contrib/python/httpcore/.dist-info/METADATA +++ /dev/null @@ -1,616 +0,0 @@ -Metadata-Version: 2.3 -Name: httpcore -Version: 1.0.7 -Summary: A minimal low-level HTTP client. -Project-URL: Documentation, https://www.encode.io/httpcore -Project-URL: Homepage, https://www.encode.io/httpcore/ -Project-URL: Source, https://github.com/encode/httpcore -Author-email: Tom Christie <[email protected]> -License: BSD-3-Clause -Classifier: Development Status :: 3 - Alpha -Classifier: Environment :: Web Environment -Classifier: Framework :: AsyncIO -Classifier: Framework :: Trio -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Internet :: WWW/HTTP -Requires-Python: >=3.8 -Requires-Dist: certifi -Requires-Dist: h11<0.15,>=0.13 -Provides-Extra: asyncio -Requires-Dist: anyio<5.0,>=4.0; extra == 'asyncio' -Provides-Extra: http2 -Requires-Dist: h2<5,>=3; extra == 'http2' -Provides-Extra: socks -Requires-Dist: socksio==1.*; extra == 'socks' -Provides-Extra: trio -Requires-Dist: trio<1.0,>=0.22.0; extra == 'trio' -Description-Content-Type: text/markdown - -# HTTP Core - -[](https://github.com/encode/httpcore/actions) -[](https://pypi.org/project/httpcore/) - -> *Do one thing, and do it well.* - -The HTTP Core package provides a minimal low-level HTTP client, which does -one thing only. Sending HTTP requests. - -It does not provide any high level model abstractions over the API, -does not handle redirects, multipart uploads, building authentication headers, -transparent HTTP caching, URL parsing, session cookie handling, -content or charset decoding, handling JSON, environment based configuration -defaults, or any of that Jazz. - -Some things HTTP Core does do: - -* Sending HTTP requests. -* Thread-safe / task-safe connection pooling. -* HTTP(S) proxy & SOCKS proxy support. -* Supports HTTP/1.1 and HTTP/2. -* Provides both sync and async interfaces. -* Async backend support for `asyncio` and `trio`. - -## Requirements - -Python 3.8+ - -## Installation - -For HTTP/1.1 only support, install with: - -```shell -$ pip install httpcore -``` - -There are also a number of optional extras available... - -```shell -$ pip install httpcore['asyncio,trio,http2,socks'] -``` - -## Sending requests - -Send an HTTP request: - -```python -import httpcore - -response = httpcore.request("GET", "https://www.example.com/") - -print(response) -# <Response [200]> -print(response.status) -# 200 -print(response.headers) -# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] -print(response.content) -# b'<!doctype html>\n<html>\n<head>\n<title>Example Domain</title>\n\n<meta charset="utf-8"/>\n ...' -``` - -The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. - -```python -import httpcore - -http = httpcore.ConnectionPool() -response = http.request("GET", "https://www.example.com/") -``` - -Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). - -## Motivation - -You *probably* don't want to be using HTTP Core directly. It might make sense if -you're writing something like a proxy service in Python, and you just want -something at the lowest possible level, but more typically you'll want to use -a higher level client library, such as `httpx`. - -The motivation for `httpcore` is: - -* To provide a reusable low-level client library, that other packages can then build on top of. -* To provide a *really clear interface split* between the networking code and client logic, - so that each is easier to understand and reason about in isolation. - -## Dependencies - -The `httpcore` package has the following dependencies... - -* `h11` -* `certifi` - -And the following optional extras... - -* `anyio` - Required by `pip install httpcore['asyncio']`. -* `trio` - Required by `pip install httpcore['trio']`. -* `h2` - Required by `pip install httpcore['http2']`. -* `socksio` - Required by `pip install httpcore['socks']`. - -## Versioning - -We use [SEMVER for our versioning policy](https://semver.org/). - -For changes between package versions please see our [project changelog](CHANGELOG.md). - -We recommend pinning your requirements either the most current major version, or a more specific version range: - -```python -pip install 'httpcore==1.*' -``` -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - -## Version 1.0.7 (November 15th, 2024) - -- Support `proxy=…` configuration on `ConnectionPool()`. (#974) - -## Version 1.0.6 (October 1st, 2024) - -- Relax `trio` dependency pinning. (#956) -- Handle `trio` raising `NotImplementedError` on unsupported platforms. (#955) -- Handle mapping `ssl.SSLError` to `httpcore.ConnectError`. (#918) - -## 1.0.5 (March 27th, 2024) - -- Handle `EndOfStream` exception for anyio backend. (#899) -- Allow trio `0.25.*` series in package dependancies. (#903) - -## 1.0.4 (February 21st, 2024) - -- Add `target` request extension. (#888) -- Fix support for connection `Upgrade` and `CONNECT` when some data in the stream has been read. (#882) - -## 1.0.3 (February 13th, 2024) - -- Fix support for async cancellations. (#880) -- Fix trace extension when used with socks proxy. (#849) -- Fix SSL context for connections using the "wss" scheme (#869) - -## 1.0.2 (November 10th, 2023) - -- Fix `float("inf")` timeouts in `Event.wait` function. (#846) - -## 1.0.1 (November 3rd, 2023) - -- Fix pool timeout to account for the total time spent retrying. (#823) -- Raise a neater RuntimeError when the correct async deps are not installed. (#826) -- Add support for synchronous TLS-in-TLS streams. (#840) - -## 1.0.0 (October 6th, 2023) - -From version 1.0 our async support is now optional, as the package has minimal dependencies by default. - -For async support use either `pip install 'httpcore[asyncio]'` or `pip install 'httpcore[trio]'`. - -The project versioning policy is now explicitly governed by SEMVER. See https://semver.org/. - -- Async support becomes fully optional. (#809) -- Add support for Python 3.12. (#807) - -## 0.18.0 (September 8th, 2023) - -- Add support for HTTPS proxies. (#745, #786) -- Drop Python 3.7 support. (#727) -- Handle `sni_hostname` extension with SOCKS proxy. (#774) -- Handle HTTP/1.1 half-closed connections gracefully. (#641) -- Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) - -## 0.17.3 (July 5th, 2023) - -- Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) -- The networking backend interface has [been added to the public API](https://www.encode.io/httpcore/network-backends). Some classes which were previously private implementation detail are now part of the top-level public API. (#699) -- Graceful handling of HTTP/2 GoAway frames, with requests being transparently retried on a new connection. (#730) -- Add exceptions when a synchronous `trace callback` is passed to an asynchronous request or an asynchronous `trace callback` is passed to a synchronous request. (#717) -- Drop Python 3.7 support. (#727) - -## 0.17.2 (May 23th, 2023) - -- Add `socket_options` argument to `ConnectionPool` and `HTTProxy` classes. (#668) -- Improve logging with per-module logger names. (#690) -- Add `sni_hostname` request extension. (#696) -- Resolve race condition during import of `anyio` package. (#692) -- Enable TCP_NODELAY for all synchronous sockets. (#651) - -## 0.17.1 (May 17th, 2023) - -- If 'retries' is set, then allow retries if an SSL handshake error occurs. (#669) -- Improve correctness of tracebacks on network exceptions, by raising properly chained exceptions. (#678) -- Prevent connection-hanging behaviour when HTTP/2 connections are closed by a server-sent 'GoAway' frame. (#679) -- Fix edge-case exception when removing requests from the connection pool. (#680) -- Fix pool timeout edge-case. (#688) - -## 0.17.0 (March 16th, 2023) - -- Add DEBUG level logging. (#648) -- Respect HTTP/2 max concurrent streams when settings updates are sent by server. (#652) -- Increase the allowable HTTP header size to 100kB. (#647) -- Add `retries` option to SOCKS proxy classes. (#643) - -## 0.16.3 (December 20th, 2022) - -- Allow `ws` and `wss` schemes. Allows us to properly support websocket upgrade connections. (#625) -- Forwarding HTTP proxies use a connection-per-remote-host. Required by some proxy implementations. (#637) -- Don't raise `RuntimeError` when closing a connection pool with active connections. Removes some error cases when cancellations are used. (#631) -- Lazy import `anyio`, so that it's no longer a hard dependancy, and isn't imported if unused. (#639) - -## 0.16.2 (November 25th, 2022) - -- Revert 'Fix async cancellation behaviour', which introduced race conditions. (#627) -- Raise `RuntimeError` if attempting to us UNIX domain sockets on Windows. (#619) - -## 0.16.1 (November 17th, 2022) - -- Fix HTTP/1.1 interim informational responses, such as "100 Continue". (#605) - -## 0.16.0 (October 11th, 2022) - -- Support HTTP/1.1 informational responses. (#581) -- Fix async cancellation behaviour. (#580) -- Support `h11` 0.14. (#579) - -## 0.15.0 (May 17th, 2022) - -- Drop Python 3.6 support (#535) -- Ensure HTTP proxy CONNECT requests include `timeout` configuration. (#506) -- Switch to explicit `typing.Optional` for type hints. (#513) -- For `trio` map OSError exceptions to `ConnectError`. (#543) - -## 0.14.7 (February 4th, 2022) - -- Requests which raise a PoolTimeout need to be removed from the pool queue. (#502) -- Fix AttributeError that happened when Socks5Connection were terminated. (#501) - -## 0.14.6 (February 1st, 2022) - -- Fix SOCKS support for `http://` URLs. (#492) -- Resolve race condition around exceptions during streaming a response. (#491) - -## 0.14.5 (January 18th, 2022) - -- SOCKS proxy support. (#478) -- Add proxy_auth argument to HTTPProxy. (#481) -- Improve error message on 'RemoteProtocolError' exception when server disconnects without sending a response. (#479) - -## 0.14.4 (January 5th, 2022) - -- Support HTTP/2 on HTTPS tunnelling proxies. (#468) -- Fix proxy headers missing on HTTP forwarding. (#456) -- Only instantiate SSL context if required. (#457) -- More robust HTTP/2 handling. (#253, #439, #440, #441) - -## 0.14.3 (November 17th, 2021) - -- Fix race condition when removing closed connections from the pool. (#437) - -## 0.14.2 (November 16th, 2021) - -- Failed connections no longer remain in the pool. (Pull #433) - -## 0.14.1 (November 12th, 2021) - -- `max_connections` becomes optional. (Pull #429) -- `certifi` is now included in the install dependancies. (Pull #428) -- `h2` is now strictly optional. (Pull #428) - -## 0.14.0 (November 11th, 2021) - -The 0.14 release is a complete reworking of `httpcore`, comprehensively addressing some underlying issues in the connection pooling, as well as substantially redesigning the API to be more user friendly. - -Some of the lower-level API design also makes the components more easily testable in isolation, and the package now has 100% test coverage. - -See [discussion #419](https://github.com/encode/httpcore/discussions/419) for a little more background. - -There's some other neat bits in there too, such as the "trace" extension, which gives a hook into inspecting the internal events that occur during the request/response cycle. This extension is needed for the HTTPX cli, in order to... - -* Log the point at which the connection is established, and the IP/port on which it is made. -* Determine if the outgoing request should log as HTTP/1.1 or HTTP/2, rather than having to assume it's HTTP/2 if the --http2 flag was passed. (Which may not actually be true.) -* Log SSL version info / certificate info. - -Note that `curio` support is not currently available in 0.14.0. If you're using `httpcore` with `curio` please get in touch, so we can assess if we ought to prioritize it as a feature or not. - -## 0.13.7 (September 13th, 2021) - -- Fix broken error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #403) - -## 0.13.6 (June 15th, 2021) - -### Fixed - -- Close sockets when read or write timeouts occur. (Pull #365) - -## 0.13.5 (June 14th, 2021) - -### Fixed - -- Resolved niggles with AnyIO EOF behaviours. (Pull #358, #362) - -## 0.13.4 (June 9th, 2021) - -### Added - -- Improved error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #354) - -### Fixed - -- Switched to `anyio` as the default backend implementation when running with `asyncio`. Resolves some awkward [TLS timeout issues](https://github.com/encode/httpx/discussions/1511). - -## 0.13.3 (May 6th, 2021) - -### Added - -- Support HTTP/2 prior knowledge, using `httpcore.SyncConnectionPool(http1=False)`. (Pull #333) - -### Fixed - -- Handle cases where environment does not provide `select.poll` support. (Pull #331) - -## 0.13.2 (April 29th, 2021) - -### Added - -- Improve error message for specific case of `RemoteProtocolError` where server disconnects without sending a response. (Pull #313) - -## 0.13.1 (April 28th, 2021) - -### Fixed - -- More resiliant testing for closed connections. (Pull #311) -- Don't raise exceptions on ungraceful connection closes. (Pull #310) - -## 0.13.0 (April 21st, 2021) - -The 0.13 release updates the core API in order to match the HTTPX Transport API, -introduced in HTTPX 0.18 onwards. - -An example of making requests with the new interface is: - -```python -with httpcore.SyncConnectionPool() as http: - status_code, headers, stream, extensions = http.handle_request( - method=b'GET', - url=(b'https', b'example.org', 443, b'/'), - headers=[(b'host', b'example.org'), (b'user-agent', b'httpcore')] - stream=httpcore.ByteStream(b''), - extensions={} - ) - body = stream.read() - print(status_code, body) -``` - -### Changed - -- The `.request()` method is now `handle_request()`. (Pull #296) -- The `.arequest()` method is now `.handle_async_request()`. (Pull #296) -- The `headers` argument is no longer optional. (Pull #296) -- The `stream` argument is no longer optional. (Pull #296) -- The `ext` argument is now named `extensions`, and is no longer optional. (Pull #296) -- The `"reason"` extension keyword is now named `"reason_phrase"`. (Pull #296) -- The `"reason_phrase"` and `"http_version"` extensions now use byte strings for their values. (Pull #296) -- The `httpcore.PlainByteStream()` class becomes `httpcore.ByteStream()`. (Pull #296) - -### Added - -- Streams now support a `.read()` interface. (Pull #296) - -### Fixed - -- Task cancellation no longer leaks connections from the connection pool. (Pull #305) - -## 0.12.3 (December 7th, 2020) - -### Fixed - -- Abort SSL connections on close rather than waiting for remote EOF when using `asyncio`. (Pull #167) -- Fix exception raised in case of connect timeouts when using the `anyio` backend. (Pull #236) -- Fix `Host` header precedence for `:authority` in HTTP/2. (Pull #241, #243) -- Handle extra edge case when detecting for socket readability when using `asyncio`. (Pull #242, #244) -- Fix `asyncio` SSL warning when using proxy tunneling. (Pull #249) - -## 0.12.2 (November 20th, 2020) - -### Fixed - -- Properly wrap connect errors on the asyncio backend. (Pull #235) -- Fix `ImportError` occurring on Python 3.9 when using the HTTP/1.1 sync client in a multithreaded context. (Pull #237) - -## 0.12.1 (November 7th, 2020) - -### Added - -- Add connect retries. (Pull #221) - -### Fixed - -- Tweak detection of dropped connections, resolving an issue with open files limits on Linux. (Pull #185) -- Avoid leaking connections when establishing an HTTP tunnel to a proxy has failed. (Pull #223) -- Properly wrap OS errors when using `trio`. (Pull #225) - -## 0.12.0 (October 6th, 2020) - -### Changed - -- HTTP header casing is now preserved, rather than always sent in lowercase. (#216 and python-hyper/h11#104) - -### Added - -- Add Python 3.9 to officially supported versions. - -### Fixed - -- Gracefully handle a stdlib asyncio bug when a connection is closed while it is in a paused-for-reading state. (#201) - -## 0.11.1 (September 28nd, 2020) - -### Fixed - -- Add await to async semaphore release() coroutine (#197) -- Drop incorrect curio classifier (#192) - -## 0.11.0 (September 22nd, 2020) - -The Transport API with 0.11.0 has a couple of significant changes. - -Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features -such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. - -The interface changes from: - -```python -def request(method, url, headers, stream, timeout): - return (http_version, status_code, reason, headers, stream) -``` - -To instead including an optional dictionary of extensions on the request and response: - -```python -def request(method, url, headers, stream, ext): - return (status_code, headers, stream, ext) -``` - -Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. - -In particular: - -* Trailing headers support. -* HTTP/2 Server Push -* sendfile. -* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. -* Exposing debug information out of the API, including template name, template context. - -Currently extensions are limited to: - -* request: `timeout` - Optional. Timeout dictionary. -* response: `http_version` - Optional. Include the HTTP version used on the response. -* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. - -See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. - -Secondly, the async version of `request` is now namespaced as `arequest`. - -This allows concrete transports to support both sync and async implementations on the same class. - -### Added - -- Add curio support. (Pull #168) -- Add anyio support, with `backend="anyio"`. (Pull #169) - -### Changed - -- Update the Transport API to use 'ext' for optional extensions. (Pull #190) -- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) - -## 0.10.2 (August 20th, 2020) - -### Added - -- Added Unix Domain Socket support. (Pull #139) - -### Fixed - -- Always include the port on proxy CONNECT requests. (Pull #154) -- Fix `max_keepalive_connections` configuration. (Pull #153) -- Fixes behaviour in HTTP/1.1 where server disconnects can be used to signal the end of the response body. (Pull #164) - -## 0.10.1 (August 7th, 2020) - -- Include `max_keepalive_connections` on `AsyncHTTPProxy`/`SyncHTTPProxy` classes. - -## 0.10.0 (August 7th, 2020) - -The most notable change in the 0.10.0 release is that HTTP/2 support is now fully optional. - -Use either `pip install httpcore` for HTTP/1.1 support only, or `pip install httpcore[http2]` for HTTP/1.1 and HTTP/2 support. - -### Added - -- HTTP/2 support becomes optional. (Pull #121, #130) -- Add `local_address=...` support. (Pull #100, #134) -- Add `PlainByteStream`, `IteratorByteStream`, `AsyncIteratorByteStream`. The `AsyncByteSteam` and `SyncByteStream` classes are now pure interface classes. (#133) -- Add `LocalProtocolError`, `RemoteProtocolError` exceptions. (Pull #129) -- Add `UnsupportedProtocol` exception. (Pull #128) -- Add `.get_connection_info()` method. (Pull #102, #137) -- Add better TRACE logs. (Pull #101) - -### Changed - -- `max_keepalive` is deprecated in favour of `max_keepalive_connections`. (Pull #140) - -### Fixed - -- Improve handling of server disconnects. (Pull #112) - -## 0.9.1 (May 27th, 2020) - -### Fixed - -- Proper host resolution for sync case, including IPv6 support. (Pull #97) -- Close outstanding connections when connection pool is closed. (Pull #98) - -## 0.9.0 (May 21th, 2020) - -### Changed - -- URL port becomes an `Optional[int]` instead of `int`. (Pull #92) - -### Fixed - -- Honor HTTP/2 max concurrent streams settings. (Pull #89, #90) -- Remove incorrect debug log. (Pull #83) - -## 0.8.4 (May 11th, 2020) - -### Added - -- Logging via HTTPCORE_LOG_LEVEL and HTTPX_LOG_LEVEL environment variables -and TRACE level logging. (Pull #79) - -### Fixed - -- Reuse of connections on HTTP/2 in close concurrency situations. (Pull #81) - -## 0.8.3 (May 6rd, 2020) - -### Fixed - -- Include `Host` and `Accept` headers on proxy "CONNECT" requests. -- De-duplicate any headers also contained in proxy_headers. -- HTTP/2 flag not being passed down to proxy connections. - -## 0.8.2 (May 3rd, 2020) - -### Fixed - -- Fix connections using proxy forwarding requests not being added to the -connection pool properly. (Pull #70) - -## 0.8.1 (April 30th, 2020) - -### Changed - -- Allow inherintance of both `httpcore.AsyncByteStream`, `httpcore.SyncByteStream` without type conflicts. - -## 0.8.0 (April 30th, 2020) - -### Fixed - -- Fixed tunnel proxy support. - -### Added - -- New `TimeoutException` base class. - -## 0.7.0 (March 5th, 2020) - -- First integration with HTTPX. diff --git a/contrib/python/httpcore/.dist-info/top_level.txt b/contrib/python/httpcore/.dist-info/top_level.txt deleted file mode 100644 index 613e43507bb..00000000000 --- a/contrib/python/httpcore/.dist-info/top_level.txt +++ /dev/null @@ -1,4 +0,0 @@ -httpcore -httpcore/_async -httpcore/_backends -httpcore/_sync diff --git a/contrib/python/httpcore/LICENSE.md b/contrib/python/httpcore/LICENSE.md deleted file mode 100644 index 311b2b56c53..00000000000 --- a/contrib/python/httpcore/LICENSE.md +++ /dev/null @@ -1,27 +0,0 @@ -Copyright © 2020, [Encode OSS Ltd](https://www.encode.io/). -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/httpcore/README.md b/contrib/python/httpcore/README.md deleted file mode 100644 index 4567ba44b40..00000000000 --- a/contrib/python/httpcore/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# HTTP Core - -[](https://github.com/encode/httpcore/actions) -[](https://pypi.org/project/httpcore/) - -> *Do one thing, and do it well.* - -The HTTP Core package provides a minimal low-level HTTP client, which does -one thing only. Sending HTTP requests. - -It does not provide any high level model abstractions over the API, -does not handle redirects, multipart uploads, building authentication headers, -transparent HTTP caching, URL parsing, session cookie handling, -content or charset decoding, handling JSON, environment based configuration -defaults, or any of that Jazz. - -Some things HTTP Core does do: - -* Sending HTTP requests. -* Thread-safe / task-safe connection pooling. -* HTTP(S) proxy & SOCKS proxy support. -* Supports HTTP/1.1 and HTTP/2. -* Provides both sync and async interfaces. -* Async backend support for `asyncio` and `trio`. - -## Requirements - -Python 3.8+ - -## Installation - -For HTTP/1.1 only support, install with: - -```shell -$ pip install httpcore -``` - -There are also a number of optional extras available... - -```shell -$ pip install httpcore['asyncio,trio,http2,socks'] -``` - -## Sending requests - -Send an HTTP request: - -```python -import httpcore - -response = httpcore.request("GET", "https://www.example.com/") - -print(response) -# <Response [200]> -print(response.status) -# 200 -print(response.headers) -# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] -print(response.content) -# b'<!doctype html>\n<html>\n<head>\n<title>Example Domain</title>\n\n<meta charset="utf-8"/>\n ...' -``` - -The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. - -```python -import httpcore - -http = httpcore.ConnectionPool() -response = http.request("GET", "https://www.example.com/") -``` - -Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). - -## Motivation - -You *probably* don't want to be using HTTP Core directly. It might make sense if -you're writing something like a proxy service in Python, and you just want -something at the lowest possible level, but more typically you'll want to use -a higher level client library, such as `httpx`. - -The motivation for `httpcore` is: - -* To provide a reusable low-level client library, that other packages can then build on top of. -* To provide a *really clear interface split* between the networking code and client logic, - so that each is easier to understand and reason about in isolation. - -## Dependencies - -The `httpcore` package has the following dependencies... - -* `h11` -* `certifi` - -And the following optional extras... - -* `anyio` - Required by `pip install httpcore['asyncio']`. -* `trio` - Required by `pip install httpcore['trio']`. -* `h2` - Required by `pip install httpcore['http2']`. -* `socksio` - Required by `pip install httpcore['socks']`. - -## Versioning - -We use [SEMVER for our versioning policy](https://semver.org/). - -For changes between package versions please see our [project changelog](CHANGELOG.md). - -We recommend pinning your requirements either the most current major version, or a more specific version range: - -```python -pip install 'httpcore==1.*' -``` diff --git a/contrib/python/httpcore/httpcore/__init__.py b/contrib/python/httpcore/httpcore/__init__.py deleted file mode 100644 index 662b1563a1e..00000000000 --- a/contrib/python/httpcore/httpcore/__init__.py +++ /dev/null @@ -1,140 +0,0 @@ -from ._api import request, stream -from ._async import ( - AsyncConnectionInterface, - AsyncConnectionPool, - AsyncHTTP2Connection, - AsyncHTTP11Connection, - AsyncHTTPConnection, - AsyncHTTPProxy, - AsyncSOCKSProxy, -) -from ._backends.base import ( - SOCKET_OPTION, - AsyncNetworkBackend, - AsyncNetworkStream, - NetworkBackend, - NetworkStream, -) -from ._backends.mock import AsyncMockBackend, AsyncMockStream, MockBackend, MockStream -from ._backends.sync import SyncBackend -from ._exceptions import ( - ConnectError, - ConnectionNotAvailable, - ConnectTimeout, - LocalProtocolError, - NetworkError, - PoolTimeout, - ProtocolError, - ProxyError, - ReadError, - ReadTimeout, - RemoteProtocolError, - TimeoutException, - UnsupportedProtocol, - WriteError, - WriteTimeout, -) -from ._models import URL, Origin, Proxy, Request, Response -from ._ssl import default_ssl_context -from ._sync import ( - ConnectionInterface, - ConnectionPool, - HTTP2Connection, - HTTP11Connection, - HTTPConnection, - HTTPProxy, - SOCKSProxy, -) - -# The 'httpcore.AnyIOBackend' class is conditional on 'anyio' being installed. -try: - from ._backends.anyio import AnyIOBackend -except ImportError: # pragma: nocover - - class AnyIOBackend: # type: ignore - def __init__(self, *args, **kwargs): # type: ignore - msg = ( - "Attempted to use 'httpcore.AnyIOBackend' but 'anyio' is not installed." - ) - raise RuntimeError(msg) - - -# The 'httpcore.TrioBackend' class is conditional on 'trio' being installed. -try: - from ._backends.trio import TrioBackend -except ImportError: # pragma: nocover - - class TrioBackend: # type: ignore - def __init__(self, *args, **kwargs): # type: ignore - msg = "Attempted to use 'httpcore.TrioBackend' but 'trio' is not installed." - raise RuntimeError(msg) - - -__all__ = [ - # top-level requests - "request", - "stream", - # models - "Origin", - "URL", - "Request", - "Response", - "Proxy", - # async - "AsyncHTTPConnection", - "AsyncConnectionPool", - "AsyncHTTPProxy", - "AsyncHTTP11Connection", - "AsyncHTTP2Connection", - "AsyncConnectionInterface", - "AsyncSOCKSProxy", - # sync - "HTTPConnection", - "ConnectionPool", - "HTTPProxy", - "HTTP11Connection", - "HTTP2Connection", - "ConnectionInterface", - "SOCKSProxy", - # network backends, implementations - "SyncBackend", - "AnyIOBackend", - "TrioBackend", - # network backends, mock implementations - "AsyncMockBackend", - "AsyncMockStream", - "MockBackend", - "MockStream", - # network backends, interface - "AsyncNetworkStream", - "AsyncNetworkBackend", - "NetworkStream", - "NetworkBackend", - # util - "default_ssl_context", - "SOCKET_OPTION", - # exceptions - "ConnectionNotAvailable", - "ProxyError", - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", - "UnsupportedProtocol", - "TimeoutException", - "PoolTimeout", - "ConnectTimeout", - "ReadTimeout", - "WriteTimeout", - "NetworkError", - "ConnectError", - "ReadError", - "WriteError", -] - -__version__ = "1.0.7" - - -__locals = locals() -for __name in __all__: - if not __name.startswith("__"): - setattr(__locals[__name], "__module__", "httpcore") # noqa diff --git a/contrib/python/httpcore/httpcore/_api.py b/contrib/python/httpcore/httpcore/_api.py deleted file mode 100644 index 38b961d10de..00000000000 --- a/contrib/python/httpcore/httpcore/_api.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import contextlib -import typing - -from ._models import URL, Extensions, HeaderTypes, Response -from ._sync.connection_pool import ConnectionPool - - -def request( - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.Iterator[bytes] | None = None, - extensions: Extensions | None = None, -) -> Response: - """ - Sends an HTTP request, returning the response. - - ``` - response = httpcore.request("GET", "https://www.example.com/") - ``` - - Arguments: - method: The HTTP method for the request. Typically one of `"GET"`, - `"OPTIONS"`, `"HEAD"`, `"POST"`, `"PUT"`, `"PATCH"`, or `"DELETE"`. - url: The URL of the HTTP request. Either as an instance of `httpcore.URL`, - or as str/bytes. - headers: The HTTP request headers. Either as a dictionary of str/bytes, - or as a list of two-tuples of str/bytes. - content: The content of the request body. Either as bytes, - or as a bytes iterator. - extensions: A dictionary of optional extra information included on the request. - Possible keys include `"timeout"`. - - Returns: - An instance of `httpcore.Response`. - """ - with ConnectionPool() as pool: - return pool.request( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) - - -def stream( - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.Iterator[bytes] | None = None, - extensions: Extensions | None = None, -) -> typing.Iterator[Response]: - """ - Sends an HTTP request, returning the response within a content manager. - - ``` - with httpcore.stream("GET", "https://www.example.com/") as response: - ... - ``` - - When using the `stream()` function, the body of the response will not be - automatically read. If you want to access the response body you should - either use `content = response.read()`, or `for chunk in response.iter_content()`. - - Arguments: - method: The HTTP method for the request. Typically one of `"GET"`, - `"OPTIONS"`, `"HEAD"`, `"POST"`, `"PUT"`, `"PATCH"`, or `"DELETE"`. - url: The URL of the HTTP request. Either as an instance of `httpcore.URL`, - or as str/bytes. - headers: The HTTP request headers. Either as a dictionary of str/bytes, - or as a list of two-tuples of str/bytes. - content: The content of the request body. Either as bytes, - or as a bytes iterator. - extensions: A dictionary of optional extra information included on the request. - Possible keys include `"timeout"`. - - Returns: - An instance of `httpcore.Response`. - """ - with ConnectionPool() as pool: - with pool.stream( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) as response: - yield response diff --git a/contrib/python/httpcore/httpcore/_async/__init__.py b/contrib/python/httpcore/httpcore/_async/__init__.py deleted file mode 100644 index 88dc7f01e13..00000000000 --- a/contrib/python/httpcore/httpcore/_async/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from .connection import AsyncHTTPConnection -from .connection_pool import AsyncConnectionPool -from .http11 import AsyncHTTP11Connection -from .http_proxy import AsyncHTTPProxy -from .interfaces import AsyncConnectionInterface - -try: - from .http2 import AsyncHTTP2Connection -except ImportError: # pragma: nocover - - class AsyncHTTP2Connection: # type: ignore - def __init__(self, *args, **kwargs) -> None: # type: ignore - raise RuntimeError( - "Attempted to use http2 support, but the `h2` package is not " - "installed. Use 'pip install httpcore[http2]'." - ) - - -try: - from .socks_proxy import AsyncSOCKSProxy -except ImportError: # pragma: nocover - - class AsyncSOCKSProxy: # type: ignore - def __init__(self, *args, **kwargs) -> None: # type: ignore - raise RuntimeError( - "Attempted to use SOCKS support, but the `socksio` package is not " - "installed. Use 'pip install httpcore[socks]'." - ) - - -__all__ = [ - "AsyncHTTPConnection", - "AsyncConnectionPool", - "AsyncHTTPProxy", - "AsyncHTTP11Connection", - "AsyncHTTP2Connection", - "AsyncConnectionInterface", - "AsyncSOCKSProxy", -] diff --git a/contrib/python/httpcore/httpcore/_async/connection.py b/contrib/python/httpcore/httpcore/_async/connection.py deleted file mode 100644 index b42581dff8a..00000000000 --- a/contrib/python/httpcore/httpcore/_async/connection.py +++ /dev/null @@ -1,222 +0,0 @@ -from __future__ import annotations - -import itertools -import logging -import ssl -import types -import typing - -from .._backends.auto import AutoBackend -from .._backends.base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream -from .._exceptions import ConnectError, ConnectTimeout -from .._models import Origin, Request, Response -from .._ssl import default_ssl_context -from .._synchronization import AsyncLock -from .._trace import Trace -from .http11 import AsyncHTTP11Connection -from .interfaces import AsyncConnectionInterface - -RETRIES_BACKOFF_FACTOR = 0.5 # 0s, 0.5s, 1s, 2s, 4s, etc. - - -logger = logging.getLogger("httpcore.connection") - - -def exponential_backoff(factor: float) -> typing.Iterator[float]: - """ - Generate a geometric sequence that has a ratio of 2 and starts with 0. - - For example: - - `factor = 2`: `0, 2, 4, 8, 16, 32, 64, ...` - - `factor = 3`: `0, 3, 6, 12, 24, 48, 96, ...` - """ - yield 0 - for n in itertools.count(): - yield factor * 2**n - - -class AsyncHTTPConnection(AsyncConnectionInterface): - def __init__( - self, - origin: Origin, - ssl_context: ssl.SSLContext | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: AsyncNetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - self._origin = origin - self._ssl_context = ssl_context - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._retries = retries - self._local_address = local_address - self._uds = uds - - self._network_backend: AsyncNetworkBackend = ( - AutoBackend() if network_backend is None else network_backend - ) - self._connection: AsyncConnectionInterface | None = None - self._connect_failed: bool = False - self._request_lock = AsyncLock() - self._socket_options = socket_options - - async def handle_async_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection to {self._origin}" - ) - - try: - async with self._request_lock: - if self._connection is None: - stream = await self._connect(request) - - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - if http2_negotiated or (self._http2 and not self._http1): - from .http2 import AsyncHTTP2Connection - - self._connection = AsyncHTTP2Connection( - origin=self._origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = AsyncHTTP11Connection( - origin=self._origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - except BaseException as exc: - self._connect_failed = True - raise exc - - return await self._connection.handle_async_request(request) - - async def _connect(self, request: Request) -> AsyncNetworkStream: - timeouts = request.extensions.get("timeout", {}) - sni_hostname = request.extensions.get("sni_hostname", None) - timeout = timeouts.get("connect", None) - - retries_left = self._retries - delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR) - - while True: - try: - if self._uds is None: - kwargs = { - "host": self._origin.host.decode("ascii"), - "port": self._origin.port, - "local_address": self._local_address, - "timeout": timeout, - "socket_options": self._socket_options, - } - async with Trace("connect_tcp", logger, request, kwargs) as trace: - stream = await self._network_backend.connect_tcp(**kwargs) - trace.return_value = stream - else: - kwargs = { - "path": self._uds, - "timeout": timeout, - "socket_options": self._socket_options, - } - async with Trace( - "connect_unix_socket", logger, request, kwargs - ) as trace: - stream = await self._network_backend.connect_unix_socket( - **kwargs - ) - trace.return_value = stream - - if self._origin.scheme in (b"https", b"wss"): - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": sni_hostname - or self._origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - return stream - except (ConnectError, ConnectTimeout): - if retries_left <= 0: - raise - retries_left -= 1 - delay = next(delays) - async with Trace("retry", logger, request, kwargs) as trace: - await self._network_backend.sleep(delay) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - async def aclose(self) -> None: - if self._connection is not None: - async with Trace("close", logger, None, {}): - await self._connection.aclose() - - def is_available(self) -> bool: - if self._connection is None: - # If HTTP/2 support is enabled, and the resulting connection could - # end up as HTTP/2 then we should indicate the connection as being - # available to service multiple requests. - return ( - self._http2 - and (self._origin.scheme == b"https" or not self._http1) - and not self._connect_failed - ) - return self._connection.is_available() - - def has_expired(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.has_expired() - - def is_idle(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.is_idle() - - def is_closed(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.is_closed() - - def info(self) -> str: - if self._connection is None: - return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" - return self._connection.info() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - async def __aenter__(self) -> AsyncHTTPConnection: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - await self.aclose() diff --git a/contrib/python/httpcore/httpcore/_async/connection_pool.py b/contrib/python/httpcore/httpcore/_async/connection_pool.py deleted file mode 100644 index 96e973d0ce2..00000000000 --- a/contrib/python/httpcore/httpcore/_async/connection_pool.py +++ /dev/null @@ -1,420 +0,0 @@ -from __future__ import annotations - -import ssl -import sys -import types -import typing - -from .._backends.auto import AutoBackend -from .._backends.base import SOCKET_OPTION, AsyncNetworkBackend -from .._exceptions import ConnectionNotAvailable, UnsupportedProtocol -from .._models import Origin, Proxy, Request, Response -from .._synchronization import AsyncEvent, AsyncShieldCancellation, AsyncThreadLock -from .connection import AsyncHTTPConnection -from .interfaces import AsyncConnectionInterface, AsyncRequestInterface - - -class AsyncPoolRequest: - def __init__(self, request: Request) -> None: - self.request = request - self.connection: AsyncConnectionInterface | None = None - self._connection_acquired = AsyncEvent() - - def assign_to_connection(self, connection: AsyncConnectionInterface | None) -> None: - self.connection = connection - self._connection_acquired.set() - - def clear_connection(self) -> None: - self.connection = None - self._connection_acquired = AsyncEvent() - - async def wait_for_connection( - self, timeout: float | None = None - ) -> AsyncConnectionInterface: - if self.connection is None: - await self._connection_acquired.wait(timeout=timeout) - assert self.connection is not None - return self.connection - - def is_queued(self) -> bool: - return self.connection is None - - -class AsyncConnectionPool(AsyncRequestInterface): - """ - A connection pool for making HTTP requests. - """ - - def __init__( - self, - ssl_context: ssl.SSLContext | None = None, - proxy: Proxy | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: AsyncNetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish a - connection. - local_address: Local address to connect from. Can also be used to connect - using a particular address family. Using `local_address="0.0.0.0"` - will connect using an `AF_INET` address (IPv4), while using - `local_address="::"` will connect using an `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - socket_options: Socket options that have to be included - in the TCP socket when the connection was established. - """ - self._ssl_context = ssl_context - self._proxy = proxy - self._max_connections = ( - sys.maxsize if max_connections is None else max_connections - ) - self._max_keepalive_connections = ( - sys.maxsize - if max_keepalive_connections is None - else max_keepalive_connections - ) - self._max_keepalive_connections = min( - self._max_connections, self._max_keepalive_connections - ) - - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._retries = retries - self._local_address = local_address - self._uds = uds - - self._network_backend = ( - AutoBackend() if network_backend is None else network_backend - ) - self._socket_options = socket_options - - # The mutable state on a connection pool is the queue of incoming requests, - # and the set of connections that are servicing those requests. - self._connections: list[AsyncConnectionInterface] = [] - self._requests: list[AsyncPoolRequest] = [] - - # We only mutate the state of the connection pool within an 'optional_thread_lock' - # context. This holds a threading lock unless we're running in async mode, - # in which case it is a no-op. - self._optional_thread_lock = AsyncThreadLock() - - def create_connection(self, origin: Origin) -> AsyncConnectionInterface: - if self._proxy is not None: - if self._proxy.url.scheme in (b"socks5", b"socks5h"): - from .socks_proxy import AsyncSocks5Connection - - return AsyncSocks5Connection( - proxy_origin=self._proxy.url.origin, - proxy_auth=self._proxy.auth, - remote_origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - elif origin.scheme == b"http": - from .http_proxy import AsyncForwardHTTPConnection - - return AsyncForwardHTTPConnection( - proxy_origin=self._proxy.url.origin, - proxy_headers=self._proxy.headers, - proxy_ssl_context=self._proxy.ssl_context, - remote_origin=origin, - keepalive_expiry=self._keepalive_expiry, - network_backend=self._network_backend, - ) - from .http_proxy import AsyncTunnelHTTPConnection - - return AsyncTunnelHTTPConnection( - proxy_origin=self._proxy.url.origin, - proxy_headers=self._proxy.headers, - proxy_ssl_context=self._proxy.ssl_context, - remote_origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - return AsyncHTTPConnection( - origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - retries=self._retries, - local_address=self._local_address, - uds=self._uds, - network_backend=self._network_backend, - socket_options=self._socket_options, - ) - - @property - def connections(self) -> list[AsyncConnectionInterface]: - """ - Return a list of the connections currently in the pool. - - For example: - - ```python - >>> pool.connections - [ - <AsyncHTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 6]>, - <AsyncHTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 9]> , - <AsyncHTTPConnection ['http://example.com:80', HTTP/1.1, IDLE, Request Count: 1]>, - ] - ``` - """ - return list(self._connections) - - async def handle_async_request(self, request: Request) -> Response: - """ - Send an HTTP request, and return an HTTP response. - - This is the core implementation that is called into by `.request()` or `.stream()`. - """ - scheme = request.url.scheme.decode() - if scheme == "": - raise UnsupportedProtocol( - "Request URL is missing an 'http://' or 'https://' protocol." - ) - if scheme not in ("http", "https", "ws", "wss"): - raise UnsupportedProtocol( - f"Request URL has an unsupported protocol '{scheme}://'." - ) - - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("pool", None) - - with self._optional_thread_lock: - # Add the incoming request to our request queue. - pool_request = AsyncPoolRequest(request) - self._requests.append(pool_request) - - try: - while True: - with self._optional_thread_lock: - # Assign incoming requests to available connections, - # closing or creating new connections as required. - closing = self._assign_requests_to_connections() - await self._close_connections(closing) - - # Wait until this request has an assigned connection. - connection = await pool_request.wait_for_connection(timeout=timeout) - - try: - # Send the request on the assigned connection. - response = await connection.handle_async_request( - pool_request.request - ) - except ConnectionNotAvailable: - # In some cases a connection may initially be available to - # handle a request, but then become unavailable. - # - # In this case we clear the connection and try again. - pool_request.clear_connection() - else: - break # pragma: nocover - - except BaseException as exc: - with self._optional_thread_lock: - # For any exception or cancellation we remove the request from - # the queue, and then re-assign requests to connections. - self._requests.remove(pool_request) - closing = self._assign_requests_to_connections() - - await self._close_connections(closing) - raise exc from None - - # Return the response. Note that in this case we still have to manage - # the point at which the response is closed. - assert isinstance(response.stream, typing.AsyncIterable) - return Response( - status=response.status, - headers=response.headers, - content=PoolByteStream( - stream=response.stream, pool_request=pool_request, pool=self - ), - extensions=response.extensions, - ) - - def _assign_requests_to_connections(self) -> list[AsyncConnectionInterface]: - """ - Manage the state of the connection pool, assigning incoming - requests to connections as available. - - Called whenever a new request is added or removed from the pool. - - Any closing connections are returned, allowing the I/O for closing - those connections to be handled seperately. - """ - closing_connections = [] - - # First we handle cleaning up any connections that are closed, - # have expired their keep-alive, or surplus idle connections. - for connection in list(self._connections): - if connection.is_closed(): - # log: "removing closed connection" - self._connections.remove(connection) - elif connection.has_expired(): - # log: "closing expired connection" - self._connections.remove(connection) - closing_connections.append(connection) - elif ( - connection.is_idle() - and len([connection.is_idle() for connection in self._connections]) - > self._max_keepalive_connections - ): - # log: "closing idle connection" - self._connections.remove(connection) - closing_connections.append(connection) - - # Assign queued requests to connections. - queued_requests = [request for request in self._requests if request.is_queued()] - for pool_request in queued_requests: - origin = pool_request.request.url.origin - available_connections = [ - connection - for connection in self._connections - if connection.can_handle_request(origin) and connection.is_available() - ] - idle_connections = [ - connection for connection in self._connections if connection.is_idle() - ] - - # There are three cases for how we may be able to handle the request: - # - # 1. There is an existing connection that can handle the request. - # 2. We can create a new connection to handle the request. - # 3. We can close an idle connection and then create a new connection - # to handle the request. - if available_connections: - # log: "reusing existing connection" - connection = available_connections[0] - pool_request.assign_to_connection(connection) - elif len(self._connections) < self._max_connections: - # log: "creating new connection" - connection = self.create_connection(origin) - self._connections.append(connection) - pool_request.assign_to_connection(connection) - elif idle_connections: - # log: "closing idle connection" - connection = idle_connections[0] - self._connections.remove(connection) - closing_connections.append(connection) - # log: "creating new connection" - connection = self.create_connection(origin) - self._connections.append(connection) - pool_request.assign_to_connection(connection) - - return closing_connections - - async def _close_connections(self, closing: list[AsyncConnectionInterface]) -> None: - # Close connections which have been removed from the pool. - with AsyncShieldCancellation(): - for connection in closing: - await connection.aclose() - - async def aclose(self) -> None: - # Explicitly close the connection pool. - # Clears all existing requests and connections. - with self._optional_thread_lock: - closing_connections = list(self._connections) - self._connections = [] - await self._close_connections(closing_connections) - - async def __aenter__(self) -> AsyncConnectionPool: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - await self.aclose() - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - with self._optional_thread_lock: - request_is_queued = [request.is_queued() for request in self._requests] - connection_is_idle = [ - connection.is_idle() for connection in self._connections - ] - - num_active_requests = request_is_queued.count(False) - num_queued_requests = request_is_queued.count(True) - num_active_connections = connection_is_idle.count(False) - num_idle_connections = connection_is_idle.count(True) - - requests_info = ( - f"Requests: {num_active_requests} active, {num_queued_requests} queued" - ) - connection_info = ( - f"Connections: {num_active_connections} active, {num_idle_connections} idle" - ) - - return f"<{class_name} [{requests_info} | {connection_info}]>" - - -class PoolByteStream: - def __init__( - self, - stream: typing.AsyncIterable[bytes], - pool_request: AsyncPoolRequest, - pool: AsyncConnectionPool, - ) -> None: - self._stream = stream - self._pool_request = pool_request - self._pool = pool - self._closed = False - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - try: - async for part in self._stream: - yield part - except BaseException as exc: - await self.aclose() - raise exc from None - - async def aclose(self) -> None: - if not self._closed: - self._closed = True - with AsyncShieldCancellation(): - if hasattr(self._stream, "aclose"): - await self._stream.aclose() - - with self._pool._optional_thread_lock: - self._pool._requests.remove(self._pool_request) - closing = self._pool._assign_requests_to_connections() - - await self._pool._close_connections(closing) diff --git a/contrib/python/httpcore/httpcore/_async/http11.py b/contrib/python/httpcore/httpcore/_async/http11.py deleted file mode 100644 index e6d6d709852..00000000000 --- a/contrib/python/httpcore/httpcore/_async/http11.py +++ /dev/null @@ -1,379 +0,0 @@ -from __future__ import annotations - -import enum -import logging -import ssl -import time -import types -import typing - -import h11 - -from .._backends.base import AsyncNetworkStream -from .._exceptions import ( - ConnectionNotAvailable, - LocalProtocolError, - RemoteProtocolError, - WriteError, - map_exceptions, -) -from .._models import Origin, Request, Response -from .._synchronization import AsyncLock, AsyncShieldCancellation -from .._trace import Trace -from .interfaces import AsyncConnectionInterface - -logger = logging.getLogger("httpcore.http11") - - -# A subset of `h11.Event` types supported by `_send_event` -H11SendEvent = typing.Union[ - h11.Request, - h11.Data, - h11.EndOfMessage, -] - - -class HTTPConnectionState(enum.IntEnum): - NEW = 0 - ACTIVE = 1 - IDLE = 2 - CLOSED = 3 - - -class AsyncHTTP11Connection(AsyncConnectionInterface): - READ_NUM_BYTES = 64 * 1024 - MAX_INCOMPLETE_EVENT_SIZE = 100 * 1024 - - def __init__( - self, - origin: Origin, - stream: AsyncNetworkStream, - keepalive_expiry: float | None = None, - ) -> None: - self._origin = origin - self._network_stream = stream - self._keepalive_expiry: float | None = keepalive_expiry - self._expire_at: float | None = None - self._state = HTTPConnectionState.NEW - self._state_lock = AsyncLock() - self._request_count = 0 - self._h11_state = h11.Connection( - our_role=h11.CLIENT, - max_incomplete_event_size=self.MAX_INCOMPLETE_EVENT_SIZE, - ) - - async def handle_async_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection " - f"to {self._origin}" - ) - - async with self._state_lock: - if self._state in (HTTPConnectionState.NEW, HTTPConnectionState.IDLE): - self._request_count += 1 - self._state = HTTPConnectionState.ACTIVE - self._expire_at = None - else: - raise ConnectionNotAvailable() - - try: - kwargs = {"request": request} - try: - async with Trace( - "send_request_headers", logger, request, kwargs - ) as trace: - await self._send_request_headers(**kwargs) - async with Trace("send_request_body", logger, request, kwargs) as trace: - await self._send_request_body(**kwargs) - except WriteError: - # If we get a write error while we're writing the request, - # then we supress this error and move on to attempting to - # read the response. Servers can sometimes close the request - # pre-emptively and then respond with a well formed HTTP - # error response. - pass - - async with Trace( - "receive_response_headers", logger, request, kwargs - ) as trace: - ( - http_version, - status, - reason_phrase, - headers, - trailing_data, - ) = await self._receive_response_headers(**kwargs) - trace.return_value = ( - http_version, - status, - reason_phrase, - headers, - ) - - network_stream = self._network_stream - - # CONNECT or Upgrade request - if (status == 101) or ( - (request.method == b"CONNECT") and (200 <= status < 300) - ): - network_stream = AsyncHTTP11UpgradeStream(network_stream, trailing_data) - - return Response( - status=status, - headers=headers, - content=HTTP11ConnectionByteStream(self, request), - extensions={ - "http_version": http_version, - "reason_phrase": reason_phrase, - "network_stream": network_stream, - }, - ) - except BaseException as exc: - with AsyncShieldCancellation(): - async with Trace("response_closed", logger, request) as trace: - await self._response_closed() - raise exc - - # Sending the request... - - async def _send_request_headers(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - with map_exceptions({h11.LocalProtocolError: LocalProtocolError}): - event = h11.Request( - method=request.method, - target=request.url.target, - headers=request.headers, - ) - await self._send_event(event, timeout=timeout) - - async def _send_request_body(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - assert isinstance(request.stream, typing.AsyncIterable) - async for chunk in request.stream: - event = h11.Data(data=chunk) - await self._send_event(event, timeout=timeout) - - await self._send_event(h11.EndOfMessage(), timeout=timeout) - - async def _send_event(self, event: h11.Event, timeout: float | None = None) -> None: - bytes_to_send = self._h11_state.send(event) - if bytes_to_send is not None: - await self._network_stream.write(bytes_to_send, timeout=timeout) - - # Receiving the response... - - async def _receive_response_headers( - self, request: Request - ) -> tuple[bytes, int, bytes, list[tuple[bytes, bytes]], bytes]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - while True: - event = await self._receive_event(timeout=timeout) - if isinstance(event, h11.Response): - break - if ( - isinstance(event, h11.InformationalResponse) - and event.status_code == 101 - ): - break - - http_version = b"HTTP/" + event.http_version - - # h11 version 0.11+ supports a `raw_items` interface to get the - # raw header casing, rather than the enforced lowercase headers. - headers = event.headers.raw_items() - - trailing_data, _ = self._h11_state.trailing_data - - return http_version, event.status_code, event.reason, headers, trailing_data - - async def _receive_response_body( - self, request: Request - ) -> typing.AsyncIterator[bytes]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - while True: - event = await self._receive_event(timeout=timeout) - if isinstance(event, h11.Data): - yield bytes(event.data) - elif isinstance(event, (h11.EndOfMessage, h11.PAUSED)): - break - - async def _receive_event( - self, timeout: float | None = None - ) -> h11.Event | type[h11.PAUSED]: - while True: - with map_exceptions({h11.RemoteProtocolError: RemoteProtocolError}): - event = self._h11_state.next_event() - - if event is h11.NEED_DATA: - data = await self._network_stream.read( - self.READ_NUM_BYTES, timeout=timeout - ) - - # If we feed this case through h11 we'll raise an exception like: - # - # httpcore.RemoteProtocolError: can't handle event type - # ConnectionClosed when role=SERVER and state=SEND_RESPONSE - # - # Which is accurate, but not very informative from an end-user - # perspective. Instead we handle this case distinctly and treat - # it as a ConnectError. - if data == b"" and self._h11_state.their_state == h11.SEND_RESPONSE: - msg = "Server disconnected without sending a response." - raise RemoteProtocolError(msg) - - self._h11_state.receive_data(data) - else: - # mypy fails to narrow the type in the above if statement above - return event # type: ignore[return-value] - - async def _response_closed(self) -> None: - async with self._state_lock: - if ( - self._h11_state.our_state is h11.DONE - and self._h11_state.their_state is h11.DONE - ): - self._state = HTTPConnectionState.IDLE - self._h11_state.start_next_cycle() - if self._keepalive_expiry is not None: - now = time.monotonic() - self._expire_at = now + self._keepalive_expiry - else: - await self.aclose() - - # Once the connection is no longer required... - - async def aclose(self) -> None: - # Note that this method unilaterally closes the connection, and does - # not have any kind of locking in place around it. - self._state = HTTPConnectionState.CLOSED - await self._network_stream.aclose() - - # The AsyncConnectionInterface methods provide information about the state of - # the connection, allowing for a connection pooling implementation to - # determine when to reuse and when to close the connection... - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - def is_available(self) -> bool: - # Note that HTTP/1.1 connections in the "NEW" state are not treated as - # being "available". The control flow which created the connection will - # be able to send an outgoing request, but the connection will not be - # acquired from the connection pool for any other request. - return self._state == HTTPConnectionState.IDLE - - def has_expired(self) -> bool: - now = time.monotonic() - keepalive_expired = self._expire_at is not None and now > self._expire_at - - # If the HTTP connection is idle but the socket is readable, then the - # only valid state is that the socket is about to return b"", indicating - # a server-initiated disconnect. - server_disconnected = ( - self._state == HTTPConnectionState.IDLE - and self._network_stream.get_extra_info("is_readable") - ) - - return keepalive_expired or server_disconnected - - def is_idle(self) -> bool: - return self._state == HTTPConnectionState.IDLE - - def is_closed(self) -> bool: - return self._state == HTTPConnectionState.CLOSED - - def info(self) -> str: - origin = str(self._origin) - return ( - f"{origin!r}, HTTP/1.1, {self._state.name}, " - f"Request Count: {self._request_count}" - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - origin = str(self._origin) - return ( - f"<{class_name} [{origin!r}, {self._state.name}, " - f"Request Count: {self._request_count}]>" - ) - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - async def __aenter__(self) -> AsyncHTTP11Connection: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - await self.aclose() - - -class HTTP11ConnectionByteStream: - def __init__(self, connection: AsyncHTTP11Connection, request: Request) -> None: - self._connection = connection - self._request = request - self._closed = False - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - kwargs = {"request": self._request} - try: - async with Trace("receive_response_body", logger, self._request, kwargs): - async for chunk in self._connection._receive_response_body(**kwargs): - yield chunk - except BaseException as exc: - # If we get an exception while streaming the response, - # we want to close the response (and possibly the connection) - # before raising that exception. - with AsyncShieldCancellation(): - await self.aclose() - raise exc - - async def aclose(self) -> None: - if not self._closed: - self._closed = True - async with Trace("response_closed", logger, self._request): - await self._connection._response_closed() - - -class AsyncHTTP11UpgradeStream(AsyncNetworkStream): - def __init__(self, stream: AsyncNetworkStream, leading_data: bytes) -> None: - self._stream = stream - self._leading_data = leading_data - - async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - if self._leading_data: - buffer = self._leading_data[:max_bytes] - self._leading_data = self._leading_data[max_bytes:] - return buffer - else: - return await self._stream.read(max_bytes, timeout) - - async def write(self, buffer: bytes, timeout: float | None = None) -> None: - await self._stream.write(buffer, timeout) - - async def aclose(self) -> None: - await self._stream.aclose() - - async def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> AsyncNetworkStream: - return await self._stream.start_tls(ssl_context, server_hostname, timeout) - - def get_extra_info(self, info: str) -> typing.Any: - return self._stream.get_extra_info(info) diff --git a/contrib/python/httpcore/httpcore/_async/http2.py b/contrib/python/httpcore/httpcore/_async/http2.py deleted file mode 100644 index c6434a04969..00000000000 --- a/contrib/python/httpcore/httpcore/_async/http2.py +++ /dev/null @@ -1,583 +0,0 @@ -from __future__ import annotations - -import enum -import logging -import time -import types -import typing - -import h2.config -import h2.connection -import h2.events -import h2.exceptions -import h2.settings - -from .._backends.base import AsyncNetworkStream -from .._exceptions import ( - ConnectionNotAvailable, - LocalProtocolError, - RemoteProtocolError, -) -from .._models import Origin, Request, Response -from .._synchronization import AsyncLock, AsyncSemaphore, AsyncShieldCancellation -from .._trace import Trace -from .interfaces import AsyncConnectionInterface - -logger = logging.getLogger("httpcore.http2") - - -def has_body_headers(request: Request) -> bool: - return any( - k.lower() == b"content-length" or k.lower() == b"transfer-encoding" - for k, v in request.headers - ) - - -class HTTPConnectionState(enum.IntEnum): - ACTIVE = 1 - IDLE = 2 - CLOSED = 3 - - -class AsyncHTTP2Connection(AsyncConnectionInterface): - READ_NUM_BYTES = 64 * 1024 - CONFIG = h2.config.H2Configuration(validate_inbound_headers=False) - - def __init__( - self, - origin: Origin, - stream: AsyncNetworkStream, - keepalive_expiry: float | None = None, - ): - self._origin = origin - self._network_stream = stream - self._keepalive_expiry: float | None = keepalive_expiry - self._h2_state = h2.connection.H2Connection(config=self.CONFIG) - self._state = HTTPConnectionState.IDLE - self._expire_at: float | None = None - self._request_count = 0 - self._init_lock = AsyncLock() - self._state_lock = AsyncLock() - self._read_lock = AsyncLock() - self._write_lock = AsyncLock() - self._sent_connection_init = False - self._used_all_stream_ids = False - self._connection_error = False - - # Mapping from stream ID to response stream events. - self._events: dict[ - int, - h2.events.ResponseReceived - | h2.events.DataReceived - | h2.events.StreamEnded - | h2.events.StreamReset, - ] = {} - - # Connection terminated events are stored as state since - # we need to handle them for all streams. - self._connection_terminated: h2.events.ConnectionTerminated | None = None - - self._read_exception: Exception | None = None - self._write_exception: Exception | None = None - - async def handle_async_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - # This cannot occur in normal operation, since the connection pool - # will only send requests on connections that handle them. - # It's in place simply for resilience as a guard against incorrect - # usage, for anyone working directly with httpcore connections. - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection " - f"to {self._origin}" - ) - - async with self._state_lock: - if self._state in (HTTPConnectionState.ACTIVE, HTTPConnectionState.IDLE): - self._request_count += 1 - self._expire_at = None - self._state = HTTPConnectionState.ACTIVE - else: - raise ConnectionNotAvailable() - - async with self._init_lock: - if not self._sent_connection_init: - try: - kwargs = {"request": request} - async with Trace("send_connection_init", logger, request, kwargs): - await self._send_connection_init(**kwargs) - except BaseException as exc: - with AsyncShieldCancellation(): - await self.aclose() - raise exc - - self._sent_connection_init = True - - # Initially start with just 1 until the remote server provides - # its max_concurrent_streams value - self._max_streams = 1 - - local_settings_max_streams = ( - self._h2_state.local_settings.max_concurrent_streams - ) - self._max_streams_semaphore = AsyncSemaphore(local_settings_max_streams) - - for _ in range(local_settings_max_streams - self._max_streams): - await self._max_streams_semaphore.acquire() - - await self._max_streams_semaphore.acquire() - - try: - stream_id = self._h2_state.get_next_available_stream_id() - self._events[stream_id] = [] - except h2.exceptions.NoAvailableStreamIDError: # pragma: nocover - self._used_all_stream_ids = True - self._request_count -= 1 - raise ConnectionNotAvailable() - - try: - kwargs = {"request": request, "stream_id": stream_id} - async with Trace("send_request_headers", logger, request, kwargs): - await self._send_request_headers(request=request, stream_id=stream_id) - async with Trace("send_request_body", logger, request, kwargs): - await self._send_request_body(request=request, stream_id=stream_id) - async with Trace( - "receive_response_headers", logger, request, kwargs - ) as trace: - status, headers = await self._receive_response( - request=request, stream_id=stream_id - ) - trace.return_value = (status, headers) - - return Response( - status=status, - headers=headers, - content=HTTP2ConnectionByteStream(self, request, stream_id=stream_id), - extensions={ - "http_version": b"HTTP/2", - "network_stream": self._network_stream, - "stream_id": stream_id, - }, - ) - except BaseException as exc: # noqa: PIE786 - with AsyncShieldCancellation(): - kwargs = {"stream_id": stream_id} - async with Trace("response_closed", logger, request, kwargs): - await self._response_closed(stream_id=stream_id) - - if isinstance(exc, h2.exceptions.ProtocolError): - # One case where h2 can raise a protocol error is when a - # closed frame has been seen by the state machine. - # - # This happens when one stream is reading, and encounters - # a GOAWAY event. Other flows of control may then raise - # a protocol error at any point they interact with the 'h2_state'. - # - # In this case we'll have stored the event, and should raise - # it as a RemoteProtocolError. - if self._connection_terminated: # pragma: nocover - raise RemoteProtocolError(self._connection_terminated) - # If h2 raises a protocol error in some other state then we - # must somehow have made a protocol violation. - raise LocalProtocolError(exc) # pragma: nocover - - raise exc - - async def _send_connection_init(self, request: Request) -> None: - """ - The HTTP/2 connection requires some initial setup before we can start - using individual request/response streams on it. - """ - # Need to set these manually here instead of manipulating via - # __setitem__() otherwise the H2Connection will emit SettingsUpdate - # frames in addition to sending the undesired defaults. - self._h2_state.local_settings = h2.settings.Settings( - client=True, - initial_values={ - # Disable PUSH_PROMISE frames from the server since we don't do anything - # with them for now. Maybe when we support caching? - h2.settings.SettingCodes.ENABLE_PUSH: 0, - # These two are taken from h2 for safe defaults - h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 100, - h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: 65536, - }, - ) - - # Some websites (*cough* Yahoo *cough*) balk at this setting being - # present in the initial handshake since it's not defined in the original - # RFC despite the RFC mandating ignoring settings you don't know about. - del self._h2_state.local_settings[ - h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL - ] - - self._h2_state.initiate_connection() - self._h2_state.increment_flow_control_window(2**24) - await self._write_outgoing_data(request) - - # Sending the request... - - async def _send_request_headers(self, request: Request, stream_id: int) -> None: - """ - Send the request headers to a given stream ID. - """ - end_stream = not has_body_headers(request) - - # In HTTP/2 the ':authority' pseudo-header is used instead of 'Host'. - # In order to gracefully handle HTTP/1.1 and HTTP/2 we always require - # HTTP/1.1 style headers, and map them appropriately if we end up on - # an HTTP/2 connection. - authority = [v for k, v in request.headers if k.lower() == b"host"][0] - - headers = [ - (b":method", request.method), - (b":authority", authority), - (b":scheme", request.url.scheme), - (b":path", request.url.target), - ] + [ - (k.lower(), v) - for k, v in request.headers - if k.lower() - not in ( - b"host", - b"transfer-encoding", - ) - ] - - self._h2_state.send_headers(stream_id, headers, end_stream=end_stream) - self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id) - await self._write_outgoing_data(request) - - async def _send_request_body(self, request: Request, stream_id: int) -> None: - """ - Iterate over the request body sending it to a given stream ID. - """ - if not has_body_headers(request): - return - - assert isinstance(request.stream, typing.AsyncIterable) - async for data in request.stream: - await self._send_stream_data(request, stream_id, data) - await self._send_end_stream(request, stream_id) - - async def _send_stream_data( - self, request: Request, stream_id: int, data: bytes - ) -> None: - """ - Send a single chunk of data in one or more data frames. - """ - while data: - max_flow = await self._wait_for_outgoing_flow(request, stream_id) - chunk_size = min(len(data), max_flow) - chunk, data = data[:chunk_size], data[chunk_size:] - self._h2_state.send_data(stream_id, chunk) - await self._write_outgoing_data(request) - - async def _send_end_stream(self, request: Request, stream_id: int) -> None: - """ - Send an empty data frame on on a given stream ID with the END_STREAM flag set. - """ - self._h2_state.end_stream(stream_id) - await self._write_outgoing_data(request) - - # Receiving the response... - - async def _receive_response( - self, request: Request, stream_id: int - ) -> tuple[int, list[tuple[bytes, bytes]]]: - """ - Return the response status code and headers for a given stream ID. - """ - while True: - event = await self._receive_stream_event(request, stream_id) - if isinstance(event, h2.events.ResponseReceived): - break - - status_code = 200 - headers = [] - for k, v in event.headers: - if k == b":status": - status_code = int(v.decode("ascii", errors="ignore")) - elif not k.startswith(b":"): - headers.append((k, v)) - - return (status_code, headers) - - async def _receive_response_body( - self, request: Request, stream_id: int - ) -> typing.AsyncIterator[bytes]: - """ - Iterator that returns the bytes of the response body for a given stream ID. - """ - while True: - event = await self._receive_stream_event(request, stream_id) - if isinstance(event, h2.events.DataReceived): - amount = event.flow_controlled_length - self._h2_state.acknowledge_received_data(amount, stream_id) - await self._write_outgoing_data(request) - yield event.data - elif isinstance(event, h2.events.StreamEnded): - break - - async def _receive_stream_event( - self, request: Request, stream_id: int - ) -> h2.events.ResponseReceived | h2.events.DataReceived | h2.events.StreamEnded: - """ - Return the next available event for a given stream ID. - - Will read more data from the network if required. - """ - while not self._events.get(stream_id): - await self._receive_events(request, stream_id) - event = self._events[stream_id].pop(0) - if isinstance(event, h2.events.StreamReset): - raise RemoteProtocolError(event) - return event - - async def _receive_events( - self, request: Request, stream_id: int | None = None - ) -> None: - """ - Read some data from the network until we see one or more events - for a given stream ID. - """ - async with self._read_lock: - if self._connection_terminated is not None: - last_stream_id = self._connection_terminated.last_stream_id - if stream_id and last_stream_id and stream_id > last_stream_id: - self._request_count -= 1 - raise ConnectionNotAvailable() - raise RemoteProtocolError(self._connection_terminated) - - # This conditional is a bit icky. We don't want to block reading if we've - # actually got an event to return for a given stream. We need to do that - # check *within* the atomic read lock. Though it also need to be optional, - # because when we call it from `_wait_for_outgoing_flow` we *do* want to - # block until we've available flow control, event when we have events - # pending for the stream ID we're attempting to send on. - if stream_id is None or not self._events.get(stream_id): - events = await self._read_incoming_data(request) - for event in events: - if isinstance(event, h2.events.RemoteSettingsChanged): - async with Trace( - "receive_remote_settings", logger, request - ) as trace: - await self._receive_remote_settings_change(event) - trace.return_value = event - - elif isinstance( - event, - ( - h2.events.ResponseReceived, - h2.events.DataReceived, - h2.events.StreamEnded, - h2.events.StreamReset, - ), - ): - if event.stream_id in self._events: - self._events[event.stream_id].append(event) - - elif isinstance(event, h2.events.ConnectionTerminated): - self._connection_terminated = event - - await self._write_outgoing_data(request) - - async def _receive_remote_settings_change(self, event: h2.events.Event) -> None: - max_concurrent_streams = event.changed_settings.get( - h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS - ) - if max_concurrent_streams: - new_max_streams = min( - max_concurrent_streams.new_value, - self._h2_state.local_settings.max_concurrent_streams, - ) - if new_max_streams and new_max_streams != self._max_streams: - while new_max_streams > self._max_streams: - await self._max_streams_semaphore.release() - self._max_streams += 1 - while new_max_streams < self._max_streams: - await self._max_streams_semaphore.acquire() - self._max_streams -= 1 - - async def _response_closed(self, stream_id: int) -> None: - await self._max_streams_semaphore.release() - del self._events[stream_id] - async with self._state_lock: - if self._connection_terminated and not self._events: - await self.aclose() - - elif self._state == HTTPConnectionState.ACTIVE and not self._events: - self._state = HTTPConnectionState.IDLE - if self._keepalive_expiry is not None: - now = time.monotonic() - self._expire_at = now + self._keepalive_expiry - if self._used_all_stream_ids: # pragma: nocover - await self.aclose() - - async def aclose(self) -> None: - # Note that this method unilaterally closes the connection, and does - # not have any kind of locking in place around it. - self._h2_state.close_connection() - self._state = HTTPConnectionState.CLOSED - await self._network_stream.aclose() - - # Wrappers around network read/write operations... - - async def _read_incoming_data(self, request: Request) -> list[h2.events.Event]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - if self._read_exception is not None: - raise self._read_exception # pragma: nocover - - try: - data = await self._network_stream.read(self.READ_NUM_BYTES, timeout) - if data == b"": - raise RemoteProtocolError("Server disconnected") - except Exception as exc: - # If we get a network error we should: - # - # 1. Save the exception and just raise it immediately on any future reads. - # (For example, this means that a single read timeout or disconnect will - # immediately close all pending streams. Without requiring multiple - # sequential timeouts.) - # 2. Mark the connection as errored, so that we don't accept any other - # incoming requests. - self._read_exception = exc - self._connection_error = True - raise exc - - events: list[h2.events.Event] = self._h2_state.receive_data(data) - - return events - - async def _write_outgoing_data(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - async with self._write_lock: - data_to_send = self._h2_state.data_to_send() - - if self._write_exception is not None: - raise self._write_exception # pragma: nocover - - try: - await self._network_stream.write(data_to_send, timeout) - except Exception as exc: # pragma: nocover - # If we get a network error we should: - # - # 1. Save the exception and just raise it immediately on any future write. - # (For example, this means that a single write timeout or disconnect will - # immediately close all pending streams. Without requiring multiple - # sequential timeouts.) - # 2. Mark the connection as errored, so that we don't accept any other - # incoming requests. - self._write_exception = exc - self._connection_error = True - raise exc - - # Flow control... - - async def _wait_for_outgoing_flow(self, request: Request, stream_id: int) -> int: - """ - Returns the maximum allowable outgoing flow for a given stream. - - If the allowable flow is zero, then waits on the network until - WindowUpdated frames have increased the flow rate. - https://tools.ietf.org/html/rfc7540#section-6.9 - """ - local_flow: int = self._h2_state.local_flow_control_window(stream_id) - max_frame_size: int = self._h2_state.max_outbound_frame_size - flow = min(local_flow, max_frame_size) - while flow == 0: - await self._receive_events(request) - local_flow = self._h2_state.local_flow_control_window(stream_id) - max_frame_size = self._h2_state.max_outbound_frame_size - flow = min(local_flow, max_frame_size) - return flow - - # Interface for connection pooling... - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - def is_available(self) -> bool: - return ( - self._state != HTTPConnectionState.CLOSED - and not self._connection_error - and not self._used_all_stream_ids - and not ( - self._h2_state.state_machine.state - == h2.connection.ConnectionState.CLOSED - ) - ) - - def has_expired(self) -> bool: - now = time.monotonic() - return self._expire_at is not None and now > self._expire_at - - def is_idle(self) -> bool: - return self._state == HTTPConnectionState.IDLE - - def is_closed(self) -> bool: - return self._state == HTTPConnectionState.CLOSED - - def info(self) -> str: - origin = str(self._origin) - return ( - f"{origin!r}, HTTP/2, {self._state.name}, " - f"Request Count: {self._request_count}" - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - origin = str(self._origin) - return ( - f"<{class_name} [{origin!r}, {self._state.name}, " - f"Request Count: {self._request_count}]>" - ) - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - async def __aenter__(self) -> AsyncHTTP2Connection: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - await self.aclose() - - -class HTTP2ConnectionByteStream: - def __init__( - self, connection: AsyncHTTP2Connection, request: Request, stream_id: int - ) -> None: - self._connection = connection - self._request = request - self._stream_id = stream_id - self._closed = False - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - kwargs = {"request": self._request, "stream_id": self._stream_id} - try: - async with Trace("receive_response_body", logger, self._request, kwargs): - async for chunk in self._connection._receive_response_body( - request=self._request, stream_id=self._stream_id - ): - yield chunk - except BaseException as exc: - # If we get an exception while streaming the response, - # we want to close the response (and possibly the connection) - # before raising that exception. - with AsyncShieldCancellation(): - await self.aclose() - raise exc - - async def aclose(self) -> None: - if not self._closed: - self._closed = True - kwargs = {"stream_id": self._stream_id} - async with Trace("response_closed", logger, self._request, kwargs): - await self._connection._response_closed(stream_id=self._stream_id) diff --git a/contrib/python/httpcore/httpcore/_async/http_proxy.py b/contrib/python/httpcore/httpcore/_async/http_proxy.py deleted file mode 100644 index cc9d92066e1..00000000000 --- a/contrib/python/httpcore/httpcore/_async/http_proxy.py +++ /dev/null @@ -1,367 +0,0 @@ -from __future__ import annotations - -import base64 -import logging -import ssl -import typing - -from .._backends.base import SOCKET_OPTION, AsyncNetworkBackend -from .._exceptions import ProxyError -from .._models import ( - URL, - Origin, - Request, - Response, - enforce_bytes, - enforce_headers, - enforce_url, -) -from .._ssl import default_ssl_context -from .._synchronization import AsyncLock -from .._trace import Trace -from .connection import AsyncHTTPConnection -from .connection_pool import AsyncConnectionPool -from .http11 import AsyncHTTP11Connection -from .interfaces import AsyncConnectionInterface - -ByteOrStr = typing.Union[bytes, str] -HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] -HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] - - -logger = logging.getLogger("httpcore.proxy") - - -def merge_headers( - default_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, - override_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, -) -> list[tuple[bytes, bytes]]: - """ - Append default_headers and override_headers, de-duplicating if a key exists - in both cases. - """ - default_headers = [] if default_headers is None else list(default_headers) - override_headers = [] if override_headers is None else list(override_headers) - has_override = set(key.lower() for key, value in override_headers) - default_headers = [ - (key, value) - for key, value in default_headers - if key.lower() not in has_override - ] - return default_headers + override_headers - - -class AsyncHTTPProxy(AsyncConnectionPool): # pragma: nocover - """ - A connection pool that sends requests via an HTTP proxy. - """ - - def __init__( - self, - proxy_url: URL | bytes | str, - proxy_auth: tuple[bytes | str, bytes | str] | None = None, - proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, - ssl_context: ssl.SSLContext | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: AsyncNetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - proxy_url: The URL to use when connecting to the proxy server. - For example `"http://127.0.0.1:8080/"`. - proxy_auth: Any proxy authentication as a two-tuple of - (username, password). May be either bytes or ascii-only str. - proxy_headers: Any HTTP headers to use for the proxy requests. - For example `{"Proxy-Authorization": "Basic <username>:<password>"}`. - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish - a connection. - local_address: Local address to connect from. Can also be used to - connect using a particular address family. Using - `local_address="0.0.0.0"` will connect using an `AF_INET` address - (IPv4), while using `local_address="::"` will connect using an - `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - """ - super().__init__( - ssl_context=ssl_context, - max_connections=max_connections, - max_keepalive_connections=max_keepalive_connections, - keepalive_expiry=keepalive_expiry, - http1=http1, - http2=http2, - network_backend=network_backend, - retries=retries, - local_address=local_address, - uds=uds, - socket_options=socket_options, - ) - - self._proxy_url = enforce_url(proxy_url, name="proxy_url") - if ( - self._proxy_url.scheme == b"http" and proxy_ssl_context is not None - ): # pragma: no cover - raise RuntimeError( - "The `proxy_ssl_context` argument is not allowed for the http scheme" - ) - - self._ssl_context = ssl_context - self._proxy_ssl_context = proxy_ssl_context - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - if proxy_auth is not None: - username = enforce_bytes(proxy_auth[0], name="proxy_auth") - password = enforce_bytes(proxy_auth[1], name="proxy_auth") - userpass = username + b":" + password - authorization = b"Basic " + base64.b64encode(userpass) - self._proxy_headers = [ - (b"Proxy-Authorization", authorization) - ] + self._proxy_headers - - def create_connection(self, origin: Origin) -> AsyncConnectionInterface: - if origin.scheme == b"http": - return AsyncForwardHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - keepalive_expiry=self._keepalive_expiry, - network_backend=self._network_backend, - proxy_ssl_context=self._proxy_ssl_context, - ) - return AsyncTunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - -class AsyncForwardHTTPConnection(AsyncConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, - keepalive_expiry: float | None = None, - network_backend: AsyncNetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - ) -> None: - self._connection = AsyncHTTPConnection( - origin=proxy_origin, - keepalive_expiry=keepalive_expiry, - network_backend=network_backend, - socket_options=socket_options, - ssl_context=proxy_ssl_context, - ) - self._proxy_origin = proxy_origin - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - self._remote_origin = remote_origin - - async def handle_async_request(self, request: Request) -> Response: - headers = merge_headers(self._proxy_headers, request.headers) - url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=bytes(request.url), - ) - proxy_request = Request( - method=request.method, - url=url, - headers=headers, - content=request.stream, - extensions=request.extensions, - ) - return await self._connection.handle_async_request(proxy_request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - async def aclose(self) -> None: - await self._connection.aclose() - - def info(self) -> str: - return self._connection.info() - - def is_available(self) -> bool: - return self._connection.is_available() - - def has_expired(self) -> bool: - return self._connection.has_expired() - - def is_idle(self) -> bool: - return self._connection.is_idle() - - def is_closed(self) -> bool: - return self._connection.is_closed() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" - - -class AsyncTunnelHTTPConnection(AsyncConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - ssl_context: ssl.SSLContext | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - proxy_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - network_backend: AsyncNetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - self._connection: AsyncConnectionInterface = AsyncHTTPConnection( - origin=proxy_origin, - keepalive_expiry=keepalive_expiry, - network_backend=network_backend, - socket_options=socket_options, - ssl_context=proxy_ssl_context, - ) - self._proxy_origin = proxy_origin - self._remote_origin = remote_origin - self._ssl_context = ssl_context - self._proxy_ssl_context = proxy_ssl_context - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._connect_lock = AsyncLock() - self._connected = False - - async def handle_async_request(self, request: Request) -> Response: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - async with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = await self._connection.handle_async_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - await self._connection.aclose() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from .http2 import AsyncHTTP2Connection - - self._connection = AsyncHTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = AsyncHTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connected = True - return await self._connection.handle_async_request(request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - async def aclose(self) -> None: - await self._connection.aclose() - - def info(self) -> str: - return self._connection.info() - - def is_available(self) -> bool: - return self._connection.is_available() - - def has_expired(self) -> bool: - return self._connection.has_expired() - - def is_idle(self) -> bool: - return self._connection.is_idle() - - def is_closed(self) -> bool: - return self._connection.is_closed() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/contrib/python/httpcore/httpcore/_async/interfaces.py b/contrib/python/httpcore/httpcore/_async/interfaces.py deleted file mode 100644 index 361583bede6..00000000000 --- a/contrib/python/httpcore/httpcore/_async/interfaces.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import contextlib -import typing - -from .._models import ( - URL, - Extensions, - HeaderTypes, - Origin, - Request, - Response, - enforce_bytes, - enforce_headers, - enforce_url, - include_request_headers, -) - - -class AsyncRequestInterface: - async def request( - self, - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.AsyncIterator[bytes] | None = None, - extensions: Extensions | None = None, - ) -> Response: - # Strict type checking on our parameters. - method = enforce_bytes(method, name="method") - url = enforce_url(url, name="url") - headers = enforce_headers(headers, name="headers") - - # Include Host header, and optionally Content-Length or Transfer-Encoding. - headers = include_request_headers(headers, url=url, content=content) - - request = Request( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) - response = await self.handle_async_request(request) - try: - await response.aread() - finally: - await response.aclose() - return response - - @contextlib.asynccontextmanager - async def stream( - self, - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.AsyncIterator[bytes] | None = None, - extensions: Extensions | None = None, - ) -> typing.AsyncIterator[Response]: - # Strict type checking on our parameters. - method = enforce_bytes(method, name="method") - url = enforce_url(url, name="url") - headers = enforce_headers(headers, name="headers") - - # Include Host header, and optionally Content-Length or Transfer-Encoding. - headers = include_request_headers(headers, url=url, content=content) - - request = Request( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) - response = await self.handle_async_request(request) - try: - yield response - finally: - await response.aclose() - - async def handle_async_request(self, request: Request) -> Response: - raise NotImplementedError() # pragma: nocover - - -class AsyncConnectionInterface(AsyncRequestInterface): - async def aclose(self) -> None: - raise NotImplementedError() # pragma: nocover - - def info(self) -> str: - raise NotImplementedError() # pragma: nocover - - def can_handle_request(self, origin: Origin) -> bool: - raise NotImplementedError() # pragma: nocover - - def is_available(self) -> bool: - """ - Return `True` if the connection is currently able to accept an - outgoing request. - - An HTTP/1.1 connection will only be available if it is currently idle. - - An HTTP/2 connection will be available so long as the stream ID space is - not yet exhausted, and the connection is not in an error state. - - While the connection is being established we may not yet know if it is going - to result in an HTTP/1.1 or HTTP/2 connection. The connection should be - treated as being available, but might ultimately raise `NewConnectionRequired` - required exceptions if multiple requests are attempted over a connection - that ends up being established as HTTP/1.1. - """ - raise NotImplementedError() # pragma: nocover - - def has_expired(self) -> bool: - """ - Return `True` if the connection is in a state where it should be closed. - - This either means that the connection is idle and it has passed the - expiry time on its keep-alive, or that server has sent an EOF. - """ - raise NotImplementedError() # pragma: nocover - - def is_idle(self) -> bool: - """ - Return `True` if the connection is currently idle. - """ - raise NotImplementedError() # pragma: nocover - - def is_closed(self) -> bool: - """ - Return `True` if the connection has been closed. - - Used when a response is closed to determine if the connection may be - returned to the connection pool or not. - """ - raise NotImplementedError() # pragma: nocover diff --git a/contrib/python/httpcore/httpcore/_async/socks_proxy.py b/contrib/python/httpcore/httpcore/_async/socks_proxy.py deleted file mode 100644 index b363f55a0b0..00000000000 --- a/contrib/python/httpcore/httpcore/_async/socks_proxy.py +++ /dev/null @@ -1,341 +0,0 @@ -from __future__ import annotations - -import logging -import ssl - -import socksio - -from .._backends.auto import AutoBackend -from .._backends.base import AsyncNetworkBackend, AsyncNetworkStream -from .._exceptions import ConnectionNotAvailable, ProxyError -from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url -from .._ssl import default_ssl_context -from .._synchronization import AsyncLock -from .._trace import Trace -from .connection_pool import AsyncConnectionPool -from .http11 import AsyncHTTP11Connection -from .interfaces import AsyncConnectionInterface - -logger = logging.getLogger("httpcore.socks") - - -AUTH_METHODS = { - b"\x00": "NO AUTHENTICATION REQUIRED", - b"\x01": "GSSAPI", - b"\x02": "USERNAME/PASSWORD", - b"\xff": "NO ACCEPTABLE METHODS", -} - -REPLY_CODES = { - b"\x00": "Succeeded", - b"\x01": "General SOCKS server failure", - b"\x02": "Connection not allowed by ruleset", - b"\x03": "Network unreachable", - b"\x04": "Host unreachable", - b"\x05": "Connection refused", - b"\x06": "TTL expired", - b"\x07": "Command not supported", - b"\x08": "Address type not supported", -} - - -async def _init_socks5_connection( - stream: AsyncNetworkStream, - *, - host: bytes, - port: int, - auth: tuple[bytes, bytes] | None = None, -) -> None: - conn = socksio.socks5.SOCKS5Connection() - - # Auth method request - auth_method = ( - socksio.socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED - if auth is None - else socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD - ) - conn.send(socksio.socks5.SOCKS5AuthMethodsRequest([auth_method])) - outgoing_bytes = conn.data_to_send() - await stream.write(outgoing_bytes) - - # Auth method response - incoming_bytes = await stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5AuthReply) - if response.method != auth_method: - requested = AUTH_METHODS.get(auth_method, "UNKNOWN") - responded = AUTH_METHODS.get(response.method, "UNKNOWN") - raise ProxyError( - f"Requested {requested} from proxy server, but got {responded}." - ) - - if response.method == socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: - # Username/password request - assert auth is not None - username, password = auth - conn.send(socksio.socks5.SOCKS5UsernamePasswordRequest(username, password)) - outgoing_bytes = conn.data_to_send() - await stream.write(outgoing_bytes) - - # Username/password response - incoming_bytes = await stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5UsernamePasswordReply) - if not response.success: - raise ProxyError("Invalid username/password") - - # Connect request - conn.send( - socksio.socks5.SOCKS5CommandRequest.from_address( - socksio.socks5.SOCKS5Command.CONNECT, (host, port) - ) - ) - outgoing_bytes = conn.data_to_send() - await stream.write(outgoing_bytes) - - # Connect response - incoming_bytes = await stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5Reply) - if response.reply_code != socksio.socks5.SOCKS5ReplyCode.SUCCEEDED: - reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") - raise ProxyError(f"Proxy Server could not connect: {reply_code}.") - - -class AsyncSOCKSProxy(AsyncConnectionPool): # pragma: nocover - """ - A connection pool that sends requests via an HTTP proxy. - """ - - def __init__( - self, - proxy_url: URL | bytes | str, - proxy_auth: tuple[bytes | str, bytes | str] | None = None, - ssl_context: ssl.SSLContext | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - network_backend: AsyncNetworkBackend | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - proxy_url: The URL to use when connecting to the proxy server. - For example `"http://127.0.0.1:8080/"`. - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish - a connection. - local_address: Local address to connect from. Can also be used to - connect using a particular address family. Using - `local_address="0.0.0.0"` will connect using an `AF_INET` address - (IPv4), while using `local_address="::"` will connect using an - `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - """ - super().__init__( - ssl_context=ssl_context, - max_connections=max_connections, - max_keepalive_connections=max_keepalive_connections, - keepalive_expiry=keepalive_expiry, - http1=http1, - http2=http2, - network_backend=network_backend, - retries=retries, - ) - self._ssl_context = ssl_context - self._proxy_url = enforce_url(proxy_url, name="proxy_url") - if proxy_auth is not None: - username, password = proxy_auth - username_bytes = enforce_bytes(username, name="proxy_auth") - password_bytes = enforce_bytes(password, name="proxy_auth") - self._proxy_auth: tuple[bytes, bytes] | None = ( - username_bytes, - password_bytes, - ) - else: - self._proxy_auth = None - - def create_connection(self, origin: Origin) -> AsyncConnectionInterface: - return AsyncSocks5Connection( - proxy_origin=self._proxy_url.origin, - remote_origin=origin, - proxy_auth=self._proxy_auth, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - -class AsyncSocks5Connection(AsyncConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - proxy_auth: tuple[bytes, bytes] | None = None, - ssl_context: ssl.SSLContext | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - network_backend: AsyncNetworkBackend | None = None, - ) -> None: - self._proxy_origin = proxy_origin - self._remote_origin = remote_origin - self._proxy_auth = proxy_auth - self._ssl_context = ssl_context - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - - self._network_backend: AsyncNetworkBackend = ( - AutoBackend() if network_backend is None else network_backend - ) - self._connect_lock = AsyncLock() - self._connection: AsyncConnectionInterface | None = None - self._connect_failed = False - - async def handle_async_request(self, request: Request) -> Response: - timeouts = request.extensions.get("timeout", {}) - sni_hostname = request.extensions.get("sni_hostname", None) - timeout = timeouts.get("connect", None) - - async with self._connect_lock: - if self._connection is None: - try: - # Connect to the proxy - kwargs = { - "host": self._proxy_origin.host.decode("ascii"), - "port": self._proxy_origin.port, - "timeout": timeout, - } - async with Trace("connect_tcp", logger, request, kwargs) as trace: - stream = await self._network_backend.connect_tcp(**kwargs) - trace.return_value = stream - - # Connect to the remote host using socks5 - kwargs = { - "stream": stream, - "host": self._remote_origin.host.decode("ascii"), - "port": self._remote_origin.port, - "auth": self._proxy_auth, - } - async with Trace( - "setup_socks5_connection", logger, request, kwargs - ) as trace: - await _init_socks5_connection(**kwargs) - trace.return_value = stream - - # Upgrade the stream to SSL - if self._remote_origin.scheme == b"https": - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ( - ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ) - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": sni_hostname - or self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or ( - self._http2 and not self._http1 - ): # pragma: nocover - from .http2 import AsyncHTTP2Connection - - self._connection = AsyncHTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = AsyncHTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - except Exception as exc: - self._connect_failed = True - raise exc - elif not self._connection.is_available(): # pragma: nocover - raise ConnectionNotAvailable() - - return await self._connection.handle_async_request(request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - async def aclose(self) -> None: - if self._connection is not None: - await self._connection.aclose() - - def is_available(self) -> bool: - if self._connection is None: # pragma: nocover - # If HTTP/2 support is enabled, and the resulting connection could - # end up as HTTP/2 then we should indicate the connection as being - # available to service multiple requests. - return ( - self._http2 - and (self._remote_origin.scheme == b"https" or not self._http1) - and not self._connect_failed - ) - return self._connection.is_available() - - def has_expired(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.has_expired() - - def is_idle(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.is_idle() - - def is_closed(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.is_closed() - - def info(self) -> str: - if self._connection is None: # pragma: nocover - return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" - return self._connection.info() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/contrib/python/httpcore/httpcore/_backends/__init__.py b/contrib/python/httpcore/httpcore/_backends/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/__init__.py +++ /dev/null diff --git a/contrib/python/httpcore/httpcore/_backends/anyio.py b/contrib/python/httpcore/httpcore/_backends/anyio.py deleted file mode 100644 index a140095e1b8..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/anyio.py +++ /dev/null @@ -1,146 +0,0 @@ -from __future__ import annotations - -import ssl -import typing - -import anyio - -from .._exceptions import ( - ConnectError, - ConnectTimeout, - ReadError, - ReadTimeout, - WriteError, - WriteTimeout, - map_exceptions, -) -from .._utils import is_socket_readable -from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream - - -class AnyIOStream(AsyncNetworkStream): - def __init__(self, stream: anyio.abc.ByteStream) -> None: - self._stream = stream - - async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - exc_map = { - TimeoutError: ReadTimeout, - anyio.BrokenResourceError: ReadError, - anyio.ClosedResourceError: ReadError, - anyio.EndOfStream: ReadError, - } - with map_exceptions(exc_map): - with anyio.fail_after(timeout): - try: - return await self._stream.receive(max_bytes=max_bytes) - except anyio.EndOfStream: # pragma: nocover - return b"" - - async def write(self, buffer: bytes, timeout: float | None = None) -> None: - if not buffer: - return - - exc_map = { - TimeoutError: WriteTimeout, - anyio.BrokenResourceError: WriteError, - anyio.ClosedResourceError: WriteError, - } - with map_exceptions(exc_map): - with anyio.fail_after(timeout): - await self._stream.send(item=buffer) - - async def aclose(self) -> None: - await self._stream.aclose() - - async def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> AsyncNetworkStream: - exc_map = { - TimeoutError: ConnectTimeout, - anyio.BrokenResourceError: ConnectError, - anyio.EndOfStream: ConnectError, - ssl.SSLError: ConnectError, - } - with map_exceptions(exc_map): - try: - with anyio.fail_after(timeout): - ssl_stream = await anyio.streams.tls.TLSStream.wrap( - self._stream, - ssl_context=ssl_context, - hostname=server_hostname, - standard_compatible=False, - server_side=False, - ) - except Exception as exc: # pragma: nocover - await self.aclose() - raise exc - return AnyIOStream(ssl_stream) - - def get_extra_info(self, info: str) -> typing.Any: - if info == "ssl_object": - return self._stream.extra(anyio.streams.tls.TLSAttribute.ssl_object, None) - if info == "client_addr": - return self._stream.extra(anyio.abc.SocketAttribute.local_address, None) - if info == "server_addr": - return self._stream.extra(anyio.abc.SocketAttribute.remote_address, None) - if info == "socket": - return self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) - if info == "is_readable": - sock = self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) - return is_socket_readable(sock) - return None - - -class AnyIOBackend(AsyncNetworkBackend): - async def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: # pragma: nocover - if socket_options is None: - socket_options = [] - exc_map = { - TimeoutError: ConnectTimeout, - OSError: ConnectError, - anyio.BrokenResourceError: ConnectError, - } - with map_exceptions(exc_map): - with anyio.fail_after(timeout): - stream: anyio.abc.ByteStream = await anyio.connect_tcp( - remote_host=host, - remote_port=port, - local_host=local_address, - ) - # By default TCP sockets opened in `asyncio` include TCP_NODELAY. - for option in socket_options: - stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover - return AnyIOStream(stream) - - async def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: # pragma: nocover - if socket_options is None: - socket_options = [] - exc_map = { - TimeoutError: ConnectTimeout, - OSError: ConnectError, - anyio.BrokenResourceError: ConnectError, - } - with map_exceptions(exc_map): - with anyio.fail_after(timeout): - stream: anyio.abc.ByteStream = await anyio.connect_unix(path) - for option in socket_options: - stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover - return AnyIOStream(stream) - - async def sleep(self, seconds: float) -> None: - await anyio.sleep(seconds) # pragma: nocover diff --git a/contrib/python/httpcore/httpcore/_backends/auto.py b/contrib/python/httpcore/httpcore/_backends/auto.py deleted file mode 100644 index 49f0e698c97..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/auto.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import typing - -from .._synchronization import current_async_library -from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream - - -class AutoBackend(AsyncNetworkBackend): - async def _init_backend(self) -> None: - if not (hasattr(self, "_backend")): - backend = current_async_library() - if backend == "trio": - from .trio import TrioBackend - - self._backend: AsyncNetworkBackend = TrioBackend() - else: - from .anyio import AnyIOBackend - - self._backend = AnyIOBackend() - - async def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - await self._init_backend() - return await self._backend.connect_tcp( - host, - port, - timeout=timeout, - local_address=local_address, - socket_options=socket_options, - ) - - async def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: # pragma: nocover - await self._init_backend() - return await self._backend.connect_unix_socket( - path, timeout=timeout, socket_options=socket_options - ) - - async def sleep(self, seconds: float) -> None: # pragma: nocover - await self._init_backend() - return await self._backend.sleep(seconds) diff --git a/contrib/python/httpcore/httpcore/_backends/base.py b/contrib/python/httpcore/httpcore/_backends/base.py deleted file mode 100644 index cf55c8b10eb..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/base.py +++ /dev/null @@ -1,101 +0,0 @@ -from __future__ import annotations - -import ssl -import time -import typing - -SOCKET_OPTION = typing.Union[ - typing.Tuple[int, int, int], - typing.Tuple[int, int, typing.Union[bytes, bytearray]], - typing.Tuple[int, int, None, int], -] - - -class NetworkStream: - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - raise NotImplementedError() # pragma: nocover - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - raise NotImplementedError() # pragma: nocover - - def close(self) -> None: - raise NotImplementedError() # pragma: nocover - - def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> NetworkStream: - raise NotImplementedError() # pragma: nocover - - def get_extra_info(self, info: str) -> typing.Any: - return None # pragma: nocover - - -class NetworkBackend: - def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: - raise NotImplementedError() # pragma: nocover - - def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: - raise NotImplementedError() # pragma: nocover - - def sleep(self, seconds: float) -> None: - time.sleep(seconds) # pragma: nocover - - -class AsyncNetworkStream: - async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - raise NotImplementedError() # pragma: nocover - - async def write(self, buffer: bytes, timeout: float | None = None) -> None: - raise NotImplementedError() # pragma: nocover - - async def aclose(self) -> None: - raise NotImplementedError() # pragma: nocover - - async def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> AsyncNetworkStream: - raise NotImplementedError() # pragma: nocover - - def get_extra_info(self, info: str) -> typing.Any: - return None # pragma: nocover - - -class AsyncNetworkBackend: - async def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - raise NotImplementedError() # pragma: nocover - - async def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - raise NotImplementedError() # pragma: nocover - - async def sleep(self, seconds: float) -> None: - raise NotImplementedError() # pragma: nocover diff --git a/contrib/python/httpcore/httpcore/_backends/mock.py b/contrib/python/httpcore/httpcore/_backends/mock.py deleted file mode 100644 index 9b6edca03d4..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/mock.py +++ /dev/null @@ -1,143 +0,0 @@ -from __future__ import annotations - -import ssl -import typing - -from .._exceptions import ReadError -from .base import ( - SOCKET_OPTION, - AsyncNetworkBackend, - AsyncNetworkStream, - NetworkBackend, - NetworkStream, -) - - -class MockSSLObject: - def __init__(self, http2: bool): - self._http2 = http2 - - def selected_alpn_protocol(self) -> str: - return "h2" if self._http2 else "http/1.1" - - -class MockStream(NetworkStream): - def __init__(self, buffer: list[bytes], http2: bool = False) -> None: - self._buffer = buffer - self._http2 = http2 - self._closed = False - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - if self._closed: - raise ReadError("Connection closed") - if not self._buffer: - return b"" - return self._buffer.pop(0) - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - pass - - def close(self) -> None: - self._closed = True - - def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> NetworkStream: - return self - - def get_extra_info(self, info: str) -> typing.Any: - return MockSSLObject(http2=self._http2) if info == "ssl_object" else None - - def __repr__(self) -> str: - return "<httpcore.MockStream>" - - -class MockBackend(NetworkBackend): - def __init__(self, buffer: list[bytes], http2: bool = False) -> None: - self._buffer = buffer - self._http2 = http2 - - def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: - return MockStream(list(self._buffer), http2=self._http2) - - def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: - return MockStream(list(self._buffer), http2=self._http2) - - def sleep(self, seconds: float) -> None: - pass - - -class AsyncMockStream(AsyncNetworkStream): - def __init__(self, buffer: list[bytes], http2: bool = False) -> None: - self._buffer = buffer - self._http2 = http2 - self._closed = False - - async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - if self._closed: - raise ReadError("Connection closed") - if not self._buffer: - return b"" - return self._buffer.pop(0) - - async def write(self, buffer: bytes, timeout: float | None = None) -> None: - pass - - async def aclose(self) -> None: - self._closed = True - - async def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> AsyncNetworkStream: - return self - - def get_extra_info(self, info: str) -> typing.Any: - return MockSSLObject(http2=self._http2) if info == "ssl_object" else None - - def __repr__(self) -> str: - return "<httpcore.AsyncMockStream>" - - -class AsyncMockBackend(AsyncNetworkBackend): - def __init__(self, buffer: list[bytes], http2: bool = False) -> None: - self._buffer = buffer - self._http2 = http2 - - async def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - return AsyncMockStream(list(self._buffer), http2=self._http2) - - async def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - return AsyncMockStream(list(self._buffer), http2=self._http2) - - async def sleep(self, seconds: float) -> None: - pass diff --git a/contrib/python/httpcore/httpcore/_backends/sync.py b/contrib/python/httpcore/httpcore/_backends/sync.py deleted file mode 100644 index 4018a09c6fb..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/sync.py +++ /dev/null @@ -1,241 +0,0 @@ -from __future__ import annotations - -import functools -import socket -import ssl -import sys -import typing - -from .._exceptions import ( - ConnectError, - ConnectTimeout, - ExceptionMapping, - ReadError, - ReadTimeout, - WriteError, - WriteTimeout, - map_exceptions, -) -from .._utils import is_socket_readable -from .base import SOCKET_OPTION, NetworkBackend, NetworkStream - - -class TLSinTLSStream(NetworkStream): # pragma: no cover - """ - Because the standard `SSLContext.wrap_socket` method does - not work for `SSLSocket` objects, we need this class - to implement TLS stream using an underlying `SSLObject` - instance in order to support TLS on top of TLS. - """ - - # Defined in RFC 8449 - TLS_RECORD_SIZE = 16384 - - def __init__( - self, - sock: socket.socket, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ): - self._sock = sock - self._incoming = ssl.MemoryBIO() - self._outgoing = ssl.MemoryBIO() - - self.ssl_obj = ssl_context.wrap_bio( - incoming=self._incoming, - outgoing=self._outgoing, - server_hostname=server_hostname, - ) - - self._sock.settimeout(timeout) - self._perform_io(self.ssl_obj.do_handshake) - - def _perform_io( - self, - func: typing.Callable[..., typing.Any], - ) -> typing.Any: - ret = None - - while True: - errno = None - try: - ret = func() - except (ssl.SSLWantReadError, ssl.SSLWantWriteError) as e: - errno = e.errno - - self._sock.sendall(self._outgoing.read()) - - if errno == ssl.SSL_ERROR_WANT_READ: - buf = self._sock.recv(self.TLS_RECORD_SIZE) - - if buf: - self._incoming.write(buf) - else: - self._incoming.write_eof() - if errno is None: - return ret - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} - with map_exceptions(exc_map): - self._sock.settimeout(timeout) - return typing.cast( - bytes, self._perform_io(functools.partial(self.ssl_obj.read, max_bytes)) - ) - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} - with map_exceptions(exc_map): - self._sock.settimeout(timeout) - while buffer: - nsent = self._perform_io(functools.partial(self.ssl_obj.write, buffer)) - buffer = buffer[nsent:] - - def close(self) -> None: - self._sock.close() - - def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> NetworkStream: - raise NotImplementedError() - - def get_extra_info(self, info: str) -> typing.Any: - if info == "ssl_object": - return self.ssl_obj - if info == "client_addr": - return self._sock.getsockname() - if info == "server_addr": - return self._sock.getpeername() - if info == "socket": - return self._sock - if info == "is_readable": - return is_socket_readable(self._sock) - return None - - -class SyncStream(NetworkStream): - def __init__(self, sock: socket.socket) -> None: - self._sock = sock - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} - with map_exceptions(exc_map): - self._sock.settimeout(timeout) - return self._sock.recv(max_bytes) - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - if not buffer: - return - - exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} - with map_exceptions(exc_map): - while buffer: - self._sock.settimeout(timeout) - n = self._sock.send(buffer) - buffer = buffer[n:] - - def close(self) -> None: - self._sock.close() - - def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> NetworkStream: - exc_map: ExceptionMapping = { - socket.timeout: ConnectTimeout, - OSError: ConnectError, - } - with map_exceptions(exc_map): - try: - if isinstance(self._sock, ssl.SSLSocket): # pragma: no cover - # If the underlying socket has already been upgraded - # to the TLS layer (i.e. is an instance of SSLSocket), - # we need some additional smarts to support TLS-in-TLS. - return TLSinTLSStream( - self._sock, ssl_context, server_hostname, timeout - ) - else: - self._sock.settimeout(timeout) - sock = ssl_context.wrap_socket( - self._sock, server_hostname=server_hostname - ) - except Exception as exc: # pragma: nocover - self.close() - raise exc - return SyncStream(sock) - - def get_extra_info(self, info: str) -> typing.Any: - if info == "ssl_object" and isinstance(self._sock, ssl.SSLSocket): - return self._sock._sslobj # type: ignore - if info == "client_addr": - return self._sock.getsockname() - if info == "server_addr": - return self._sock.getpeername() - if info == "socket": - return self._sock - if info == "is_readable": - return is_socket_readable(self._sock) - return None - - -class SyncBackend(NetworkBackend): - def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: - # Note that we automatically include `TCP_NODELAY` - # in addition to any other custom socket options. - if socket_options is None: - socket_options = [] # pragma: no cover - address = (host, port) - source_address = None if local_address is None else (local_address, 0) - exc_map: ExceptionMapping = { - socket.timeout: ConnectTimeout, - OSError: ConnectError, - } - - with map_exceptions(exc_map): - sock = socket.create_connection( - address, - timeout, - source_address=source_address, - ) - for option in socket_options: - sock.setsockopt(*option) # pragma: no cover - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return SyncStream(sock) - - def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> NetworkStream: # pragma: nocover - if sys.platform == "win32": - raise RuntimeError( - "Attempted to connect to a UNIX socket on a Windows system." - ) - if socket_options is None: - socket_options = [] - - exc_map: ExceptionMapping = { - socket.timeout: ConnectTimeout, - OSError: ConnectError, - } - with map_exceptions(exc_map): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - for option in socket_options: - sock.setsockopt(*option) - sock.settimeout(timeout) - sock.connect(path) - return SyncStream(sock) diff --git a/contrib/python/httpcore/httpcore/_backends/trio.py b/contrib/python/httpcore/httpcore/_backends/trio.py deleted file mode 100644 index 6f53f5f2a02..00000000000 --- a/contrib/python/httpcore/httpcore/_backends/trio.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import ssl -import typing - -import trio - -from .._exceptions import ( - ConnectError, - ConnectTimeout, - ExceptionMapping, - ReadError, - ReadTimeout, - WriteError, - WriteTimeout, - map_exceptions, -) -from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream - - -class TrioStream(AsyncNetworkStream): - def __init__(self, stream: trio.abc.Stream) -> None: - self._stream = stream - - async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - timeout_or_inf = float("inf") if timeout is None else timeout - exc_map: ExceptionMapping = { - trio.TooSlowError: ReadTimeout, - trio.BrokenResourceError: ReadError, - trio.ClosedResourceError: ReadError, - } - with map_exceptions(exc_map): - with trio.fail_after(timeout_or_inf): - data: bytes = await self._stream.receive_some(max_bytes=max_bytes) - return data - - async def write(self, buffer: bytes, timeout: float | None = None) -> None: - if not buffer: - return - - timeout_or_inf = float("inf") if timeout is None else timeout - exc_map: ExceptionMapping = { - trio.TooSlowError: WriteTimeout, - trio.BrokenResourceError: WriteError, - trio.ClosedResourceError: WriteError, - } - with map_exceptions(exc_map): - with trio.fail_after(timeout_or_inf): - await self._stream.send_all(data=buffer) - - async def aclose(self) -> None: - await self._stream.aclose() - - async def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> AsyncNetworkStream: - timeout_or_inf = float("inf") if timeout is None else timeout - exc_map: ExceptionMapping = { - trio.TooSlowError: ConnectTimeout, - trio.BrokenResourceError: ConnectError, - } - ssl_stream = trio.SSLStream( - self._stream, - ssl_context=ssl_context, - server_hostname=server_hostname, - https_compatible=True, - server_side=False, - ) - with map_exceptions(exc_map): - try: - with trio.fail_after(timeout_or_inf): - await ssl_stream.do_handshake() - except Exception as exc: # pragma: nocover - await self.aclose() - raise exc - return TrioStream(ssl_stream) - - def get_extra_info(self, info: str) -> typing.Any: - if info == "ssl_object" and isinstance(self._stream, trio.SSLStream): - # Type checkers cannot see `_ssl_object` attribute because trio._ssl.SSLStream uses __getattr__/__setattr__. - # Tracked at https://github.com/python-trio/trio/issues/542 - return self._stream._ssl_object # type: ignore[attr-defined] - if info == "client_addr": - return self._get_socket_stream().socket.getsockname() - if info == "server_addr": - return self._get_socket_stream().socket.getpeername() - if info == "socket": - stream = self._stream - while isinstance(stream, trio.SSLStream): - stream = stream.transport_stream - assert isinstance(stream, trio.SocketStream) - return stream.socket - if info == "is_readable": - socket = self.get_extra_info("socket") - return socket.is_readable() - return None - - def _get_socket_stream(self) -> trio.SocketStream: - stream = self._stream - while isinstance(stream, trio.SSLStream): - stream = stream.transport_stream - assert isinstance(stream, trio.SocketStream) - return stream - - -class TrioBackend(AsyncNetworkBackend): - async def connect_tcp( - self, - host: str, - port: int, - timeout: float | None = None, - local_address: str | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: - # By default for TCP sockets, trio enables TCP_NODELAY. - # https://trio.readthedocs.io/en/stable/reference-io.html#trio.SocketStream - if socket_options is None: - socket_options = [] # pragma: no cover - timeout_or_inf = float("inf") if timeout is None else timeout - exc_map: ExceptionMapping = { - trio.TooSlowError: ConnectTimeout, - trio.BrokenResourceError: ConnectError, - OSError: ConnectError, - } - with map_exceptions(exc_map): - with trio.fail_after(timeout_or_inf): - stream: trio.abc.Stream = await trio.open_tcp_stream( - host=host, port=port, local_address=local_address - ) - for option in socket_options: - stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover - return TrioStream(stream) - - async def connect_unix_socket( - self, - path: str, - timeout: float | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> AsyncNetworkStream: # pragma: nocover - if socket_options is None: - socket_options = [] - timeout_or_inf = float("inf") if timeout is None else timeout - exc_map: ExceptionMapping = { - trio.TooSlowError: ConnectTimeout, - trio.BrokenResourceError: ConnectError, - OSError: ConnectError, - } - with map_exceptions(exc_map): - with trio.fail_after(timeout_or_inf): - stream: trio.abc.Stream = await trio.open_unix_socket(path) - for option in socket_options: - stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover - return TrioStream(stream) - - async def sleep(self, seconds: float) -> None: - await trio.sleep(seconds) # pragma: nocover diff --git a/contrib/python/httpcore/httpcore/_exceptions.py b/contrib/python/httpcore/httpcore/_exceptions.py deleted file mode 100644 index bc28d44f55b..00000000000 --- a/contrib/python/httpcore/httpcore/_exceptions.py +++ /dev/null @@ -1,81 +0,0 @@ -import contextlib -import typing - -ExceptionMapping = typing.Mapping[typing.Type[Exception], typing.Type[Exception]] - - -def map_exceptions(map: ExceptionMapping) -> typing.Iterator[None]: - try: - yield - except Exception as exc: # noqa: PIE786 - for from_exc, to_exc in map.items(): - if isinstance(exc, from_exc): - raise to_exc(exc) from exc - raise # pragma: nocover - - -class ConnectionNotAvailable(Exception): - pass - - -class ProxyError(Exception): - pass - - -class UnsupportedProtocol(Exception): - pass - - -class ProtocolError(Exception): - pass - - -class RemoteProtocolError(ProtocolError): - pass - - -class LocalProtocolError(ProtocolError): - pass - - -# Timeout errors - - -class TimeoutException(Exception): - pass - - -class PoolTimeout(TimeoutException): - pass - - -class ConnectTimeout(TimeoutException): - pass - - -class ReadTimeout(TimeoutException): - pass - - -class WriteTimeout(TimeoutException): - pass - - -# Network errors - - -class NetworkError(Exception): - pass - - -class ConnectError(NetworkError): - pass - - -class ReadError(NetworkError): - pass - - -class WriteError(NetworkError): - pass diff --git a/contrib/python/httpcore/httpcore/_models.py b/contrib/python/httpcore/httpcore/_models.py deleted file mode 100644 index 8a65f13347d..00000000000 --- a/contrib/python/httpcore/httpcore/_models.py +++ /dev/null @@ -1,516 +0,0 @@ -from __future__ import annotations - -import base64 -import ssl -import typing -import urllib.parse - -# Functions for typechecking... - - -ByteOrStr = typing.Union[bytes, str] -HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] -HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] -HeaderTypes = typing.Union[HeadersAsSequence, HeadersAsMapping, None] - -Extensions = typing.MutableMapping[str, typing.Any] - - -def enforce_bytes(value: bytes | str, *, name: str) -> bytes: - """ - Any arguments that are ultimately represented as bytes can be specified - either as bytes or as strings. - - However we enforce that any string arguments must only contain characters in - the plain ASCII range. chr(0)...chr(127). If you need to use characters - outside that range then be precise, and use a byte-wise argument. - """ - if isinstance(value, str): - try: - return value.encode("ascii") - except UnicodeEncodeError: - raise TypeError(f"{name} strings may not include unicode characters.") - elif isinstance(value, bytes): - return value - - seen_type = type(value).__name__ - raise TypeError(f"{name} must be bytes or str, but got {seen_type}.") - - -def enforce_url(value: URL | bytes | str, *, name: str) -> URL: - """ - Type check for URL parameters. - """ - if isinstance(value, (bytes, str)): - return URL(value) - elif isinstance(value, URL): - return value - - seen_type = type(value).__name__ - raise TypeError(f"{name} must be a URL, bytes, or str, but got {seen_type}.") - - -def enforce_headers( - value: HeadersAsMapping | HeadersAsSequence | None = None, *, name: str -) -> list[tuple[bytes, bytes]]: - """ - Convienence function that ensure all items in request or response headers - are either bytes or strings in the plain ASCII range. - """ - if value is None: - return [] - elif isinstance(value, typing.Mapping): - return [ - ( - enforce_bytes(k, name="header name"), - enforce_bytes(v, name="header value"), - ) - for k, v in value.items() - ] - elif isinstance(value, typing.Sequence): - return [ - ( - enforce_bytes(k, name="header name"), - enforce_bytes(v, name="header value"), - ) - for k, v in value - ] - - seen_type = type(value).__name__ - raise TypeError( - f"{name} must be a mapping or sequence of two-tuples, but got {seen_type}." - ) - - -def enforce_stream( - value: bytes | typing.Iterable[bytes] | typing.AsyncIterable[bytes] | None, - *, - name: str, -) -> typing.Iterable[bytes] | typing.AsyncIterable[bytes]: - if value is None: - return ByteStream(b"") - elif isinstance(value, bytes): - return ByteStream(value) - return value - - -# * https://tools.ietf.org/html/rfc3986#section-3.2.3 -# * https://url.spec.whatwg.org/#url-miscellaneous -# * https://url.spec.whatwg.org/#scheme-state -DEFAULT_PORTS = { - b"ftp": 21, - b"http": 80, - b"https": 443, - b"ws": 80, - b"wss": 443, -} - - -def include_request_headers( - headers: list[tuple[bytes, bytes]], - *, - url: "URL", - content: None | bytes | typing.Iterable[bytes] | typing.AsyncIterable[bytes], -) -> list[tuple[bytes, bytes]]: - headers_set = set(k.lower() for k, v in headers) - - if b"host" not in headers_set: - default_port = DEFAULT_PORTS.get(url.scheme) - if url.port is None or url.port == default_port: - header_value = url.host - else: - header_value = b"%b:%d" % (url.host, url.port) - headers = [(b"Host", header_value)] + headers - - if ( - content is not None - and b"content-length" not in headers_set - and b"transfer-encoding" not in headers_set - ): - if isinstance(content, bytes): - content_length = str(len(content)).encode("ascii") - headers += [(b"Content-Length", content_length)] - else: - headers += [(b"Transfer-Encoding", b"chunked")] # pragma: nocover - - return headers - - -# Interfaces for byte streams... - - -class ByteStream: - """ - A container for non-streaming content, and that supports both sync and async - stream iteration. - """ - - def __init__(self, content: bytes) -> None: - self._content = content - - def __iter__(self) -> typing.Iterator[bytes]: - yield self._content - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - yield self._content - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{len(self._content)} bytes]>" - - -class Origin: - def __init__(self, scheme: bytes, host: bytes, port: int) -> None: - self.scheme = scheme - self.host = host - self.port = port - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, Origin) - and self.scheme == other.scheme - and self.host == other.host - and self.port == other.port - ) - - def __str__(self) -> str: - scheme = self.scheme.decode("ascii") - host = self.host.decode("ascii") - port = str(self.port) - return f"{scheme}://{host}:{port}" - - -class URL: - """ - Represents the URL against which an HTTP request may be made. - - The URL may either be specified as a plain string, for convienence: - - ```python - url = httpcore.URL("https://www.example.com/") - ``` - - Or be constructed with explicitily pre-parsed components: - - ```python - url = httpcore.URL(scheme=b'https', host=b'www.example.com', port=None, target=b'/') - ``` - - Using this second more explicit style allows integrations that are using - `httpcore` to pass through URLs that have already been parsed in order to use - libraries such as `rfc-3986` rather than relying on the stdlib. It also ensures - that URL parsing is treated identically at both the networking level and at any - higher layers of abstraction. - - The four components are important here, as they allow the URL to be precisely - specified in a pre-parsed format. They also allow certain types of request to - be created that could not otherwise be expressed. - - For example, an HTTP request to `http://www.example.com/` forwarded via a proxy - at `http://localhost:8080`... - - ```python - # Constructs an HTTP request with a complete URL as the target: - # GET https://www.example.com/ HTTP/1.1 - url = httpcore.URL( - scheme=b'http', - host=b'localhost', - port=8080, - target=b'https://www.example.com/' - ) - request = httpcore.Request( - method="GET", - url=url - ) - ``` - - Another example is constructing an `OPTIONS *` request... - - ```python - # Constructs an 'OPTIONS *' HTTP request: - # OPTIONS * HTTP/1.1 - url = httpcore.URL(scheme=b'https', host=b'www.example.com', target=b'*') - request = httpcore.Request(method="OPTIONS", url=url) - ``` - - This kind of request is not possible to formulate with a URL string, - because the `/` delimiter is always used to demark the target from the - host/port portion of the URL. - - For convenience, string-like arguments may be specified either as strings or - as bytes. However, once a request is being issue over-the-wire, the URL - components are always ultimately required to be a bytewise representation. - - In order to avoid any ambiguity over character encodings, when strings are used - as arguments, they must be strictly limited to the ASCII range `chr(0)`-`chr(127)`. - If you require a bytewise representation that is outside this range you must - handle the character encoding directly, and pass a bytes instance. - """ - - def __init__( - self, - url: bytes | str = "", - *, - scheme: bytes | str = b"", - host: bytes | str = b"", - port: int | None = None, - target: bytes | str = b"", - ) -> None: - """ - Parameters: - url: The complete URL as a string or bytes. - scheme: The URL scheme as a string or bytes. - Typically either `"http"` or `"https"`. - host: The URL host as a string or bytes. Such as `"www.example.com"`. - port: The port to connect to. Either an integer or `None`. - target: The target of the HTTP request. Such as `"/items?search=red"`. - """ - if url: - parsed = urllib.parse.urlparse(enforce_bytes(url, name="url")) - self.scheme = parsed.scheme - self.host = parsed.hostname or b"" - self.port = parsed.port - self.target = (parsed.path or b"/") + ( - b"?" + parsed.query if parsed.query else b"" - ) - else: - self.scheme = enforce_bytes(scheme, name="scheme") - self.host = enforce_bytes(host, name="host") - self.port = port - self.target = enforce_bytes(target, name="target") - - @property - def origin(self) -> Origin: - default_port = { - b"http": 80, - b"https": 443, - b"ws": 80, - b"wss": 443, - b"socks5": 1080, - b"socks5h": 1080, - }[self.scheme] - return Origin( - scheme=self.scheme, host=self.host, port=self.port or default_port - ) - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, URL) - and other.scheme == self.scheme - and other.host == self.host - and other.port == self.port - and other.target == self.target - ) - - def __bytes__(self) -> bytes: - if self.port is None: - return b"%b://%b%b" % (self.scheme, self.host, self.target) - return b"%b://%b:%d%b" % (self.scheme, self.host, self.port, self.target) - - def __repr__(self) -> str: - return ( - f"{self.__class__.__name__}(scheme={self.scheme!r}, " - f"host={self.host!r}, port={self.port!r}, target={self.target!r})" - ) - - -class Request: - """ - An HTTP request. - """ - - def __init__( - self, - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes - | typing.Iterable[bytes] - | typing.AsyncIterable[bytes] - | None = None, - extensions: Extensions | None = None, - ) -> None: - """ - Parameters: - method: The HTTP request method, either as a string or bytes. - For example: `GET`. - url: The request URL, either as a `URL` instance, or as a string or bytes. - For example: `"https://www.example.com".` - headers: The HTTP request headers. - content: The content of the request body. - extensions: A dictionary of optional extra information included on - the request. Possible keys include `"timeout"`, and `"trace"`. - """ - self.method: bytes = enforce_bytes(method, name="method") - self.url: URL = enforce_url(url, name="url") - self.headers: list[tuple[bytes, bytes]] = enforce_headers( - headers, name="headers" - ) - self.stream: typing.Iterable[bytes] | typing.AsyncIterable[bytes] = ( - enforce_stream(content, name="content") - ) - self.extensions = {} if extensions is None else extensions - - if "target" in self.extensions: - self.url = URL( - scheme=self.url.scheme, - host=self.url.host, - port=self.url.port, - target=self.extensions["target"], - ) - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.method!r}]>" - - -class Response: - """ - An HTTP response. - """ - - def __init__( - self, - status: int, - *, - headers: HeaderTypes = None, - content: bytes - | typing.Iterable[bytes] - | typing.AsyncIterable[bytes] - | None = None, - extensions: Extensions | None = None, - ) -> None: - """ - Parameters: - status: The HTTP status code of the response. For example `200`. - headers: The HTTP response headers. - content: The content of the response body. - extensions: A dictionary of optional extra information included on - the responseself.Possible keys include `"http_version"`, - `"reason_phrase"`, and `"network_stream"`. - """ - self.status: int = status - self.headers: list[tuple[bytes, bytes]] = enforce_headers( - headers, name="headers" - ) - self.stream: typing.Iterable[bytes] | typing.AsyncIterable[bytes] = ( - enforce_stream(content, name="content") - ) - self.extensions = {} if extensions is None else extensions - - self._stream_consumed = False - - @property - def content(self) -> bytes: - if not hasattr(self, "_content"): - if isinstance(self.stream, typing.Iterable): - raise RuntimeError( - "Attempted to access 'response.content' on a streaming response. " - "Call 'response.read()' first." - ) - else: - raise RuntimeError( - "Attempted to access 'response.content' on a streaming response. " - "Call 'await response.aread()' first." - ) - return self._content - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.status}]>" - - # Sync interface... - - def read(self) -> bytes: - if not isinstance(self.stream, typing.Iterable): # pragma: nocover - raise RuntimeError( - "Attempted to read an asynchronous response using 'response.read()'. " - "You should use 'await response.aread()' instead." - ) - if not hasattr(self, "_content"): - self._content = b"".join([part for part in self.iter_stream()]) - return self._content - - def iter_stream(self) -> typing.Iterator[bytes]: - if not isinstance(self.stream, typing.Iterable): # pragma: nocover - raise RuntimeError( - "Attempted to stream an asynchronous response using 'for ... in " - "response.iter_stream()'. " - "You should use 'async for ... in response.aiter_stream()' instead." - ) - if self._stream_consumed: - raise RuntimeError( - "Attempted to call 'for ... in response.iter_stream()' more than once." - ) - self._stream_consumed = True - for chunk in self.stream: - yield chunk - - def close(self) -> None: - if not isinstance(self.stream, typing.Iterable): # pragma: nocover - raise RuntimeError( - "Attempted to close an asynchronous response using 'response.close()'. " - "You should use 'await response.aclose()' instead." - ) - if hasattr(self.stream, "close"): - self.stream.close() - - # Async interface... - - async def aread(self) -> bytes: - if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover - raise RuntimeError( - "Attempted to read an synchronous response using " - "'await response.aread()'. " - "You should use 'response.read()' instead." - ) - if not hasattr(self, "_content"): - self._content = b"".join([part async for part in self.aiter_stream()]) - return self._content - - async def aiter_stream(self) -> typing.AsyncIterator[bytes]: - if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover - raise RuntimeError( - "Attempted to stream an synchronous response using 'async for ... in " - "response.aiter_stream()'. " - "You should use 'for ... in response.iter_stream()' instead." - ) - if self._stream_consumed: - raise RuntimeError( - "Attempted to call 'async for ... in response.aiter_stream()' " - "more than once." - ) - self._stream_consumed = True - async for chunk in self.stream: - yield chunk - - async def aclose(self) -> None: - if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover - raise RuntimeError( - "Attempted to close a synchronous response using " - "'await response.aclose()'. " - "You should use 'response.close()' instead." - ) - if hasattr(self.stream, "aclose"): - await self.stream.aclose() - - -class Proxy: - def __init__( - self, - url: URL | bytes | str, - auth: tuple[bytes | str, bytes | str] | None = None, - headers: HeadersAsMapping | HeadersAsSequence | None = None, - ssl_context: ssl.SSLContext | None = None, - ): - self.url = enforce_url(url, name="url") - self.headers = enforce_headers(headers, name="headers") - self.ssl_context = ssl_context - - if auth is not None: - username = enforce_bytes(auth[0], name="auth") - password = enforce_bytes(auth[1], name="auth") - userpass = username + b":" + password - authorization = b"Basic " + base64.b64encode(userpass) - self.auth: tuple[bytes, bytes] | None = (username, password) - self.headers = [(b"Proxy-Authorization", authorization)] + self.headers - else: - self.auth = None diff --git a/contrib/python/httpcore/httpcore/_ssl.py b/contrib/python/httpcore/httpcore/_ssl.py deleted file mode 100644 index c99c5a67945..00000000000 --- a/contrib/python/httpcore/httpcore/_ssl.py +++ /dev/null @@ -1,9 +0,0 @@ -import ssl - -import certifi - - -def default_ssl_context() -> ssl.SSLContext: - context = ssl.create_default_context() - context.load_verify_locations(certifi.where()) - return context diff --git a/contrib/python/httpcore/httpcore/_sync/__init__.py b/contrib/python/httpcore/httpcore/_sync/__init__.py deleted file mode 100644 index b476d76d9a7..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from .connection import HTTPConnection -from .connection_pool import ConnectionPool -from .http11 import HTTP11Connection -from .http_proxy import HTTPProxy -from .interfaces import ConnectionInterface - -try: - from .http2 import HTTP2Connection -except ImportError: # pragma: nocover - - class HTTP2Connection: # type: ignore - def __init__(self, *args, **kwargs) -> None: # type: ignore - raise RuntimeError( - "Attempted to use http2 support, but the `h2` package is not " - "installed. Use 'pip install httpcore[http2]'." - ) - - -try: - from .socks_proxy import SOCKSProxy -except ImportError: # pragma: nocover - - class SOCKSProxy: # type: ignore - def __init__(self, *args, **kwargs) -> None: # type: ignore - raise RuntimeError( - "Attempted to use SOCKS support, but the `socksio` package is not " - "installed. Use 'pip install httpcore[socks]'." - ) - - -__all__ = [ - "HTTPConnection", - "ConnectionPool", - "HTTPProxy", - "HTTP11Connection", - "HTTP2Connection", - "ConnectionInterface", - "SOCKSProxy", -] diff --git a/contrib/python/httpcore/httpcore/_sync/connection.py b/contrib/python/httpcore/httpcore/_sync/connection.py deleted file mode 100644 index 363f8be819d..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/connection.py +++ /dev/null @@ -1,222 +0,0 @@ -from __future__ import annotations - -import itertools -import logging -import ssl -import types -import typing - -from .._backends.sync import SyncBackend -from .._backends.base import SOCKET_OPTION, NetworkBackend, NetworkStream -from .._exceptions import ConnectError, ConnectTimeout -from .._models import Origin, Request, Response -from .._ssl import default_ssl_context -from .._synchronization import Lock -from .._trace import Trace -from .http11 import HTTP11Connection -from .interfaces import ConnectionInterface - -RETRIES_BACKOFF_FACTOR = 0.5 # 0s, 0.5s, 1s, 2s, 4s, etc. - - -logger = logging.getLogger("httpcore.connection") - - -def exponential_backoff(factor: float) -> typing.Iterator[float]: - """ - Generate a geometric sequence that has a ratio of 2 and starts with 0. - - For example: - - `factor = 2`: `0, 2, 4, 8, 16, 32, 64, ...` - - `factor = 3`: `0, 3, 6, 12, 24, 48, 96, ...` - """ - yield 0 - for n in itertools.count(): - yield factor * 2**n - - -class HTTPConnection(ConnectionInterface): - def __init__( - self, - origin: Origin, - ssl_context: ssl.SSLContext | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: NetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - self._origin = origin - self._ssl_context = ssl_context - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._retries = retries - self._local_address = local_address - self._uds = uds - - self._network_backend: NetworkBackend = ( - SyncBackend() if network_backend is None else network_backend - ) - self._connection: ConnectionInterface | None = None - self._connect_failed: bool = False - self._request_lock = Lock() - self._socket_options = socket_options - - def handle_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection to {self._origin}" - ) - - try: - with self._request_lock: - if self._connection is None: - stream = self._connect(request) - - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - if http2_negotiated or (self._http2 and not self._http1): - from .http2 import HTTP2Connection - - self._connection = HTTP2Connection( - origin=self._origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = HTTP11Connection( - origin=self._origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - except BaseException as exc: - self._connect_failed = True - raise exc - - return self._connection.handle_request(request) - - def _connect(self, request: Request) -> NetworkStream: - timeouts = request.extensions.get("timeout", {}) - sni_hostname = request.extensions.get("sni_hostname", None) - timeout = timeouts.get("connect", None) - - retries_left = self._retries - delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR) - - while True: - try: - if self._uds is None: - kwargs = { - "host": self._origin.host.decode("ascii"), - "port": self._origin.port, - "local_address": self._local_address, - "timeout": timeout, - "socket_options": self._socket_options, - } - with Trace("connect_tcp", logger, request, kwargs) as trace: - stream = self._network_backend.connect_tcp(**kwargs) - trace.return_value = stream - else: - kwargs = { - "path": self._uds, - "timeout": timeout, - "socket_options": self._socket_options, - } - with Trace( - "connect_unix_socket", logger, request, kwargs - ) as trace: - stream = self._network_backend.connect_unix_socket( - **kwargs - ) - trace.return_value = stream - - if self._origin.scheme in (b"https", b"wss"): - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": sni_hostname - or self._origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - return stream - except (ConnectError, ConnectTimeout): - if retries_left <= 0: - raise - retries_left -= 1 - delay = next(delays) - with Trace("retry", logger, request, kwargs) as trace: - self._network_backend.sleep(delay) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - def close(self) -> None: - if self._connection is not None: - with Trace("close", logger, None, {}): - self._connection.close() - - def is_available(self) -> bool: - if self._connection is None: - # If HTTP/2 support is enabled, and the resulting connection could - # end up as HTTP/2 then we should indicate the connection as being - # available to service multiple requests. - return ( - self._http2 - and (self._origin.scheme == b"https" or not self._http1) - and not self._connect_failed - ) - return self._connection.is_available() - - def has_expired(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.has_expired() - - def is_idle(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.is_idle() - - def is_closed(self) -> bool: - if self._connection is None: - return self._connect_failed - return self._connection.is_closed() - - def info(self) -> str: - if self._connection is None: - return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" - return self._connection.info() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - def __enter__(self) -> HTTPConnection: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self.close() diff --git a/contrib/python/httpcore/httpcore/_sync/connection_pool.py b/contrib/python/httpcore/httpcore/_sync/connection_pool.py deleted file mode 100644 index 9ccfa53e597..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/connection_pool.py +++ /dev/null @@ -1,420 +0,0 @@ -from __future__ import annotations - -import ssl -import sys -import types -import typing - -from .._backends.sync import SyncBackend -from .._backends.base import SOCKET_OPTION, NetworkBackend -from .._exceptions import ConnectionNotAvailable, UnsupportedProtocol -from .._models import Origin, Proxy, Request, Response -from .._synchronization import Event, ShieldCancellation, ThreadLock -from .connection import HTTPConnection -from .interfaces import ConnectionInterface, RequestInterface - - -class PoolRequest: - def __init__(self, request: Request) -> None: - self.request = request - self.connection: ConnectionInterface | None = None - self._connection_acquired = Event() - - def assign_to_connection(self, connection: ConnectionInterface | None) -> None: - self.connection = connection - self._connection_acquired.set() - - def clear_connection(self) -> None: - self.connection = None - self._connection_acquired = Event() - - def wait_for_connection( - self, timeout: float | None = None - ) -> ConnectionInterface: - if self.connection is None: - self._connection_acquired.wait(timeout=timeout) - assert self.connection is not None - return self.connection - - def is_queued(self) -> bool: - return self.connection is None - - -class ConnectionPool(RequestInterface): - """ - A connection pool for making HTTP requests. - """ - - def __init__( - self, - ssl_context: ssl.SSLContext | None = None, - proxy: Proxy | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: NetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish a - connection. - local_address: Local address to connect from. Can also be used to connect - using a particular address family. Using `local_address="0.0.0.0"` - will connect using an `AF_INET` address (IPv4), while using - `local_address="::"` will connect using an `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - socket_options: Socket options that have to be included - in the TCP socket when the connection was established. - """ - self._ssl_context = ssl_context - self._proxy = proxy - self._max_connections = ( - sys.maxsize if max_connections is None else max_connections - ) - self._max_keepalive_connections = ( - sys.maxsize - if max_keepalive_connections is None - else max_keepalive_connections - ) - self._max_keepalive_connections = min( - self._max_connections, self._max_keepalive_connections - ) - - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._retries = retries - self._local_address = local_address - self._uds = uds - - self._network_backend = ( - SyncBackend() if network_backend is None else network_backend - ) - self._socket_options = socket_options - - # The mutable state on a connection pool is the queue of incoming requests, - # and the set of connections that are servicing those requests. - self._connections: list[ConnectionInterface] = [] - self._requests: list[PoolRequest] = [] - - # We only mutate the state of the connection pool within an 'optional_thread_lock' - # context. This holds a threading lock unless we're running in async mode, - # in which case it is a no-op. - self._optional_thread_lock = ThreadLock() - - def create_connection(self, origin: Origin) -> ConnectionInterface: - if self._proxy is not None: - if self._proxy.url.scheme in (b"socks5", b"socks5h"): - from .socks_proxy import Socks5Connection - - return Socks5Connection( - proxy_origin=self._proxy.url.origin, - proxy_auth=self._proxy.auth, - remote_origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - elif origin.scheme == b"http": - from .http_proxy import ForwardHTTPConnection - - return ForwardHTTPConnection( - proxy_origin=self._proxy.url.origin, - proxy_headers=self._proxy.headers, - proxy_ssl_context=self._proxy.ssl_context, - remote_origin=origin, - keepalive_expiry=self._keepalive_expiry, - network_backend=self._network_backend, - ) - from .http_proxy import TunnelHTTPConnection - - return TunnelHTTPConnection( - proxy_origin=self._proxy.url.origin, - proxy_headers=self._proxy.headers, - proxy_ssl_context=self._proxy.ssl_context, - remote_origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - return HTTPConnection( - origin=origin, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - retries=self._retries, - local_address=self._local_address, - uds=self._uds, - network_backend=self._network_backend, - socket_options=self._socket_options, - ) - - @property - def connections(self) -> list[ConnectionInterface]: - """ - Return a list of the connections currently in the pool. - - For example: - - ```python - >>> pool.connections - [ - <HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 6]>, - <HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 9]> , - <HTTPConnection ['http://example.com:80', HTTP/1.1, IDLE, Request Count: 1]>, - ] - ``` - """ - return list(self._connections) - - def handle_request(self, request: Request) -> Response: - """ - Send an HTTP request, and return an HTTP response. - - This is the core implementation that is called into by `.request()` or `.stream()`. - """ - scheme = request.url.scheme.decode() - if scheme == "": - raise UnsupportedProtocol( - "Request URL is missing an 'http://' or 'https://' protocol." - ) - if scheme not in ("http", "https", "ws", "wss"): - raise UnsupportedProtocol( - f"Request URL has an unsupported protocol '{scheme}://'." - ) - - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("pool", None) - - with self._optional_thread_lock: - # Add the incoming request to our request queue. - pool_request = PoolRequest(request) - self._requests.append(pool_request) - - try: - while True: - with self._optional_thread_lock: - # Assign incoming requests to available connections, - # closing or creating new connections as required. - closing = self._assign_requests_to_connections() - self._close_connections(closing) - - # Wait until this request has an assigned connection. - connection = pool_request.wait_for_connection(timeout=timeout) - - try: - # Send the request on the assigned connection. - response = connection.handle_request( - pool_request.request - ) - except ConnectionNotAvailable: - # In some cases a connection may initially be available to - # handle a request, but then become unavailable. - # - # In this case we clear the connection and try again. - pool_request.clear_connection() - else: - break # pragma: nocover - - except BaseException as exc: - with self._optional_thread_lock: - # For any exception or cancellation we remove the request from - # the queue, and then re-assign requests to connections. - self._requests.remove(pool_request) - closing = self._assign_requests_to_connections() - - self._close_connections(closing) - raise exc from None - - # Return the response. Note that in this case we still have to manage - # the point at which the response is closed. - assert isinstance(response.stream, typing.Iterable) - return Response( - status=response.status, - headers=response.headers, - content=PoolByteStream( - stream=response.stream, pool_request=pool_request, pool=self - ), - extensions=response.extensions, - ) - - def _assign_requests_to_connections(self) -> list[ConnectionInterface]: - """ - Manage the state of the connection pool, assigning incoming - requests to connections as available. - - Called whenever a new request is added or removed from the pool. - - Any closing connections are returned, allowing the I/O for closing - those connections to be handled seperately. - """ - closing_connections = [] - - # First we handle cleaning up any connections that are closed, - # have expired their keep-alive, or surplus idle connections. - for connection in list(self._connections): - if connection.is_closed(): - # log: "removing closed connection" - self._connections.remove(connection) - elif connection.has_expired(): - # log: "closing expired connection" - self._connections.remove(connection) - closing_connections.append(connection) - elif ( - connection.is_idle() - and len([connection.is_idle() for connection in self._connections]) - > self._max_keepalive_connections - ): - # log: "closing idle connection" - self._connections.remove(connection) - closing_connections.append(connection) - - # Assign queued requests to connections. - queued_requests = [request for request in self._requests if request.is_queued()] - for pool_request in queued_requests: - origin = pool_request.request.url.origin - available_connections = [ - connection - for connection in self._connections - if connection.can_handle_request(origin) and connection.is_available() - ] - idle_connections = [ - connection for connection in self._connections if connection.is_idle() - ] - - # There are three cases for how we may be able to handle the request: - # - # 1. There is an existing connection that can handle the request. - # 2. We can create a new connection to handle the request. - # 3. We can close an idle connection and then create a new connection - # to handle the request. - if available_connections: - # log: "reusing existing connection" - connection = available_connections[0] - pool_request.assign_to_connection(connection) - elif len(self._connections) < self._max_connections: - # log: "creating new connection" - connection = self.create_connection(origin) - self._connections.append(connection) - pool_request.assign_to_connection(connection) - elif idle_connections: - # log: "closing idle connection" - connection = idle_connections[0] - self._connections.remove(connection) - closing_connections.append(connection) - # log: "creating new connection" - connection = self.create_connection(origin) - self._connections.append(connection) - pool_request.assign_to_connection(connection) - - return closing_connections - - def _close_connections(self, closing: list[ConnectionInterface]) -> None: - # Close connections which have been removed from the pool. - with ShieldCancellation(): - for connection in closing: - connection.close() - - def close(self) -> None: - # Explicitly close the connection pool. - # Clears all existing requests and connections. - with self._optional_thread_lock: - closing_connections = list(self._connections) - self._connections = [] - self._close_connections(closing_connections) - - def __enter__(self) -> ConnectionPool: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self.close() - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - with self._optional_thread_lock: - request_is_queued = [request.is_queued() for request in self._requests] - connection_is_idle = [ - connection.is_idle() for connection in self._connections - ] - - num_active_requests = request_is_queued.count(False) - num_queued_requests = request_is_queued.count(True) - num_active_connections = connection_is_idle.count(False) - num_idle_connections = connection_is_idle.count(True) - - requests_info = ( - f"Requests: {num_active_requests} active, {num_queued_requests} queued" - ) - connection_info = ( - f"Connections: {num_active_connections} active, {num_idle_connections} idle" - ) - - return f"<{class_name} [{requests_info} | {connection_info}]>" - - -class PoolByteStream: - def __init__( - self, - stream: typing.Iterable[bytes], - pool_request: PoolRequest, - pool: ConnectionPool, - ) -> None: - self._stream = stream - self._pool_request = pool_request - self._pool = pool - self._closed = False - - def __iter__(self) -> typing.Iterator[bytes]: - try: - for part in self._stream: - yield part - except BaseException as exc: - self.close() - raise exc from None - - def close(self) -> None: - if not self._closed: - self._closed = True - with ShieldCancellation(): - if hasattr(self._stream, "close"): - self._stream.close() - - with self._pool._optional_thread_lock: - self._pool._requests.remove(self._pool_request) - closing = self._pool._assign_requests_to_connections() - - self._pool._close_connections(closing) diff --git a/contrib/python/httpcore/httpcore/_sync/http11.py b/contrib/python/httpcore/httpcore/_sync/http11.py deleted file mode 100644 index ebd3a97480c..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/http11.py +++ /dev/null @@ -1,379 +0,0 @@ -from __future__ import annotations - -import enum -import logging -import ssl -import time -import types -import typing - -import h11 - -from .._backends.base import NetworkStream -from .._exceptions import ( - ConnectionNotAvailable, - LocalProtocolError, - RemoteProtocolError, - WriteError, - map_exceptions, -) -from .._models import Origin, Request, Response -from .._synchronization import Lock, ShieldCancellation -from .._trace import Trace -from .interfaces import ConnectionInterface - -logger = logging.getLogger("httpcore.http11") - - -# A subset of `h11.Event` types supported by `_send_event` -H11SendEvent = typing.Union[ - h11.Request, - h11.Data, - h11.EndOfMessage, -] - - -class HTTPConnectionState(enum.IntEnum): - NEW = 0 - ACTIVE = 1 - IDLE = 2 - CLOSED = 3 - - -class HTTP11Connection(ConnectionInterface): - READ_NUM_BYTES = 64 * 1024 - MAX_INCOMPLETE_EVENT_SIZE = 100 * 1024 - - def __init__( - self, - origin: Origin, - stream: NetworkStream, - keepalive_expiry: float | None = None, - ) -> None: - self._origin = origin - self._network_stream = stream - self._keepalive_expiry: float | None = keepalive_expiry - self._expire_at: float | None = None - self._state = HTTPConnectionState.NEW - self._state_lock = Lock() - self._request_count = 0 - self._h11_state = h11.Connection( - our_role=h11.CLIENT, - max_incomplete_event_size=self.MAX_INCOMPLETE_EVENT_SIZE, - ) - - def handle_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection " - f"to {self._origin}" - ) - - with self._state_lock: - if self._state in (HTTPConnectionState.NEW, HTTPConnectionState.IDLE): - self._request_count += 1 - self._state = HTTPConnectionState.ACTIVE - self._expire_at = None - else: - raise ConnectionNotAvailable() - - try: - kwargs = {"request": request} - try: - with Trace( - "send_request_headers", logger, request, kwargs - ) as trace: - self._send_request_headers(**kwargs) - with Trace("send_request_body", logger, request, kwargs) as trace: - self._send_request_body(**kwargs) - except WriteError: - # If we get a write error while we're writing the request, - # then we supress this error and move on to attempting to - # read the response. Servers can sometimes close the request - # pre-emptively and then respond with a well formed HTTP - # error response. - pass - - with Trace( - "receive_response_headers", logger, request, kwargs - ) as trace: - ( - http_version, - status, - reason_phrase, - headers, - trailing_data, - ) = self._receive_response_headers(**kwargs) - trace.return_value = ( - http_version, - status, - reason_phrase, - headers, - ) - - network_stream = self._network_stream - - # CONNECT or Upgrade request - if (status == 101) or ( - (request.method == b"CONNECT") and (200 <= status < 300) - ): - network_stream = HTTP11UpgradeStream(network_stream, trailing_data) - - return Response( - status=status, - headers=headers, - content=HTTP11ConnectionByteStream(self, request), - extensions={ - "http_version": http_version, - "reason_phrase": reason_phrase, - "network_stream": network_stream, - }, - ) - except BaseException as exc: - with ShieldCancellation(): - with Trace("response_closed", logger, request) as trace: - self._response_closed() - raise exc - - # Sending the request... - - def _send_request_headers(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - with map_exceptions({h11.LocalProtocolError: LocalProtocolError}): - event = h11.Request( - method=request.method, - target=request.url.target, - headers=request.headers, - ) - self._send_event(event, timeout=timeout) - - def _send_request_body(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - assert isinstance(request.stream, typing.Iterable) - for chunk in request.stream: - event = h11.Data(data=chunk) - self._send_event(event, timeout=timeout) - - self._send_event(h11.EndOfMessage(), timeout=timeout) - - def _send_event(self, event: h11.Event, timeout: float | None = None) -> None: - bytes_to_send = self._h11_state.send(event) - if bytes_to_send is not None: - self._network_stream.write(bytes_to_send, timeout=timeout) - - # Receiving the response... - - def _receive_response_headers( - self, request: Request - ) -> tuple[bytes, int, bytes, list[tuple[bytes, bytes]], bytes]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - while True: - event = self._receive_event(timeout=timeout) - if isinstance(event, h11.Response): - break - if ( - isinstance(event, h11.InformationalResponse) - and event.status_code == 101 - ): - break - - http_version = b"HTTP/" + event.http_version - - # h11 version 0.11+ supports a `raw_items` interface to get the - # raw header casing, rather than the enforced lowercase headers. - headers = event.headers.raw_items() - - trailing_data, _ = self._h11_state.trailing_data - - return http_version, event.status_code, event.reason, headers, trailing_data - - def _receive_response_body( - self, request: Request - ) -> typing.Iterator[bytes]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - while True: - event = self._receive_event(timeout=timeout) - if isinstance(event, h11.Data): - yield bytes(event.data) - elif isinstance(event, (h11.EndOfMessage, h11.PAUSED)): - break - - def _receive_event( - self, timeout: float | None = None - ) -> h11.Event | type[h11.PAUSED]: - while True: - with map_exceptions({h11.RemoteProtocolError: RemoteProtocolError}): - event = self._h11_state.next_event() - - if event is h11.NEED_DATA: - data = self._network_stream.read( - self.READ_NUM_BYTES, timeout=timeout - ) - - # If we feed this case through h11 we'll raise an exception like: - # - # httpcore.RemoteProtocolError: can't handle event type - # ConnectionClosed when role=SERVER and state=SEND_RESPONSE - # - # Which is accurate, but not very informative from an end-user - # perspective. Instead we handle this case distinctly and treat - # it as a ConnectError. - if data == b"" and self._h11_state.their_state == h11.SEND_RESPONSE: - msg = "Server disconnected without sending a response." - raise RemoteProtocolError(msg) - - self._h11_state.receive_data(data) - else: - # mypy fails to narrow the type in the above if statement above - return event # type: ignore[return-value] - - def _response_closed(self) -> None: - with self._state_lock: - if ( - self._h11_state.our_state is h11.DONE - and self._h11_state.their_state is h11.DONE - ): - self._state = HTTPConnectionState.IDLE - self._h11_state.start_next_cycle() - if self._keepalive_expiry is not None: - now = time.monotonic() - self._expire_at = now + self._keepalive_expiry - else: - self.close() - - # Once the connection is no longer required... - - def close(self) -> None: - # Note that this method unilaterally closes the connection, and does - # not have any kind of locking in place around it. - self._state = HTTPConnectionState.CLOSED - self._network_stream.close() - - # The ConnectionInterface methods provide information about the state of - # the connection, allowing for a connection pooling implementation to - # determine when to reuse and when to close the connection... - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - def is_available(self) -> bool: - # Note that HTTP/1.1 connections in the "NEW" state are not treated as - # being "available". The control flow which created the connection will - # be able to send an outgoing request, but the connection will not be - # acquired from the connection pool for any other request. - return self._state == HTTPConnectionState.IDLE - - def has_expired(self) -> bool: - now = time.monotonic() - keepalive_expired = self._expire_at is not None and now > self._expire_at - - # If the HTTP connection is idle but the socket is readable, then the - # only valid state is that the socket is about to return b"", indicating - # a server-initiated disconnect. - server_disconnected = ( - self._state == HTTPConnectionState.IDLE - and self._network_stream.get_extra_info("is_readable") - ) - - return keepalive_expired or server_disconnected - - def is_idle(self) -> bool: - return self._state == HTTPConnectionState.IDLE - - def is_closed(self) -> bool: - return self._state == HTTPConnectionState.CLOSED - - def info(self) -> str: - origin = str(self._origin) - return ( - f"{origin!r}, HTTP/1.1, {self._state.name}, " - f"Request Count: {self._request_count}" - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - origin = str(self._origin) - return ( - f"<{class_name} [{origin!r}, {self._state.name}, " - f"Request Count: {self._request_count}]>" - ) - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - def __enter__(self) -> HTTP11Connection: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self.close() - - -class HTTP11ConnectionByteStream: - def __init__(self, connection: HTTP11Connection, request: Request) -> None: - self._connection = connection - self._request = request - self._closed = False - - def __iter__(self) -> typing.Iterator[bytes]: - kwargs = {"request": self._request} - try: - with Trace("receive_response_body", logger, self._request, kwargs): - for chunk in self._connection._receive_response_body(**kwargs): - yield chunk - except BaseException as exc: - # If we get an exception while streaming the response, - # we want to close the response (and possibly the connection) - # before raising that exception. - with ShieldCancellation(): - self.close() - raise exc - - def close(self) -> None: - if not self._closed: - self._closed = True - with Trace("response_closed", logger, self._request): - self._connection._response_closed() - - -class HTTP11UpgradeStream(NetworkStream): - def __init__(self, stream: NetworkStream, leading_data: bytes) -> None: - self._stream = stream - self._leading_data = leading_data - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - if self._leading_data: - buffer = self._leading_data[:max_bytes] - self._leading_data = self._leading_data[max_bytes:] - return buffer - else: - return self._stream.read(max_bytes, timeout) - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - self._stream.write(buffer, timeout) - - def close(self) -> None: - self._stream.close() - - def start_tls( - self, - ssl_context: ssl.SSLContext, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> NetworkStream: - return self._stream.start_tls(ssl_context, server_hostname, timeout) - - def get_extra_info(self, info: str) -> typing.Any: - return self._stream.get_extra_info(info) diff --git a/contrib/python/httpcore/httpcore/_sync/http2.py b/contrib/python/httpcore/httpcore/_sync/http2.py deleted file mode 100644 index ca4dd724325..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/http2.py +++ /dev/null @@ -1,583 +0,0 @@ -from __future__ import annotations - -import enum -import logging -import time -import types -import typing - -import h2.config -import h2.connection -import h2.events -import h2.exceptions -import h2.settings - -from .._backends.base import NetworkStream -from .._exceptions import ( - ConnectionNotAvailable, - LocalProtocolError, - RemoteProtocolError, -) -from .._models import Origin, Request, Response -from .._synchronization import Lock, Semaphore, ShieldCancellation -from .._trace import Trace -from .interfaces import ConnectionInterface - -logger = logging.getLogger("httpcore.http2") - - -def has_body_headers(request: Request) -> bool: - return any( - k.lower() == b"content-length" or k.lower() == b"transfer-encoding" - for k, v in request.headers - ) - - -class HTTPConnectionState(enum.IntEnum): - ACTIVE = 1 - IDLE = 2 - CLOSED = 3 - - -class HTTP2Connection(ConnectionInterface): - READ_NUM_BYTES = 64 * 1024 - CONFIG = h2.config.H2Configuration(validate_inbound_headers=False) - - def __init__( - self, - origin: Origin, - stream: NetworkStream, - keepalive_expiry: float | None = None, - ): - self._origin = origin - self._network_stream = stream - self._keepalive_expiry: float | None = keepalive_expiry - self._h2_state = h2.connection.H2Connection(config=self.CONFIG) - self._state = HTTPConnectionState.IDLE - self._expire_at: float | None = None - self._request_count = 0 - self._init_lock = Lock() - self._state_lock = Lock() - self._read_lock = Lock() - self._write_lock = Lock() - self._sent_connection_init = False - self._used_all_stream_ids = False - self._connection_error = False - - # Mapping from stream ID to response stream events. - self._events: dict[ - int, - h2.events.ResponseReceived - | h2.events.DataReceived - | h2.events.StreamEnded - | h2.events.StreamReset, - ] = {} - - # Connection terminated events are stored as state since - # we need to handle them for all streams. - self._connection_terminated: h2.events.ConnectionTerminated | None = None - - self._read_exception: Exception | None = None - self._write_exception: Exception | None = None - - def handle_request(self, request: Request) -> Response: - if not self.can_handle_request(request.url.origin): - # This cannot occur in normal operation, since the connection pool - # will only send requests on connections that handle them. - # It's in place simply for resilience as a guard against incorrect - # usage, for anyone working directly with httpcore connections. - raise RuntimeError( - f"Attempted to send request to {request.url.origin} on connection " - f"to {self._origin}" - ) - - with self._state_lock: - if self._state in (HTTPConnectionState.ACTIVE, HTTPConnectionState.IDLE): - self._request_count += 1 - self._expire_at = None - self._state = HTTPConnectionState.ACTIVE - else: - raise ConnectionNotAvailable() - - with self._init_lock: - if not self._sent_connection_init: - try: - kwargs = {"request": request} - with Trace("send_connection_init", logger, request, kwargs): - self._send_connection_init(**kwargs) - except BaseException as exc: - with ShieldCancellation(): - self.close() - raise exc - - self._sent_connection_init = True - - # Initially start with just 1 until the remote server provides - # its max_concurrent_streams value - self._max_streams = 1 - - local_settings_max_streams = ( - self._h2_state.local_settings.max_concurrent_streams - ) - self._max_streams_semaphore = Semaphore(local_settings_max_streams) - - for _ in range(local_settings_max_streams - self._max_streams): - self._max_streams_semaphore.acquire() - - self._max_streams_semaphore.acquire() - - try: - stream_id = self._h2_state.get_next_available_stream_id() - self._events[stream_id] = [] - except h2.exceptions.NoAvailableStreamIDError: # pragma: nocover - self._used_all_stream_ids = True - self._request_count -= 1 - raise ConnectionNotAvailable() - - try: - kwargs = {"request": request, "stream_id": stream_id} - with Trace("send_request_headers", logger, request, kwargs): - self._send_request_headers(request=request, stream_id=stream_id) - with Trace("send_request_body", logger, request, kwargs): - self._send_request_body(request=request, stream_id=stream_id) - with Trace( - "receive_response_headers", logger, request, kwargs - ) as trace: - status, headers = self._receive_response( - request=request, stream_id=stream_id - ) - trace.return_value = (status, headers) - - return Response( - status=status, - headers=headers, - content=HTTP2ConnectionByteStream(self, request, stream_id=stream_id), - extensions={ - "http_version": b"HTTP/2", - "network_stream": self._network_stream, - "stream_id": stream_id, - }, - ) - except BaseException as exc: # noqa: PIE786 - with ShieldCancellation(): - kwargs = {"stream_id": stream_id} - with Trace("response_closed", logger, request, kwargs): - self._response_closed(stream_id=stream_id) - - if isinstance(exc, h2.exceptions.ProtocolError): - # One case where h2 can raise a protocol error is when a - # closed frame has been seen by the state machine. - # - # This happens when one stream is reading, and encounters - # a GOAWAY event. Other flows of control may then raise - # a protocol error at any point they interact with the 'h2_state'. - # - # In this case we'll have stored the event, and should raise - # it as a RemoteProtocolError. - if self._connection_terminated: # pragma: nocover - raise RemoteProtocolError(self._connection_terminated) - # If h2 raises a protocol error in some other state then we - # must somehow have made a protocol violation. - raise LocalProtocolError(exc) # pragma: nocover - - raise exc - - def _send_connection_init(self, request: Request) -> None: - """ - The HTTP/2 connection requires some initial setup before we can start - using individual request/response streams on it. - """ - # Need to set these manually here instead of manipulating via - # __setitem__() otherwise the H2Connection will emit SettingsUpdate - # frames in addition to sending the undesired defaults. - self._h2_state.local_settings = h2.settings.Settings( - client=True, - initial_values={ - # Disable PUSH_PROMISE frames from the server since we don't do anything - # with them for now. Maybe when we support caching? - h2.settings.SettingCodes.ENABLE_PUSH: 0, - # These two are taken from h2 for safe defaults - h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 100, - h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: 65536, - }, - ) - - # Some websites (*cough* Yahoo *cough*) balk at this setting being - # present in the initial handshake since it's not defined in the original - # RFC despite the RFC mandating ignoring settings you don't know about. - del self._h2_state.local_settings[ - h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL - ] - - self._h2_state.initiate_connection() - self._h2_state.increment_flow_control_window(2**24) - self._write_outgoing_data(request) - - # Sending the request... - - def _send_request_headers(self, request: Request, stream_id: int) -> None: - """ - Send the request headers to a given stream ID. - """ - end_stream = not has_body_headers(request) - - # In HTTP/2 the ':authority' pseudo-header is used instead of 'Host'. - # In order to gracefully handle HTTP/1.1 and HTTP/2 we always require - # HTTP/1.1 style headers, and map them appropriately if we end up on - # an HTTP/2 connection. - authority = [v for k, v in request.headers if k.lower() == b"host"][0] - - headers = [ - (b":method", request.method), - (b":authority", authority), - (b":scheme", request.url.scheme), - (b":path", request.url.target), - ] + [ - (k.lower(), v) - for k, v in request.headers - if k.lower() - not in ( - b"host", - b"transfer-encoding", - ) - ] - - self._h2_state.send_headers(stream_id, headers, end_stream=end_stream) - self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id) - self._write_outgoing_data(request) - - def _send_request_body(self, request: Request, stream_id: int) -> None: - """ - Iterate over the request body sending it to a given stream ID. - """ - if not has_body_headers(request): - return - - assert isinstance(request.stream, typing.Iterable) - for data in request.stream: - self._send_stream_data(request, stream_id, data) - self._send_end_stream(request, stream_id) - - def _send_stream_data( - self, request: Request, stream_id: int, data: bytes - ) -> None: - """ - Send a single chunk of data in one or more data frames. - """ - while data: - max_flow = self._wait_for_outgoing_flow(request, stream_id) - chunk_size = min(len(data), max_flow) - chunk, data = data[:chunk_size], data[chunk_size:] - self._h2_state.send_data(stream_id, chunk) - self._write_outgoing_data(request) - - def _send_end_stream(self, request: Request, stream_id: int) -> None: - """ - Send an empty data frame on on a given stream ID with the END_STREAM flag set. - """ - self._h2_state.end_stream(stream_id) - self._write_outgoing_data(request) - - # Receiving the response... - - def _receive_response( - self, request: Request, stream_id: int - ) -> tuple[int, list[tuple[bytes, bytes]]]: - """ - Return the response status code and headers for a given stream ID. - """ - while True: - event = self._receive_stream_event(request, stream_id) - if isinstance(event, h2.events.ResponseReceived): - break - - status_code = 200 - headers = [] - for k, v in event.headers: - if k == b":status": - status_code = int(v.decode("ascii", errors="ignore")) - elif not k.startswith(b":"): - headers.append((k, v)) - - return (status_code, headers) - - def _receive_response_body( - self, request: Request, stream_id: int - ) -> typing.Iterator[bytes]: - """ - Iterator that returns the bytes of the response body for a given stream ID. - """ - while True: - event = self._receive_stream_event(request, stream_id) - if isinstance(event, h2.events.DataReceived): - amount = event.flow_controlled_length - self._h2_state.acknowledge_received_data(amount, stream_id) - self._write_outgoing_data(request) - yield event.data - elif isinstance(event, h2.events.StreamEnded): - break - - def _receive_stream_event( - self, request: Request, stream_id: int - ) -> h2.events.ResponseReceived | h2.events.DataReceived | h2.events.StreamEnded: - """ - Return the next available event for a given stream ID. - - Will read more data from the network if required. - """ - while not self._events.get(stream_id): - self._receive_events(request, stream_id) - event = self._events[stream_id].pop(0) - if isinstance(event, h2.events.StreamReset): - raise RemoteProtocolError(event) - return event - - def _receive_events( - self, request: Request, stream_id: int | None = None - ) -> None: - """ - Read some data from the network until we see one or more events - for a given stream ID. - """ - with self._read_lock: - if self._connection_terminated is not None: - last_stream_id = self._connection_terminated.last_stream_id - if stream_id and last_stream_id and stream_id > last_stream_id: - self._request_count -= 1 - raise ConnectionNotAvailable() - raise RemoteProtocolError(self._connection_terminated) - - # This conditional is a bit icky. We don't want to block reading if we've - # actually got an event to return for a given stream. We need to do that - # check *within* the atomic read lock. Though it also need to be optional, - # because when we call it from `_wait_for_outgoing_flow` we *do* want to - # block until we've available flow control, event when we have events - # pending for the stream ID we're attempting to send on. - if stream_id is None or not self._events.get(stream_id): - events = self._read_incoming_data(request) - for event in events: - if isinstance(event, h2.events.RemoteSettingsChanged): - with Trace( - "receive_remote_settings", logger, request - ) as trace: - self._receive_remote_settings_change(event) - trace.return_value = event - - elif isinstance( - event, - ( - h2.events.ResponseReceived, - h2.events.DataReceived, - h2.events.StreamEnded, - h2.events.StreamReset, - ), - ): - if event.stream_id in self._events: - self._events[event.stream_id].append(event) - - elif isinstance(event, h2.events.ConnectionTerminated): - self._connection_terminated = event - - self._write_outgoing_data(request) - - def _receive_remote_settings_change(self, event: h2.events.Event) -> None: - max_concurrent_streams = event.changed_settings.get( - h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS - ) - if max_concurrent_streams: - new_max_streams = min( - max_concurrent_streams.new_value, - self._h2_state.local_settings.max_concurrent_streams, - ) - if new_max_streams and new_max_streams != self._max_streams: - while new_max_streams > self._max_streams: - self._max_streams_semaphore.release() - self._max_streams += 1 - while new_max_streams < self._max_streams: - self._max_streams_semaphore.acquire() - self._max_streams -= 1 - - def _response_closed(self, stream_id: int) -> None: - self._max_streams_semaphore.release() - del self._events[stream_id] - with self._state_lock: - if self._connection_terminated and not self._events: - self.close() - - elif self._state == HTTPConnectionState.ACTIVE and not self._events: - self._state = HTTPConnectionState.IDLE - if self._keepalive_expiry is not None: - now = time.monotonic() - self._expire_at = now + self._keepalive_expiry - if self._used_all_stream_ids: # pragma: nocover - self.close() - - def close(self) -> None: - # Note that this method unilaterally closes the connection, and does - # not have any kind of locking in place around it. - self._h2_state.close_connection() - self._state = HTTPConnectionState.CLOSED - self._network_stream.close() - - # Wrappers around network read/write operations... - - def _read_incoming_data(self, request: Request) -> list[h2.events.Event]: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("read", None) - - if self._read_exception is not None: - raise self._read_exception # pragma: nocover - - try: - data = self._network_stream.read(self.READ_NUM_BYTES, timeout) - if data == b"": - raise RemoteProtocolError("Server disconnected") - except Exception as exc: - # If we get a network error we should: - # - # 1. Save the exception and just raise it immediately on any future reads. - # (For example, this means that a single read timeout or disconnect will - # immediately close all pending streams. Without requiring multiple - # sequential timeouts.) - # 2. Mark the connection as errored, so that we don't accept any other - # incoming requests. - self._read_exception = exc - self._connection_error = True - raise exc - - events: list[h2.events.Event] = self._h2_state.receive_data(data) - - return events - - def _write_outgoing_data(self, request: Request) -> None: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("write", None) - - with self._write_lock: - data_to_send = self._h2_state.data_to_send() - - if self._write_exception is not None: - raise self._write_exception # pragma: nocover - - try: - self._network_stream.write(data_to_send, timeout) - except Exception as exc: # pragma: nocover - # If we get a network error we should: - # - # 1. Save the exception and just raise it immediately on any future write. - # (For example, this means that a single write timeout or disconnect will - # immediately close all pending streams. Without requiring multiple - # sequential timeouts.) - # 2. Mark the connection as errored, so that we don't accept any other - # incoming requests. - self._write_exception = exc - self._connection_error = True - raise exc - - # Flow control... - - def _wait_for_outgoing_flow(self, request: Request, stream_id: int) -> int: - """ - Returns the maximum allowable outgoing flow for a given stream. - - If the allowable flow is zero, then waits on the network until - WindowUpdated frames have increased the flow rate. - https://tools.ietf.org/html/rfc7540#section-6.9 - """ - local_flow: int = self._h2_state.local_flow_control_window(stream_id) - max_frame_size: int = self._h2_state.max_outbound_frame_size - flow = min(local_flow, max_frame_size) - while flow == 0: - self._receive_events(request) - local_flow = self._h2_state.local_flow_control_window(stream_id) - max_frame_size = self._h2_state.max_outbound_frame_size - flow = min(local_flow, max_frame_size) - return flow - - # Interface for connection pooling... - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._origin - - def is_available(self) -> bool: - return ( - self._state != HTTPConnectionState.CLOSED - and not self._connection_error - and not self._used_all_stream_ids - and not ( - self._h2_state.state_machine.state - == h2.connection.ConnectionState.CLOSED - ) - ) - - def has_expired(self) -> bool: - now = time.monotonic() - return self._expire_at is not None and now > self._expire_at - - def is_idle(self) -> bool: - return self._state == HTTPConnectionState.IDLE - - def is_closed(self) -> bool: - return self._state == HTTPConnectionState.CLOSED - - def info(self) -> str: - origin = str(self._origin) - return ( - f"{origin!r}, HTTP/2, {self._state.name}, " - f"Request Count: {self._request_count}" - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - origin = str(self._origin) - return ( - f"<{class_name} [{origin!r}, {self._state.name}, " - f"Request Count: {self._request_count}]>" - ) - - # These context managers are not used in the standard flow, but are - # useful for testing or working with connection instances directly. - - def __enter__(self) -> HTTP2Connection: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self.close() - - -class HTTP2ConnectionByteStream: - def __init__( - self, connection: HTTP2Connection, request: Request, stream_id: int - ) -> None: - self._connection = connection - self._request = request - self._stream_id = stream_id - self._closed = False - - def __iter__(self) -> typing.Iterator[bytes]: - kwargs = {"request": self._request, "stream_id": self._stream_id} - try: - with Trace("receive_response_body", logger, self._request, kwargs): - for chunk in self._connection._receive_response_body( - request=self._request, stream_id=self._stream_id - ): - yield chunk - except BaseException as exc: - # If we get an exception while streaming the response, - # we want to close the response (and possibly the connection) - # before raising that exception. - with ShieldCancellation(): - self.close() - raise exc - - def close(self) -> None: - if not self._closed: - self._closed = True - kwargs = {"stream_id": self._stream_id} - with Trace("response_closed", logger, self._request, kwargs): - self._connection._response_closed(stream_id=self._stream_id) diff --git a/contrib/python/httpcore/httpcore/_sync/http_proxy.py b/contrib/python/httpcore/httpcore/_sync/http_proxy.py deleted file mode 100644 index ecca88f7dc9..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/http_proxy.py +++ /dev/null @@ -1,367 +0,0 @@ -from __future__ import annotations - -import base64 -import logging -import ssl -import typing - -from .._backends.base import SOCKET_OPTION, NetworkBackend -from .._exceptions import ProxyError -from .._models import ( - URL, - Origin, - Request, - Response, - enforce_bytes, - enforce_headers, - enforce_url, -) -from .._ssl import default_ssl_context -from .._synchronization import Lock -from .._trace import Trace -from .connection import HTTPConnection -from .connection_pool import ConnectionPool -from .http11 import HTTP11Connection -from .interfaces import ConnectionInterface - -ByteOrStr = typing.Union[bytes, str] -HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] -HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] - - -logger = logging.getLogger("httpcore.proxy") - - -def merge_headers( - default_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, - override_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, -) -> list[tuple[bytes, bytes]]: - """ - Append default_headers and override_headers, de-duplicating if a key exists - in both cases. - """ - default_headers = [] if default_headers is None else list(default_headers) - override_headers = [] if override_headers is None else list(override_headers) - has_override = set(key.lower() for key, value in override_headers) - default_headers = [ - (key, value) - for key, value in default_headers - if key.lower() not in has_override - ] - return default_headers + override_headers - - -class HTTPProxy(ConnectionPool): # pragma: nocover - """ - A connection pool that sends requests via an HTTP proxy. - """ - - def __init__( - self, - proxy_url: URL | bytes | str, - proxy_auth: tuple[bytes | str, bytes | str] | None = None, - proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, - ssl_context: ssl.SSLContext | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - local_address: str | None = None, - uds: str | None = None, - network_backend: NetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - proxy_url: The URL to use when connecting to the proxy server. - For example `"http://127.0.0.1:8080/"`. - proxy_auth: Any proxy authentication as a two-tuple of - (username, password). May be either bytes or ascii-only str. - proxy_headers: Any HTTP headers to use for the proxy requests. - For example `{"Proxy-Authorization": "Basic <username>:<password>"}`. - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish - a connection. - local_address: Local address to connect from. Can also be used to - connect using a particular address family. Using - `local_address="0.0.0.0"` will connect using an `AF_INET` address - (IPv4), while using `local_address="::"` will connect using an - `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - """ - super().__init__( - ssl_context=ssl_context, - max_connections=max_connections, - max_keepalive_connections=max_keepalive_connections, - keepalive_expiry=keepalive_expiry, - http1=http1, - http2=http2, - network_backend=network_backend, - retries=retries, - local_address=local_address, - uds=uds, - socket_options=socket_options, - ) - - self._proxy_url = enforce_url(proxy_url, name="proxy_url") - if ( - self._proxy_url.scheme == b"http" and proxy_ssl_context is not None - ): # pragma: no cover - raise RuntimeError( - "The `proxy_ssl_context` argument is not allowed for the http scheme" - ) - - self._ssl_context = ssl_context - self._proxy_ssl_context = proxy_ssl_context - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - if proxy_auth is not None: - username = enforce_bytes(proxy_auth[0], name="proxy_auth") - password = enforce_bytes(proxy_auth[1], name="proxy_auth") - userpass = username + b":" + password - authorization = b"Basic " + base64.b64encode(userpass) - self._proxy_headers = [ - (b"Proxy-Authorization", authorization) - ] + self._proxy_headers - - def create_connection(self, origin: Origin) -> ConnectionInterface: - if origin.scheme == b"http": - return ForwardHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - keepalive_expiry=self._keepalive_expiry, - network_backend=self._network_backend, - proxy_ssl_context=self._proxy_ssl_context, - ) - return TunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - -class ForwardHTTPConnection(ConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, - keepalive_expiry: float | None = None, - network_backend: NetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - ) -> None: - self._connection = HTTPConnection( - origin=proxy_origin, - keepalive_expiry=keepalive_expiry, - network_backend=network_backend, - socket_options=socket_options, - ssl_context=proxy_ssl_context, - ) - self._proxy_origin = proxy_origin - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - self._remote_origin = remote_origin - - def handle_request(self, request: Request) -> Response: - headers = merge_headers(self._proxy_headers, request.headers) - url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=bytes(request.url), - ) - proxy_request = Request( - method=request.method, - url=url, - headers=headers, - content=request.stream, - extensions=request.extensions, - ) - return self._connection.handle_request(proxy_request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - def close(self) -> None: - self._connection.close() - - def info(self) -> str: - return self._connection.info() - - def is_available(self) -> bool: - return self._connection.is_available() - - def has_expired(self) -> bool: - return self._connection.has_expired() - - def is_idle(self) -> bool: - return self._connection.is_idle() - - def is_closed(self) -> bool: - return self._connection.is_closed() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" - - -class TunnelHTTPConnection(ConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - ssl_context: ssl.SSLContext | None = None, - proxy_ssl_context: ssl.SSLContext | None = None, - proxy_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - network_backend: NetworkBackend | None = None, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - self._connection: ConnectionInterface = HTTPConnection( - origin=proxy_origin, - keepalive_expiry=keepalive_expiry, - network_backend=network_backend, - socket_options=socket_options, - ssl_context=proxy_ssl_context, - ) - self._proxy_origin = proxy_origin - self._remote_origin = remote_origin - self._ssl_context = ssl_context - self._proxy_ssl_context = proxy_ssl_context - self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - self._connect_lock = Lock() - self._connected = False - - def handle_request(self, request: Request) -> Response: - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = self._connection.handle_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - self._connection.close() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from .http2 import HTTP2Connection - - self._connection = HTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = HTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connected = True - return self._connection.handle_request(request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - def close(self) -> None: - self._connection.close() - - def info(self) -> str: - return self._connection.info() - - def is_available(self) -> bool: - return self._connection.is_available() - - def has_expired(self) -> bool: - return self._connection.has_expired() - - def is_idle(self) -> bool: - return self._connection.is_idle() - - def is_closed(self) -> bool: - return self._connection.is_closed() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/contrib/python/httpcore/httpcore/_sync/interfaces.py b/contrib/python/httpcore/httpcore/_sync/interfaces.py deleted file mode 100644 index e673d4cc1b1..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/interfaces.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import contextlib -import typing - -from .._models import ( - URL, - Extensions, - HeaderTypes, - Origin, - Request, - Response, - enforce_bytes, - enforce_headers, - enforce_url, - include_request_headers, -) - - -class RequestInterface: - def request( - self, - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.Iterator[bytes] | None = None, - extensions: Extensions | None = None, - ) -> Response: - # Strict type checking on our parameters. - method = enforce_bytes(method, name="method") - url = enforce_url(url, name="url") - headers = enforce_headers(headers, name="headers") - - # Include Host header, and optionally Content-Length or Transfer-Encoding. - headers = include_request_headers(headers, url=url, content=content) - - request = Request( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) - response = self.handle_request(request) - try: - response.read() - finally: - response.close() - return response - - @contextlib.contextmanager - def stream( - self, - method: bytes | str, - url: URL | bytes | str, - *, - headers: HeaderTypes = None, - content: bytes | typing.Iterator[bytes] | None = None, - extensions: Extensions | None = None, - ) -> typing.Iterator[Response]: - # Strict type checking on our parameters. - method = enforce_bytes(method, name="method") - url = enforce_url(url, name="url") - headers = enforce_headers(headers, name="headers") - - # Include Host header, and optionally Content-Length or Transfer-Encoding. - headers = include_request_headers(headers, url=url, content=content) - - request = Request( - method=method, - url=url, - headers=headers, - content=content, - extensions=extensions, - ) - response = self.handle_request(request) - try: - yield response - finally: - response.close() - - def handle_request(self, request: Request) -> Response: - raise NotImplementedError() # pragma: nocover - - -class ConnectionInterface(RequestInterface): - def close(self) -> None: - raise NotImplementedError() # pragma: nocover - - def info(self) -> str: - raise NotImplementedError() # pragma: nocover - - def can_handle_request(self, origin: Origin) -> bool: - raise NotImplementedError() # pragma: nocover - - def is_available(self) -> bool: - """ - Return `True` if the connection is currently able to accept an - outgoing request. - - An HTTP/1.1 connection will only be available if it is currently idle. - - An HTTP/2 connection will be available so long as the stream ID space is - not yet exhausted, and the connection is not in an error state. - - While the connection is being established we may not yet know if it is going - to result in an HTTP/1.1 or HTTP/2 connection. The connection should be - treated as being available, but might ultimately raise `NewConnectionRequired` - required exceptions if multiple requests are attempted over a connection - that ends up being established as HTTP/1.1. - """ - raise NotImplementedError() # pragma: nocover - - def has_expired(self) -> bool: - """ - Return `True` if the connection is in a state where it should be closed. - - This either means that the connection is idle and it has passed the - expiry time on its keep-alive, or that server has sent an EOF. - """ - raise NotImplementedError() # pragma: nocover - - def is_idle(self) -> bool: - """ - Return `True` if the connection is currently idle. - """ - raise NotImplementedError() # pragma: nocover - - def is_closed(self) -> bool: - """ - Return `True` if the connection has been closed. - - Used when a response is closed to determine if the connection may be - returned to the connection pool or not. - """ - raise NotImplementedError() # pragma: nocover diff --git a/contrib/python/httpcore/httpcore/_sync/socks_proxy.py b/contrib/python/httpcore/httpcore/_sync/socks_proxy.py deleted file mode 100644 index 0ca96ddfb58..00000000000 --- a/contrib/python/httpcore/httpcore/_sync/socks_proxy.py +++ /dev/null @@ -1,341 +0,0 @@ -from __future__ import annotations - -import logging -import ssl - -import socksio - -from .._backends.sync import SyncBackend -from .._backends.base import NetworkBackend, NetworkStream -from .._exceptions import ConnectionNotAvailable, ProxyError -from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url -from .._ssl import default_ssl_context -from .._synchronization import Lock -from .._trace import Trace -from .connection_pool import ConnectionPool -from .http11 import HTTP11Connection -from .interfaces import ConnectionInterface - -logger = logging.getLogger("httpcore.socks") - - -AUTH_METHODS = { - b"\x00": "NO AUTHENTICATION REQUIRED", - b"\x01": "GSSAPI", - b"\x02": "USERNAME/PASSWORD", - b"\xff": "NO ACCEPTABLE METHODS", -} - -REPLY_CODES = { - b"\x00": "Succeeded", - b"\x01": "General SOCKS server failure", - b"\x02": "Connection not allowed by ruleset", - b"\x03": "Network unreachable", - b"\x04": "Host unreachable", - b"\x05": "Connection refused", - b"\x06": "TTL expired", - b"\x07": "Command not supported", - b"\x08": "Address type not supported", -} - - -def _init_socks5_connection( - stream: NetworkStream, - *, - host: bytes, - port: int, - auth: tuple[bytes, bytes] | None = None, -) -> None: - conn = socksio.socks5.SOCKS5Connection() - - # Auth method request - auth_method = ( - socksio.socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED - if auth is None - else socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD - ) - conn.send(socksio.socks5.SOCKS5AuthMethodsRequest([auth_method])) - outgoing_bytes = conn.data_to_send() - stream.write(outgoing_bytes) - - # Auth method response - incoming_bytes = stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5AuthReply) - if response.method != auth_method: - requested = AUTH_METHODS.get(auth_method, "UNKNOWN") - responded = AUTH_METHODS.get(response.method, "UNKNOWN") - raise ProxyError( - f"Requested {requested} from proxy server, but got {responded}." - ) - - if response.method == socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: - # Username/password request - assert auth is not None - username, password = auth - conn.send(socksio.socks5.SOCKS5UsernamePasswordRequest(username, password)) - outgoing_bytes = conn.data_to_send() - stream.write(outgoing_bytes) - - # Username/password response - incoming_bytes = stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5UsernamePasswordReply) - if not response.success: - raise ProxyError("Invalid username/password") - - # Connect request - conn.send( - socksio.socks5.SOCKS5CommandRequest.from_address( - socksio.socks5.SOCKS5Command.CONNECT, (host, port) - ) - ) - outgoing_bytes = conn.data_to_send() - stream.write(outgoing_bytes) - - # Connect response - incoming_bytes = stream.read(max_bytes=4096) - response = conn.receive_data(incoming_bytes) - assert isinstance(response, socksio.socks5.SOCKS5Reply) - if response.reply_code != socksio.socks5.SOCKS5ReplyCode.SUCCEEDED: - reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") - raise ProxyError(f"Proxy Server could not connect: {reply_code}.") - - -class SOCKSProxy(ConnectionPool): # pragma: nocover - """ - A connection pool that sends requests via an HTTP proxy. - """ - - def __init__( - self, - proxy_url: URL | bytes | str, - proxy_auth: tuple[bytes | str, bytes | str] | None = None, - ssl_context: ssl.SSLContext | None = None, - max_connections: int | None = 10, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - retries: int = 0, - network_backend: NetworkBackend | None = None, - ) -> None: - """ - A connection pool for making HTTP requests. - - Parameters: - proxy_url: The URL to use when connecting to the proxy server. - For example `"http://127.0.0.1:8080/"`. - ssl_context: An SSL context to use for verifying connections. - If not specified, the default `httpcore.default_ssl_context()` - will be used. - max_connections: The maximum number of concurrent HTTP connections that - the pool should allow. Any attempt to send a request on a pool that - would exceed this amount will block until a connection is available. - max_keepalive_connections: The maximum number of idle HTTP connections - that will be maintained in the pool. - keepalive_expiry: The duration in seconds that an idle HTTP connection - may be maintained for before being expired from the pool. - http1: A boolean indicating if HTTP/1.1 requests should be supported - by the connection pool. Defaults to True. - http2: A boolean indicating if HTTP/2 requests should be supported by - the connection pool. Defaults to False. - retries: The maximum number of retries when trying to establish - a connection. - local_address: Local address to connect from. Can also be used to - connect using a particular address family. Using - `local_address="0.0.0.0"` will connect using an `AF_INET` address - (IPv4), while using `local_address="::"` will connect using an - `AF_INET6` address (IPv6). - uds: Path to a Unix Domain Socket to use instead of TCP sockets. - network_backend: A backend instance to use for handling network I/O. - """ - super().__init__( - ssl_context=ssl_context, - max_connections=max_connections, - max_keepalive_connections=max_keepalive_connections, - keepalive_expiry=keepalive_expiry, - http1=http1, - http2=http2, - network_backend=network_backend, - retries=retries, - ) - self._ssl_context = ssl_context - self._proxy_url = enforce_url(proxy_url, name="proxy_url") - if proxy_auth is not None: - username, password = proxy_auth - username_bytes = enforce_bytes(username, name="proxy_auth") - password_bytes = enforce_bytes(password, name="proxy_auth") - self._proxy_auth: tuple[bytes, bytes] | None = ( - username_bytes, - password_bytes, - ) - else: - self._proxy_auth = None - - def create_connection(self, origin: Origin) -> ConnectionInterface: - return Socks5Connection( - proxy_origin=self._proxy_url.origin, - remote_origin=origin, - proxy_auth=self._proxy_auth, - ssl_context=self._ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - - -class Socks5Connection(ConnectionInterface): - def __init__( - self, - proxy_origin: Origin, - remote_origin: Origin, - proxy_auth: tuple[bytes, bytes] | None = None, - ssl_context: ssl.SSLContext | None = None, - keepalive_expiry: float | None = None, - http1: bool = True, - http2: bool = False, - network_backend: NetworkBackend | None = None, - ) -> None: - self._proxy_origin = proxy_origin - self._remote_origin = remote_origin - self._proxy_auth = proxy_auth - self._ssl_context = ssl_context - self._keepalive_expiry = keepalive_expiry - self._http1 = http1 - self._http2 = http2 - - self._network_backend: NetworkBackend = ( - SyncBackend() if network_backend is None else network_backend - ) - self._connect_lock = Lock() - self._connection: ConnectionInterface | None = None - self._connect_failed = False - - def handle_request(self, request: Request) -> Response: - timeouts = request.extensions.get("timeout", {}) - sni_hostname = request.extensions.get("sni_hostname", None) - timeout = timeouts.get("connect", None) - - with self._connect_lock: - if self._connection is None: - try: - # Connect to the proxy - kwargs = { - "host": self._proxy_origin.host.decode("ascii"), - "port": self._proxy_origin.port, - "timeout": timeout, - } - with Trace("connect_tcp", logger, request, kwargs) as trace: - stream = self._network_backend.connect_tcp(**kwargs) - trace.return_value = stream - - # Connect to the remote host using socks5 - kwargs = { - "stream": stream, - "host": self._remote_origin.host.decode("ascii"), - "port": self._remote_origin.port, - "auth": self._proxy_auth, - } - with Trace( - "setup_socks5_connection", logger, request, kwargs - ) as trace: - _init_socks5_connection(**kwargs) - trace.return_value = stream - - # Upgrade the stream to SSL - if self._remote_origin.scheme == b"https": - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ( - ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ) - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": sni_hostname - or self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or ( - self._http2 and not self._http1 - ): # pragma: nocover - from .http2 import HTTP2Connection - - self._connection = HTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = HTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - except Exception as exc: - self._connect_failed = True - raise exc - elif not self._connection.is_available(): # pragma: nocover - raise ConnectionNotAvailable() - - return self._connection.handle_request(request) - - def can_handle_request(self, origin: Origin) -> bool: - return origin == self._remote_origin - - def close(self) -> None: - if self._connection is not None: - self._connection.close() - - def is_available(self) -> bool: - if self._connection is None: # pragma: nocover - # If HTTP/2 support is enabled, and the resulting connection could - # end up as HTTP/2 then we should indicate the connection as being - # available to service multiple requests. - return ( - self._http2 - and (self._remote_origin.scheme == b"https" or not self._http1) - and not self._connect_failed - ) - return self._connection.is_available() - - def has_expired(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.has_expired() - - def is_idle(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.is_idle() - - def is_closed(self) -> bool: - if self._connection is None: # pragma: nocover - return self._connect_failed - return self._connection.is_closed() - - def info(self) -> str: - if self._connection is None: # pragma: nocover - return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" - return self._connection.info() - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/contrib/python/httpcore/httpcore/_synchronization.py b/contrib/python/httpcore/httpcore/_synchronization.py deleted file mode 100644 index 2ecc9e9c363..00000000000 --- a/contrib/python/httpcore/httpcore/_synchronization.py +++ /dev/null @@ -1,318 +0,0 @@ -from __future__ import annotations - -import threading -import types - -from ._exceptions import ExceptionMapping, PoolTimeout, map_exceptions - -# Our async synchronization primatives use either 'anyio' or 'trio' depending -# on if they're running under asyncio or trio. - -try: - import trio -except (ImportError, NotImplementedError): # pragma: nocover - trio = None # type: ignore - -try: - import anyio -except ImportError: # pragma: nocover - anyio = None # type: ignore - - -def current_async_library() -> str: - # Determine if we're running under trio or asyncio. - # See https://sniffio.readthedocs.io/en/latest/ - try: - import sniffio - except ImportError: # pragma: nocover - environment = "asyncio" - else: - environment = sniffio.current_async_library() - - if environment not in ("asyncio", "trio"): # pragma: nocover - raise RuntimeError("Running under an unsupported async environment.") - - if environment == "asyncio" and anyio is None: # pragma: nocover - raise RuntimeError( - "Running with asyncio requires installation of 'httpcore[asyncio]'." - ) - - if environment == "trio" and trio is None: # pragma: nocover - raise RuntimeError( - "Running with trio requires installation of 'httpcore[trio]'." - ) - - return environment - - -class AsyncLock: - """ - This is a standard lock. - - In the sync case `Lock` provides thread locking. - In the async case `AsyncLock` provides async locking. - """ - - def __init__(self) -> None: - self._backend = "" - - def setup(self) -> None: - """ - Detect if we're running under 'asyncio' or 'trio' and create - a lock with the correct implementation. - """ - self._backend = current_async_library() - if self._backend == "trio": - self._trio_lock = trio.Lock() - elif self._backend == "asyncio": - self._anyio_lock = anyio.Lock() - - async def __aenter__(self) -> AsyncLock: - if not self._backend: - self.setup() - - if self._backend == "trio": - await self._trio_lock.acquire() - elif self._backend == "asyncio": - await self._anyio_lock.acquire() - - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - if self._backend == "trio": - self._trio_lock.release() - elif self._backend == "asyncio": - self._anyio_lock.release() - - -class AsyncThreadLock: - """ - This is a threading-only lock for no-I/O contexts. - - In the sync case `ThreadLock` provides thread locking. - In the async case `AsyncThreadLock` is a no-op. - """ - - def __enter__(self) -> AsyncThreadLock: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - pass - - -class AsyncEvent: - def __init__(self) -> None: - self._backend = "" - - def setup(self) -> None: - """ - Detect if we're running under 'asyncio' or 'trio' and create - a lock with the correct implementation. - """ - self._backend = current_async_library() - if self._backend == "trio": - self._trio_event = trio.Event() - elif self._backend == "asyncio": - self._anyio_event = anyio.Event() - - def set(self) -> None: - if not self._backend: - self.setup() - - if self._backend == "trio": - self._trio_event.set() - elif self._backend == "asyncio": - self._anyio_event.set() - - async def wait(self, timeout: float | None = None) -> None: - if not self._backend: - self.setup() - - if self._backend == "trio": - trio_exc_map: ExceptionMapping = {trio.TooSlowError: PoolTimeout} - timeout_or_inf = float("inf") if timeout is None else timeout - with map_exceptions(trio_exc_map): - with trio.fail_after(timeout_or_inf): - await self._trio_event.wait() - elif self._backend == "asyncio": - anyio_exc_map: ExceptionMapping = {TimeoutError: PoolTimeout} - with map_exceptions(anyio_exc_map): - with anyio.fail_after(timeout): - await self._anyio_event.wait() - - -class AsyncSemaphore: - def __init__(self, bound: int) -> None: - self._bound = bound - self._backend = "" - - def setup(self) -> None: - """ - Detect if we're running under 'asyncio' or 'trio' and create - a semaphore with the correct implementation. - """ - self._backend = current_async_library() - if self._backend == "trio": - self._trio_semaphore = trio.Semaphore( - initial_value=self._bound, max_value=self._bound - ) - elif self._backend == "asyncio": - self._anyio_semaphore = anyio.Semaphore( - initial_value=self._bound, max_value=self._bound - ) - - async def acquire(self) -> None: - if not self._backend: - self.setup() - - if self._backend == "trio": - await self._trio_semaphore.acquire() - elif self._backend == "asyncio": - await self._anyio_semaphore.acquire() - - async def release(self) -> None: - if self._backend == "trio": - self._trio_semaphore.release() - elif self._backend == "asyncio": - self._anyio_semaphore.release() - - -class AsyncShieldCancellation: - # For certain portions of our codebase where we're dealing with - # closing connections during exception handling we want to shield - # the operation from being cancelled. - # - # with AsyncShieldCancellation(): - # ... # clean-up operations, shielded from cancellation. - - def __init__(self) -> None: - """ - Detect if we're running under 'asyncio' or 'trio' and create - a shielded scope with the correct implementation. - """ - self._backend = current_async_library() - - if self._backend == "trio": - self._trio_shield = trio.CancelScope(shield=True) - elif self._backend == "asyncio": - self._anyio_shield = anyio.CancelScope(shield=True) - - def __enter__(self) -> AsyncShieldCancellation: - if self._backend == "trio": - self._trio_shield.__enter__() - elif self._backend == "asyncio": - self._anyio_shield.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - if self._backend == "trio": - self._trio_shield.__exit__(exc_type, exc_value, traceback) - elif self._backend == "asyncio": - self._anyio_shield.__exit__(exc_type, exc_value, traceback) - - -# Our thread-based synchronization primitives... - - -class Lock: - """ - This is a standard lock. - - In the sync case `Lock` provides thread locking. - In the async case `AsyncLock` provides async locking. - """ - - def __init__(self) -> None: - self._lock = threading.Lock() - - def __enter__(self) -> Lock: - self._lock.acquire() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self._lock.release() - - -class ThreadLock: - """ - This is a threading-only lock for no-I/O contexts. - - In the sync case `ThreadLock` provides thread locking. - In the async case `AsyncThreadLock` is a no-op. - """ - - def __init__(self) -> None: - self._lock = threading.Lock() - - def __enter__(self) -> ThreadLock: - self._lock.acquire() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - self._lock.release() - - -class Event: - def __init__(self) -> None: - self._event = threading.Event() - - def set(self) -> None: - self._event.set() - - def wait(self, timeout: float | None = None) -> None: - if timeout == float("inf"): # pragma: no cover - timeout = None - if not self._event.wait(timeout=timeout): - raise PoolTimeout() # pragma: nocover - - -class Semaphore: - def __init__(self, bound: int) -> None: - self._semaphore = threading.Semaphore(value=bound) - - def acquire(self) -> None: - self._semaphore.acquire() - - def release(self) -> None: - self._semaphore.release() - - -class ShieldCancellation: - # Thread-synchronous codebases don't support cancellation semantics. - # We have this class because we need to mirror the async and sync - # cases within our package, but it's just a no-op. - def __enter__(self) -> ShieldCancellation: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - pass diff --git a/contrib/python/httpcore/httpcore/_trace.py b/contrib/python/httpcore/httpcore/_trace.py deleted file mode 100644 index 5f1cd7c4782..00000000000 --- a/contrib/python/httpcore/httpcore/_trace.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import annotations - -import inspect -import logging -import types -import typing - -from ._models import Request - - -class Trace: - def __init__( - self, - name: str, - logger: logging.Logger, - request: Request | None = None, - kwargs: dict[str, typing.Any] | None = None, - ) -> None: - self.name = name - self.logger = logger - self.trace_extension = ( - None if request is None else request.extensions.get("trace") - ) - self.debug = self.logger.isEnabledFor(logging.DEBUG) - self.kwargs = kwargs or {} - self.return_value: typing.Any = None - self.should_trace = self.debug or self.trace_extension is not None - self.prefix = self.logger.name.split(".")[-1] - - def trace(self, name: str, info: dict[str, typing.Any]) -> None: - if self.trace_extension is not None: - prefix_and_name = f"{self.prefix}.{name}" - ret = self.trace_extension(prefix_and_name, info) - if inspect.iscoroutine(ret): # pragma: no cover - raise TypeError( - "If you are using a synchronous interface, " - "the callback of the `trace` extension should " - "be a normal function instead of an asynchronous function." - ) - - if self.debug: - if not info or "return_value" in info and info["return_value"] is None: - message = name - else: - args = " ".join([f"{key}={value!r}" for key, value in info.items()]) - message = f"{name} {args}" - self.logger.debug(message) - - def __enter__(self) -> Trace: - if self.should_trace: - info = self.kwargs - self.trace(f"{self.name}.started", info) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - if self.should_trace: - if exc_value is None: - info = {"return_value": self.return_value} - self.trace(f"{self.name}.complete", info) - else: - info = {"exception": exc_value} - self.trace(f"{self.name}.failed", info) - - async def atrace(self, name: str, info: dict[str, typing.Any]) -> None: - if self.trace_extension is not None: - prefix_and_name = f"{self.prefix}.{name}" - coro = self.trace_extension(prefix_and_name, info) - if not inspect.iscoroutine(coro): # pragma: no cover - raise TypeError( - "If you're using an asynchronous interface, " - "the callback of the `trace` extension should " - "be an asynchronous function rather than a normal function." - ) - await coro - - if self.debug: - if not info or "return_value" in info and info["return_value"] is None: - message = name - else: - args = " ".join([f"{key}={value!r}" for key, value in info.items()]) - message = f"{name} {args}" - self.logger.debug(message) - - async def __aenter__(self) -> Trace: - if self.should_trace: - info = self.kwargs - await self.atrace(f"{self.name}.started", info) - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: types.TracebackType | None = None, - ) -> None: - if self.should_trace: - if exc_value is None: - info = {"return_value": self.return_value} - await self.atrace(f"{self.name}.complete", info) - else: - info = {"exception": exc_value} - await self.atrace(f"{self.name}.failed", info) diff --git a/contrib/python/httpcore/httpcore/_utils.py b/contrib/python/httpcore/httpcore/_utils.py deleted file mode 100644 index c44ff93cb2f..00000000000 --- a/contrib/python/httpcore/httpcore/_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import select -import socket -import sys - - -def is_socket_readable(sock: socket.socket | None) -> bool: - """ - Return whether a socket, as identifed by its file descriptor, is readable. - "A socket is readable" means that the read buffer isn't empty, i.e. that calling - .recv() on it would immediately return some data. - """ - # NOTE: we want check for readability without actually attempting to read, because - # we don't want to block forever if it's not readable. - - # In the case that the socket no longer exists, or cannot return a file - # descriptor, we treat it as being readable, as if it the next read operation - # on it is ready to return the terminating `b""`. - sock_fd = None if sock is None else sock.fileno() - if sock_fd is None or sock_fd < 0: # pragma: nocover - return True - - # The implementation below was stolen from: - # https://github.com/python-trio/trio/blob/20ee2b1b7376db637435d80e266212a35837ddcc/trio/_socket.py#L471-L478 - # See also: https://github.com/encode/httpcore/pull/193#issuecomment-703129316 - - # Use select.select on Windows, and when poll is unavailable and select.poll - # everywhere else. (E.g. When eventlet is in use. See #327) - if ( - sys.platform == "win32" or getattr(select, "poll", None) is None - ): # pragma: nocover - rready, _, _ = select.select([sock_fd], [], [], 0) - return bool(rready) - p = select.poll() - p.register(sock_fd, select.POLLIN) - return bool(p.poll(0)) diff --git a/contrib/python/httpcore/httpcore/py.typed b/contrib/python/httpcore/httpcore/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/httpcore/httpcore/py.typed +++ /dev/null diff --git a/contrib/python/httpcore/ya.make b/contrib/python/httpcore/ya.make deleted file mode 100644 index 6d4f3507cab..00000000000 --- a/contrib/python/httpcore/ya.make +++ /dev/null @@ -1,66 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(1.0.7) - -LICENSE(BSD-3-Clause) - -PEERDIR( - contrib/python/certifi - contrib/python/h11 -) - -NO_LINT() - -NO_CHECK_IMPORTS( - httpcore._async.http2 - httpcore._async.socks_proxy - httpcore._backends.trio - httpcore._sync.http2 - httpcore._sync.socks_proxy -) - -PY_SRCS( - TOP_LEVEL - httpcore/__init__.py - httpcore/_api.py - httpcore/_async/__init__.py - httpcore/_async/connection.py - httpcore/_async/connection_pool.py - httpcore/_async/http11.py - httpcore/_async/http2.py - httpcore/_async/http_proxy.py - httpcore/_async/interfaces.py - httpcore/_async/socks_proxy.py - httpcore/_backends/__init__.py - httpcore/_backends/anyio.py - httpcore/_backends/auto.py - httpcore/_backends/base.py - httpcore/_backends/mock.py - httpcore/_backends/sync.py - httpcore/_backends/trio.py - httpcore/_exceptions.py - httpcore/_models.py - httpcore/_ssl.py - httpcore/_sync/__init__.py - httpcore/_sync/connection.py - httpcore/_sync/connection_pool.py - httpcore/_sync/http11.py - httpcore/_sync/http2.py - httpcore/_sync/http_proxy.py - httpcore/_sync/interfaces.py - httpcore/_sync/socks_proxy.py - httpcore/_synchronization.py - httpcore/_trace.py - httpcore/_utils.py -) - -RESOURCE_FILES( - PREFIX contrib/python/httpcore/ - .dist-info/METADATA - .dist-info/top_level.txt - httpcore/py.typed -) - -END() diff --git a/contrib/python/httpx/.dist-info/METADATA b/contrib/python/httpx/.dist-info/METADATA deleted file mode 100644 index 4c6a0800344..00000000000 --- a/contrib/python/httpx/.dist-info/METADATA +++ /dev/null @@ -1,207 +0,0 @@ -Metadata-Version: 2.3 -Name: httpx -Version: 0.27.2 -Summary: The next generation HTTP client. -Project-URL: Changelog, https://github.com/encode/httpx/blob/master/CHANGELOG.md -Project-URL: Documentation, https://www.python-httpx.org -Project-URL: Homepage, https://github.com/encode/httpx -Project-URL: Source, https://github.com/encode/httpx -Author-email: Tom Christie <[email protected]> -License-Expression: BSD-3-Clause -License-File: LICENSE.md -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Web Environment -Classifier: Framework :: AsyncIO -Classifier: Framework :: Trio -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Internet :: WWW/HTTP -Requires-Python: >=3.8 -Requires-Dist: anyio -Requires-Dist: certifi -Requires-Dist: httpcore==1.* -Requires-Dist: idna -Requires-Dist: sniffio -Provides-Extra: brotli -Requires-Dist: brotli; (platform_python_implementation == 'CPython') and extra == 'brotli' -Requires-Dist: brotlicffi; (platform_python_implementation != 'CPython') and extra == 'brotli' -Provides-Extra: cli -Requires-Dist: click==8.*; extra == 'cli' -Requires-Dist: pygments==2.*; extra == 'cli' -Requires-Dist: rich<14,>=10; extra == 'cli' -Provides-Extra: http2 -Requires-Dist: h2<5,>=3; extra == 'http2' -Provides-Extra: socks -Requires-Dist: socksio==1.*; extra == 'socks' -Provides-Extra: zstd -Requires-Dist: zstandard>=0.18.0; extra == 'zstd' -Description-Content-Type: text/markdown - -<p align="center"> - <a href="https://www.python-httpx.org/"><img width="350" height="208" src="https://raw.githubusercontent.com/encode/httpx/master/docs/img/butterfly.png" alt='HTTPX'></a> -</p> - -<p align="center"><strong>HTTPX</strong> <em>- A next-generation HTTP client for Python.</em></p> - -<p align="center"> -<a href="https://github.com/encode/httpx/actions"> - <img src="https://github.com/encode/httpx/workflows/Test%20Suite/badge.svg" alt="Test Suite"> -</a> -<a href="https://pypi.org/project/httpx/"> - <img src="https://badge.fury.io/py/httpx.svg" alt="Package version"> -</a> -</p> - -HTTPX is a fully featured HTTP client library for Python 3. It includes **an integrated -command line client**, has support for both **HTTP/1.1 and HTTP/2**, and provides both **sync -and async APIs**. - ---- - -Install HTTPX using pip: - -```shell -pip install httpx -``` - -Now, let's get started: - -```pycon ->>> import httpx ->>> r = httpx.get('https://www.example.org/') ->>> r -<Response [200 OK]> ->>> r.status_code -200 ->>> r.headers['content-type'] -'text/html; charset=UTF-8' ->>> r.text -'<!doctype html>\n<html>\n<head>\n<title>Example Domain</title>...' -``` - -Or, using the command-line client. - -```shell -pip install 'httpx[cli]' # The command line client is an optional dependency. -``` - -Which now allows us to use HTTPX directly from the command-line... - -<p align="center"> - <img width="700" src="https://raw.githubusercontent.com/encode/httpx/master/docs/img/httpx-help.png" alt='httpx --help'> -</p> - -Sending a request... - -<p align="center"> - <img width="700" src="https://raw.githubusercontent.com/encode/httpx/master/docs/img/httpx-request.png" alt='httpx http://httpbin.org/json'> -</p> - -## Features - -HTTPX builds on the well-established usability of `requests`, and gives you: - -* A broadly [requests-compatible API](https://www.python-httpx.org/compatibility/). -* An integrated command-line client. -* HTTP/1.1 [and HTTP/2 support](https://www.python-httpx.org/http2/). -* Standard synchronous interface, but with [async support if you need it](https://www.python-httpx.org/async/). -* Ability to make requests directly to [WSGI applications](https://www.python-httpx.org/advanced/transports/#wsgi-transport) or [ASGI applications](https://www.python-httpx.org/advanced/transports/#asgi-transport). -* Strict timeouts everywhere. -* Fully type annotated. -* 100% test coverage. - -Plus all the standard features of `requests`... - -* International Domains and URLs -* Keep-Alive & Connection Pooling -* Sessions with Cookie Persistence -* Browser-style SSL Verification -* Basic/Digest Authentication -* Elegant Key/Value Cookies -* Automatic Decompression -* Automatic Content Decoding -* Unicode Response Bodies -* Multipart File Uploads -* HTTP(S) Proxy Support -* Connection Timeouts -* Streaming Downloads -* .netrc Support -* Chunked Requests - -## Installation - -Install with pip: - -```shell -pip install httpx -``` - -Or, to include the optional HTTP/2 support, use: - -```shell -pip install httpx[http2] -``` - -HTTPX requires Python 3.8+. - -## Documentation - -Project documentation is available at [https://www.python-httpx.org/](https://www.python-httpx.org/). - -For a run-through of all the basics, head over to the [QuickStart](https://www.python-httpx.org/quickstart/). - -For more advanced topics, see the [Advanced Usage](https://www.python-httpx.org/advanced/) section, the [async support](https://www.python-httpx.org/async/) section, or the [HTTP/2](https://www.python-httpx.org/http2/) section. - -The [Developer Interface](https://www.python-httpx.org/api/) provides a comprehensive API reference. - -To find out about tools that integrate with HTTPX, see [Third Party Packages](https://www.python-httpx.org/third_party_packages/). - -## Contribute - -If you want to contribute with HTTPX check out the [Contributing Guide](https://www.python-httpx.org/contributing/) to learn how to start. - -## Dependencies - -The HTTPX project relies on these excellent libraries: - -* `httpcore` - The underlying transport implementation for `httpx`. - * `h11` - HTTP/1.1 support. -* `certifi` - SSL certificates. -* `idna` - Internationalized domain name support. -* `sniffio` - Async library autodetection. - -As well as these optional installs: - -* `h2` - HTTP/2 support. *(Optional, with `httpx[http2]`)* -* `socksio` - SOCKS proxy support. *(Optional, with `httpx[socks]`)* -* `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* -* `click` - Command line client support. *(Optional, with `httpx[cli]`)* -* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* -* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* - -A huge amount of credit is due to `requests` for the API layout that -much of this work follows, as well as to `urllib3` for plenty of design -inspiration around the lower-level networking details. - ---- - -<p align="center"><i>HTTPX is <a href="https://github.com/encode/httpx/blob/master/LICENSE.md">BSD licensed</a> code.<br/>Designed & crafted with care.</i><br/>— 🦋 —</p> - -## Release Information - -### Fixed - -* Reintroduced supposedly-private `URLTypes` shortcut. (#2673) - - ---- - -[Full changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) diff --git a/contrib/python/httpx/.dist-info/entry_points.txt b/contrib/python/httpx/.dist-info/entry_points.txt deleted file mode 100644 index 8ae96007f7d..00000000000 --- a/contrib/python/httpx/.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -httpx = httpx:main diff --git a/contrib/python/httpx/.dist-info/top_level.txt b/contrib/python/httpx/.dist-info/top_level.txt deleted file mode 100644 index c180eb2f8ea..00000000000 --- a/contrib/python/httpx/.dist-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -httpx -httpx/_transports diff --git a/contrib/python/httpx/LICENSE.md b/contrib/python/httpx/LICENSE.md deleted file mode 100644 index ab79d16a3f4..00000000000 --- a/contrib/python/httpx/LICENSE.md +++ /dev/null @@ -1,12 +0,0 @@ -Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/). -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/httpx/README.md b/contrib/python/httpx/README.md deleted file mode 100644 index 5e459a2814a..00000000000 --- a/contrib/python/httpx/README.md +++ /dev/null @@ -1,149 +0,0 @@ -<p align="center"> - <a href="https://www.python-httpx.org/"><img width="350" height="208" src="https://raw.githubusercontent.com/encode/httpx/master/docs/img/butterfly.png" alt='HTTPX'></a> -</p> - -<p align="center"><strong>HTTPX</strong> <em>- A next-generation HTTP client for Python.</em></p> - -<p align="center"> -<a href="https://github.com/encode/httpx/actions"> - <img src="https://github.com/encode/httpx/workflows/Test%20Suite/badge.svg" alt="Test Suite"> -</a> -<a href="https://pypi.org/project/httpx/"> - <img src="https://badge.fury.io/py/httpx.svg" alt="Package version"> -</a> -</p> - -HTTPX is a fully featured HTTP client library for Python 3. It includes **an integrated -command line client**, has support for both **HTTP/1.1 and HTTP/2**, and provides both **sync -and async APIs**. - ---- - -Install HTTPX using pip: - -```shell -pip install httpx -``` - -Now, let's get started: - -```pycon ->>> import httpx ->>> r = httpx.get('https://www.example.org/') ->>> r -<Response [200 OK]> ->>> r.status_code -200 ->>> r.headers['content-type'] -'text/html; charset=UTF-8' ->>> r.text -'<!doctype html>\n<html>\n<head>\n<title>Example Domain</title>...' -``` - -Or, using the command-line client. - -```shell -pip install 'httpx[cli]' # The command line client is an optional dependency. -``` - -Which now allows us to use HTTPX directly from the command-line... - -<p align="center"> - <img width="700" src="docs/img/httpx-help.png" alt='httpx --help'> -</p> - -Sending a request... - -<p align="center"> - <img width="700" src="docs/img/httpx-request.png" alt='httpx http://httpbin.org/json'> -</p> - -## Features - -HTTPX builds on the well-established usability of `requests`, and gives you: - -* A broadly [requests-compatible API](https://www.python-httpx.org/compatibility/). -* An integrated command-line client. -* HTTP/1.1 [and HTTP/2 support](https://www.python-httpx.org/http2/). -* Standard synchronous interface, but with [async support if you need it](https://www.python-httpx.org/async/). -* Ability to make requests directly to [WSGI applications](https://www.python-httpx.org/advanced/transports/#wsgi-transport) or [ASGI applications](https://www.python-httpx.org/advanced/transports/#asgi-transport). -* Strict timeouts everywhere. -* Fully type annotated. -* 100% test coverage. - -Plus all the standard features of `requests`... - -* International Domains and URLs -* Keep-Alive & Connection Pooling -* Sessions with Cookie Persistence -* Browser-style SSL Verification -* Basic/Digest Authentication -* Elegant Key/Value Cookies -* Automatic Decompression -* Automatic Content Decoding -* Unicode Response Bodies -* Multipart File Uploads -* HTTP(S) Proxy Support -* Connection Timeouts -* Streaming Downloads -* .netrc Support -* Chunked Requests - -## Installation - -Install with pip: - -```shell -pip install httpx -``` - -Or, to include the optional HTTP/2 support, use: - -```shell -pip install httpx[http2] -``` - -HTTPX requires Python 3.8+. - -## Documentation - -Project documentation is available at [https://www.python-httpx.org/](https://www.python-httpx.org/). - -For a run-through of all the basics, head over to the [QuickStart](https://www.python-httpx.org/quickstart/). - -For more advanced topics, see the [Advanced Usage](https://www.python-httpx.org/advanced/) section, the [async support](https://www.python-httpx.org/async/) section, or the [HTTP/2](https://www.python-httpx.org/http2/) section. - -The [Developer Interface](https://www.python-httpx.org/api/) provides a comprehensive API reference. - -To find out about tools that integrate with HTTPX, see [Third Party Packages](https://www.python-httpx.org/third_party_packages/). - -## Contribute - -If you want to contribute with HTTPX check out the [Contributing Guide](https://www.python-httpx.org/contributing/) to learn how to start. - -## Dependencies - -The HTTPX project relies on these excellent libraries: - -* `httpcore` - The underlying transport implementation for `httpx`. - * `h11` - HTTP/1.1 support. -* `certifi` - SSL certificates. -* `idna` - Internationalized domain name support. -* `sniffio` - Async library autodetection. - -As well as these optional installs: - -* `h2` - HTTP/2 support. *(Optional, with `httpx[http2]`)* -* `socksio` - SOCKS proxy support. *(Optional, with `httpx[socks]`)* -* `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* -* `click` - Command line client support. *(Optional, with `httpx[cli]`)* -* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* -* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* - -A huge amount of credit is due to `requests` for the API layout that -much of this work follows, as well as to `urllib3` for plenty of design -inspiration around the lower-level networking details. - ---- - -<p align="center"><i>HTTPX is <a href="https://github.com/encode/httpx/blob/master/LICENSE.md">BSD licensed</a> code.<br/>Designed & crafted with care.</i><br/>— 🦋 —</p> diff --git a/contrib/python/httpx/httpx/__init__.py b/contrib/python/httpx/httpx/__init__.py deleted file mode 100644 index e9addde071f..00000000000 --- a/contrib/python/httpx/httpx/__init__.py +++ /dev/null @@ -1,105 +0,0 @@ -from .__version__ import __description__, __title__, __version__ -from ._api import * -from ._auth import * -from ._client import * -from ._config import * -from ._content import * -from ._exceptions import * -from ._models import * -from ._status_codes import * -from ._transports import * -from ._types import * -from ._urls import * - -try: - from ._main import main -except ImportError: # pragma: no cover - - def main() -> None: # type: ignore - import sys - - print( - "The httpx command line client could not run because the required " - "dependencies were not installed.\nMake sure you've installed " - "everything with: pip install 'httpx[cli]'" - ) - sys.exit(1) - - -__all__ = [ - "__description__", - "__title__", - "__version__", - "ASGITransport", - "AsyncBaseTransport", - "AsyncByteStream", - "AsyncClient", - "AsyncHTTPTransport", - "Auth", - "BaseTransport", - "BasicAuth", - "ByteStream", - "Client", - "CloseError", - "codes", - "ConnectError", - "ConnectTimeout", - "CookieConflict", - "Cookies", - "create_ssl_context", - "DecodingError", - "delete", - "DigestAuth", - "get", - "head", - "Headers", - "HTTPError", - "HTTPStatusError", - "HTTPTransport", - "InvalidURL", - "Limits", - "LocalProtocolError", - "main", - "MockTransport", - "NetRCAuth", - "NetworkError", - "options", - "patch", - "PoolTimeout", - "post", - "ProtocolError", - "Proxy", - "ProxyError", - "put", - "QueryParams", - "ReadError", - "ReadTimeout", - "RemoteProtocolError", - "request", - "Request", - "RequestError", - "RequestNotRead", - "Response", - "ResponseNotRead", - "stream", - "StreamClosed", - "StreamConsumed", - "StreamError", - "SyncByteStream", - "Timeout", - "TimeoutException", - "TooManyRedirects", - "TransportError", - "UnsupportedProtocol", - "URL", - "USE_CLIENT_DEFAULT", - "WriteError", - "WriteTimeout", - "WSGITransport", -] - - -__locals = locals() -for __name in __all__: - if not __name.startswith("__"): - setattr(__locals[__name], "__module__", "httpx") # noqa diff --git a/contrib/python/httpx/httpx/__version__.py b/contrib/python/httpx/httpx/__version__.py deleted file mode 100644 index 5eaaddbac99..00000000000 --- a/contrib/python/httpx/httpx/__version__.py +++ /dev/null @@ -1,3 +0,0 @@ -__title__ = "httpx" -__description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.27.2" diff --git a/contrib/python/httpx/httpx/_api.py b/contrib/python/httpx/httpx/_api.py deleted file mode 100644 index 4e98b606948..00000000000 --- a/contrib/python/httpx/httpx/_api.py +++ /dev/null @@ -1,479 +0,0 @@ -from __future__ import annotations - -import typing -from contextlib import contextmanager - -from ._client import Client -from ._config import DEFAULT_TIMEOUT_CONFIG -from ._models import Response -from ._types import ( - AuthTypes, - CertTypes, - CookieTypes, - HeaderTypes, - ProxiesTypes, - ProxyTypes, - QueryParamTypes, - RequestContent, - RequestData, - RequestFiles, - TimeoutTypes, - VerifyTypes, -) -from ._urls import URL - -__all__ = [ - "delete", - "get", - "head", - "options", - "patch", - "post", - "put", - "request", - "stream", -] - - -def request( - method: str, - url: URL | str, - *, - params: QueryParamTypes | None = None, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - follow_redirects: bool = False, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - trust_env: bool = True, -) -> Response: - """ - Sends an HTTP request. - - **Parameters:** - - * **method** - HTTP method for the new `Request` object: `GET`, `OPTIONS`, - `HEAD`, `POST`, `PUT`, `PATCH`, or `DELETE`. - * **url** - URL for the new `Request` object. - * **params** - *(optional)* Query parameters to include in the URL, as a - string, dictionary, or sequence of two-tuples. - * **content** - *(optional)* Binary content to include in the body of the - request, as bytes or a byte iterator. - * **data** - *(optional)* Form data to include in the body of the request, - as a dictionary. - * **files** - *(optional)* A dictionary of upload files to include in the - body of the request. - * **json** - *(optional)* A JSON serializable object to include in the body - of the request. - * **headers** - *(optional)* Dictionary of HTTP headers to include in the - request. - * **cookies** - *(optional)* Dictionary of Cookie items to include in the - request. - * **auth** - *(optional)* An authentication class to use when sending the - request. - * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. - * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. - * **timeout** - *(optional)* The timeout configuration to use when sending - the request. - * **follow_redirects** - *(optional)* Enables or disables HTTP redirects. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). - * **trust_env** - *(optional)* Enables or disables usage of environment - variables for configuration. - - **Returns:** `Response` - - Usage: - - ``` - >>> import httpx - >>> response = httpx.request('GET', 'https://httpbin.org/get') - >>> response - <Response [200 OK]> - ``` - """ - with Client( - cookies=cookies, - proxy=proxy, - proxies=proxies, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) as client: - return client.request( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - auth=auth, - follow_redirects=follow_redirects, - ) - - -@contextmanager -def stream( - method: str, - url: URL | str, - *, - params: QueryParamTypes | None = None, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - follow_redirects: bool = False, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - trust_env: bool = True, -) -> typing.Iterator[Response]: - """ - Alternative to `httpx.request()` that streams the response body - instead of loading it into memory at once. - - **Parameters**: See `httpx.request`. - - See also: [Streaming Responses][0] - - [0]: /quickstart#streaming-responses - """ - with Client( - cookies=cookies, - proxy=proxy, - proxies=proxies, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) as client: - with client.stream( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - auth=auth, - follow_redirects=follow_redirects, - ) as response: - yield response - - -def get( - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `GET` request. - - **Parameters**: See `httpx.request`. - - Note that the `data`, `files`, `json` and `content` parameters are not available - on this function, as `GET` requests should not include a request body. - """ - return request( - "GET", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def options( - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends an `OPTIONS` request. - - **Parameters**: See `httpx.request`. - - Note that the `data`, `files`, `json` and `content` parameters are not available - on this function, as `OPTIONS` requests should not include a request body. - """ - return request( - "OPTIONS", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def head( - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `HEAD` request. - - **Parameters**: See `httpx.request`. - - Note that the `data`, `files`, `json` and `content` parameters are not available - on this function, as `HEAD` requests should not include a request body. - """ - return request( - "HEAD", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def post( - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `POST` request. - - **Parameters**: See `httpx.request`. - """ - return request( - "POST", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def put( - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `PUT` request. - - **Parameters**: See `httpx.request`. - """ - return request( - "PUT", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def patch( - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `PATCH` request. - - **Parameters**: See `httpx.request`. - """ - return request( - "PATCH", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) - - -def delete( - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | None = None, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - trust_env: bool = True, -) -> Response: - """ - Sends a `DELETE` request. - - **Parameters**: See `httpx.request`. - - Note that the `data`, `files`, `json` and `content` parameters are not available - on this function, as `DELETE` requests should not include a request body. - """ - return request( - "DELETE", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - proxy=proxy, - proxies=proxies, - follow_redirects=follow_redirects, - cert=cert, - verify=verify, - timeout=timeout, - trust_env=trust_env, - ) diff --git a/contrib/python/httpx/httpx/_auth.py b/contrib/python/httpx/httpx/_auth.py deleted file mode 100644 index b03971ab4b3..00000000000 --- a/contrib/python/httpx/httpx/_auth.py +++ /dev/null @@ -1,348 +0,0 @@ -from __future__ import annotations - -import hashlib -import os -import re -import time -import typing -from base64 import b64encode -from urllib.request import parse_http_list - -from ._exceptions import ProtocolError -from ._models import Cookies, Request, Response -from ._utils import to_bytes, to_str, unquote - -if typing.TYPE_CHECKING: # pragma: no cover - from hashlib import _Hash - - -__all__ = ["Auth", "BasicAuth", "DigestAuth", "NetRCAuth"] - - -class Auth: - """ - Base class for all authentication schemes. - - To implement a custom authentication scheme, subclass `Auth` and override - the `.auth_flow()` method. - - If the authentication scheme does I/O such as disk access or network calls, or uses - synchronization primitives such as locks, you should override `.sync_auth_flow()` - and/or `.async_auth_flow()` instead of `.auth_flow()` to provide specialized - implementations that will be used by `Client` and `AsyncClient` respectively. - """ - - requires_request_body = False - requires_response_body = False - - def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - """ - Execute the authentication flow. - - To dispatch a request, `yield` it: - - ``` - yield request - ``` - - The client will `.send()` the response back into the flow generator. You can - access it like so: - - ``` - response = yield request - ``` - - A `return` (or reaching the end of the generator) will result in the - client returning the last response obtained from the server. - - You can dispatch as many requests as is necessary. - """ - yield request - - def sync_auth_flow( - self, request: Request - ) -> typing.Generator[Request, Response, None]: - """ - Execute the authentication flow synchronously. - - By default, this defers to `.auth_flow()`. You should override this method - when the authentication scheme does I/O and/or uses concurrency primitives. - """ - if self.requires_request_body: - request.read() - - flow = self.auth_flow(request) - request = next(flow) - - while True: - response = yield request - if self.requires_response_body: - response.read() - - try: - request = flow.send(response) - except StopIteration: - break - - async def async_auth_flow( - self, request: Request - ) -> typing.AsyncGenerator[Request, Response]: - """ - Execute the authentication flow asynchronously. - - By default, this defers to `.auth_flow()`. You should override this method - when the authentication scheme does I/O and/or uses concurrency primitives. - """ - if self.requires_request_body: - await request.aread() - - flow = self.auth_flow(request) - request = next(flow) - - while True: - response = yield request - if self.requires_response_body: - await response.aread() - - try: - request = flow.send(response) - except StopIteration: - break - - -class FunctionAuth(Auth): - """ - Allows the 'auth' argument to be passed as a simple callable function, - that takes the request, and returns a new, modified request. - """ - - def __init__(self, func: typing.Callable[[Request], Request]) -> None: - self._func = func - - def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - yield self._func(request) - - -class BasicAuth(Auth): - """ - Allows the 'auth' argument to be passed as a (username, password) pair, - and uses HTTP Basic authentication. - """ - - def __init__(self, username: str | bytes, password: str | bytes) -> None: - self._auth_header = self._build_auth_header(username, password) - - def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - request.headers["Authorization"] = self._auth_header - yield request - - def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: - userpass = b":".join((to_bytes(username), to_bytes(password))) - token = b64encode(userpass).decode() - return f"Basic {token}" - - -class NetRCAuth(Auth): - """ - Use a 'netrc' file to lookup basic auth credentials based on the url host. - """ - - def __init__(self, file: str | None = None) -> None: - # Lazily import 'netrc'. - # There's no need for us to load this module unless 'NetRCAuth' is being used. - import netrc - - self._netrc_info = netrc.netrc(file) - - def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - auth_info = self._netrc_info.authenticators(request.url.host) - if auth_info is None or not auth_info[2]: - # The netrc file did not have authentication credentials for this host. - yield request - else: - # Build a basic auth header with credentials from the netrc file. - request.headers["Authorization"] = self._build_auth_header( - username=auth_info[0], password=auth_info[2] - ) - yield request - - def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: - userpass = b":".join((to_bytes(username), to_bytes(password))) - token = b64encode(userpass).decode() - return f"Basic {token}" - - -class DigestAuth(Auth): - _ALGORITHM_TO_HASH_FUNCTION: dict[str, typing.Callable[[bytes], _Hash]] = { - "MD5": hashlib.md5, - "MD5-SESS": hashlib.md5, - "SHA": hashlib.sha1, - "SHA-SESS": hashlib.sha1, - "SHA-256": hashlib.sha256, - "SHA-256-SESS": hashlib.sha256, - "SHA-512": hashlib.sha512, - "SHA-512-SESS": hashlib.sha512, - } - - def __init__(self, username: str | bytes, password: str | bytes) -> None: - self._username = to_bytes(username) - self._password = to_bytes(password) - self._last_challenge: _DigestAuthChallenge | None = None - self._nonce_count = 1 - - def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - if self._last_challenge: - request.headers["Authorization"] = self._build_auth_header( - request, self._last_challenge - ) - - response = yield request - - if response.status_code != 401 or "www-authenticate" not in response.headers: - # If the response is not a 401 then we don't - # need to build an authenticated request. - return - - for auth_header in response.headers.get_list("www-authenticate"): - if auth_header.lower().startswith("digest "): - break - else: - # If the response does not include a 'WWW-Authenticate: Digest ...' - # header, then we don't need to build an authenticated request. - return - - self._last_challenge = self._parse_challenge(request, response, auth_header) - self._nonce_count = 1 - - request.headers["Authorization"] = self._build_auth_header( - request, self._last_challenge - ) - if response.cookies: - Cookies(response.cookies).set_cookie_header(request=request) - yield request - - def _parse_challenge( - self, request: Request, response: Response, auth_header: str - ) -> _DigestAuthChallenge: - """ - Returns a challenge from a Digest WWW-Authenticate header. - These take the form of: - `Digest realm="[email protected]",qop="auth,auth-int",nonce="abc",opaque="xyz"` - """ - scheme, _, fields = auth_header.partition(" ") - - # This method should only ever have been called with a Digest auth header. - assert scheme.lower() == "digest" - - header_dict: dict[str, str] = {} - for field in parse_http_list(fields): - key, value = field.strip().split("=", 1) - header_dict[key] = unquote(value) - - try: - realm = header_dict["realm"].encode() - nonce = header_dict["nonce"].encode() - algorithm = header_dict.get("algorithm", "MD5") - opaque = header_dict["opaque"].encode() if "opaque" in header_dict else None - qop = header_dict["qop"].encode() if "qop" in header_dict else None - return _DigestAuthChallenge( - realm=realm, nonce=nonce, algorithm=algorithm, opaque=opaque, qop=qop - ) - except KeyError as exc: - message = "Malformed Digest WWW-Authenticate header" - raise ProtocolError(message, request=request) from exc - - def _build_auth_header( - self, request: Request, challenge: _DigestAuthChallenge - ) -> str: - hash_func = self._ALGORITHM_TO_HASH_FUNCTION[challenge.algorithm.upper()] - - def digest(data: bytes) -> bytes: - return hash_func(data).hexdigest().encode() - - A1 = b":".join((self._username, challenge.realm, self._password)) - - path = request.url.raw_path - A2 = b":".join((request.method.encode(), path)) - # TODO: implement auth-int - HA2 = digest(A2) - - nc_value = b"%08x" % self._nonce_count - cnonce = self._get_client_nonce(self._nonce_count, challenge.nonce) - self._nonce_count += 1 - - HA1 = digest(A1) - if challenge.algorithm.lower().endswith("-sess"): - HA1 = digest(b":".join((HA1, challenge.nonce, cnonce))) - - qop = self._resolve_qop(challenge.qop, request=request) - if qop is None: - # Following RFC 2069 - digest_data = [HA1, challenge.nonce, HA2] - else: - # Following RFC 2617/7616 - digest_data = [HA1, challenge.nonce, nc_value, cnonce, qop, HA2] - - format_args = { - "username": self._username, - "realm": challenge.realm, - "nonce": challenge.nonce, - "uri": path, - "response": digest(b":".join(digest_data)), - "algorithm": challenge.algorithm.encode(), - } - if challenge.opaque: - format_args["opaque"] = challenge.opaque - if qop: - format_args["qop"] = b"auth" - format_args["nc"] = nc_value - format_args["cnonce"] = cnonce - - return "Digest " + self._get_header_value(format_args) - - def _get_client_nonce(self, nonce_count: int, nonce: bytes) -> bytes: - s = str(nonce_count).encode() - s += nonce - s += time.ctime().encode() - s += os.urandom(8) - - return hashlib.sha1(s).hexdigest()[:16].encode() - - def _get_header_value(self, header_fields: dict[str, bytes]) -> str: - NON_QUOTED_FIELDS = ("algorithm", "qop", "nc") - QUOTED_TEMPLATE = '{}="{}"' - NON_QUOTED_TEMPLATE = "{}={}" - - header_value = "" - for i, (field, value) in enumerate(header_fields.items()): - if i > 0: - header_value += ", " - template = ( - QUOTED_TEMPLATE - if field not in NON_QUOTED_FIELDS - else NON_QUOTED_TEMPLATE - ) - header_value += template.format(field, to_str(value)) - - return header_value - - def _resolve_qop(self, qop: bytes | None, request: Request) -> bytes | None: - if qop is None: - return None - qops = re.split(b", ?", qop) - if b"auth" in qops: - return b"auth" - - if qops == [b"auth-int"]: - raise NotImplementedError("Digest auth-int support is not yet implemented") - - message = f'Unexpected qop value "{qop!r}" in digest auth' - raise ProtocolError(message, request=request) - - -class _DigestAuthChallenge(typing.NamedTuple): - realm: bytes - nonce: bytes - algorithm: str - opaque: bytes | None - qop: bytes | None diff --git a/contrib/python/httpx/httpx/_client.py b/contrib/python/httpx/httpx/_client.py deleted file mode 100644 index 26610f6e87c..00000000000 --- a/contrib/python/httpx/httpx/_client.py +++ /dev/null @@ -1,2065 +0,0 @@ -from __future__ import annotations - -import datetime -import enum -import logging -import typing -import warnings -from contextlib import asynccontextmanager, contextmanager -from types import TracebackType - -from .__version__ import __version__ -from ._auth import Auth, BasicAuth, FunctionAuth -from ._config import ( - DEFAULT_LIMITS, - DEFAULT_MAX_REDIRECTS, - DEFAULT_TIMEOUT_CONFIG, - Limits, - Proxy, - Timeout, -) -from ._decoders import SUPPORTED_DECODERS -from ._exceptions import ( - InvalidURL, - RemoteProtocolError, - TooManyRedirects, - request_context, -) -from ._models import Cookies, Headers, Request, Response -from ._status_codes import codes -from ._transports.asgi import ASGITransport -from ._transports.base import AsyncBaseTransport, BaseTransport -from ._transports.default import AsyncHTTPTransport, HTTPTransport -from ._transports.wsgi import WSGITransport -from ._types import ( - AsyncByteStream, - AuthTypes, - CertTypes, - CookieTypes, - HeaderTypes, - ProxiesTypes, - ProxyTypes, - QueryParamTypes, - RequestContent, - RequestData, - RequestExtensions, - RequestFiles, - SyncByteStream, - TimeoutTypes, - VerifyTypes, -) -from ._urls import URL, QueryParams -from ._utils import ( - Timer, - URLPattern, - get_environment_proxies, - is_https_redirect, - same_origin, -) - -__all__ = ["USE_CLIENT_DEFAULT", "AsyncClient", "Client"] - -# The type annotation for @classmethod and context managers here follows PEP 484 -# https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods -T = typing.TypeVar("T", bound="Client") -U = typing.TypeVar("U", bound="AsyncClient") - - -class UseClientDefault: - """ - For some parameters such as `auth=...` and `timeout=...` we need to be able - to indicate the default "unset" state, in a way that is distinctly different - to using `None`. - - The default "unset" state indicates that whatever default is set on the - client should be used. This is different to setting `None`, which - explicitly disables the parameter, possibly overriding a client default. - - For example we use `timeout=USE_CLIENT_DEFAULT` in the `request()` signature. - Omitting the `timeout` parameter will send a request using whatever default - timeout has been configured on the client. Including `timeout=None` will - ensure no timeout is used. - - Note that user code shouldn't need to use the `USE_CLIENT_DEFAULT` constant, - but it is used internally when a parameter is not included. - """ - - -USE_CLIENT_DEFAULT = UseClientDefault() - - -logger = logging.getLogger("httpx") - -USER_AGENT = f"python-httpx/{__version__}" -ACCEPT_ENCODING = ", ".join( - [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] -) - - -class ClientState(enum.Enum): - # UNOPENED: - # The client has been instantiated, but has not been used to send a request, - # or been opened by entering the context of a `with` block. - UNOPENED = 1 - # OPENED: - # The client has either sent a request, or is within a `with` block. - OPENED = 2 - # CLOSED: - # The client has either exited the `with` block, or `close()` has - # been called explicitly. - CLOSED = 3 - - -class BoundSyncStream(SyncByteStream): - """ - A byte stream that is bound to a given response instance, and that - ensures the `response.elapsed` is set once the response is closed. - """ - - def __init__( - self, stream: SyncByteStream, response: Response, timer: Timer - ) -> None: - self._stream = stream - self._response = response - self._timer = timer - - def __iter__(self) -> typing.Iterator[bytes]: - for chunk in self._stream: - yield chunk - - def close(self) -> None: - seconds = self._timer.sync_elapsed() - self._response.elapsed = datetime.timedelta(seconds=seconds) - self._stream.close() - - -class BoundAsyncStream(AsyncByteStream): - """ - An async byte stream that is bound to a given response instance, and that - ensures the `response.elapsed` is set once the response is closed. - """ - - def __init__( - self, stream: AsyncByteStream, response: Response, timer: Timer - ) -> None: - self._stream = stream - self._response = response - self._timer = timer - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - async for chunk in self._stream: - yield chunk - - async def aclose(self) -> None: - seconds = await self._timer.async_elapsed() - self._response.elapsed = datetime.timedelta(seconds=seconds) - await self._stream.aclose() - - -EventHook = typing.Callable[..., typing.Any] - - -class BaseClient: - def __init__( - self, - *, - auth: AuthTypes | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - follow_redirects: bool = False, - max_redirects: int = DEFAULT_MAX_REDIRECTS, - event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, - base_url: URL | str = "", - trust_env: bool = True, - default_encoding: str | typing.Callable[[bytes], str] = "utf-8", - ) -> None: - event_hooks = {} if event_hooks is None else event_hooks - - self._base_url = self._enforce_trailing_slash(URL(base_url)) - - self._auth = self._build_auth(auth) - self._params = QueryParams(params) - self.headers = Headers(headers) - self._cookies = Cookies(cookies) - self._timeout = Timeout(timeout) - self.follow_redirects = follow_redirects - self.max_redirects = max_redirects - self._event_hooks = { - "request": list(event_hooks.get("request", [])), - "response": list(event_hooks.get("response", [])), - } - self._trust_env = trust_env - self._default_encoding = default_encoding - self._state = ClientState.UNOPENED - - @property - def is_closed(self) -> bool: - """ - Check if the client being closed - """ - return self._state == ClientState.CLOSED - - @property - def trust_env(self) -> bool: - return self._trust_env - - def _enforce_trailing_slash(self, url: URL) -> URL: - if url.raw_path.endswith(b"/"): - return url - return url.copy_with(raw_path=url.raw_path + b"/") - - def _get_proxy_map( - self, proxies: ProxiesTypes | None, allow_env_proxies: bool - ) -> dict[str, Proxy | None]: - if proxies is None: - if allow_env_proxies: - return { - key: None if url is None else Proxy(url=url) - for key, url in get_environment_proxies().items() - } - return {} - if isinstance(proxies, dict): - new_proxies = {} - for key, value in proxies.items(): - proxy = Proxy(url=value) if isinstance(value, (str, URL)) else value - new_proxies[str(key)] = proxy - return new_proxies - else: - proxy = Proxy(url=proxies) if isinstance(proxies, (str, URL)) else proxies - return {"all://": proxy} - - @property - def timeout(self) -> Timeout: - return self._timeout - - @timeout.setter - def timeout(self, timeout: TimeoutTypes) -> None: - self._timeout = Timeout(timeout) - - @property - def event_hooks(self) -> dict[str, list[EventHook]]: - return self._event_hooks - - @event_hooks.setter - def event_hooks(self, event_hooks: dict[str, list[EventHook]]) -> None: - self._event_hooks = { - "request": list(event_hooks.get("request", [])), - "response": list(event_hooks.get("response", [])), - } - - @property - def auth(self) -> Auth | None: - """ - Authentication class used when none is passed at the request-level. - - See also [Authentication][0]. - - [0]: /quickstart/#authentication - """ - return self._auth - - @auth.setter - def auth(self, auth: AuthTypes) -> None: - self._auth = self._build_auth(auth) - - @property - def base_url(self) -> URL: - """ - Base URL to use when sending requests with relative URLs. - """ - return self._base_url - - @base_url.setter - def base_url(self, url: URL | str) -> None: - self._base_url = self._enforce_trailing_slash(URL(url)) - - @property - def headers(self) -> Headers: - """ - HTTP headers to include when sending requests. - """ - return self._headers - - @headers.setter - def headers(self, headers: HeaderTypes) -> None: - client_headers = Headers( - { - b"Accept": b"*/*", - b"Accept-Encoding": ACCEPT_ENCODING.encode("ascii"), - b"Connection": b"keep-alive", - b"User-Agent": USER_AGENT.encode("ascii"), - } - ) - client_headers.update(headers) - self._headers = client_headers - - @property - def cookies(self) -> Cookies: - """ - Cookie values to include when sending requests. - """ - return self._cookies - - @cookies.setter - def cookies(self, cookies: CookieTypes) -> None: - self._cookies = Cookies(cookies) - - @property - def params(self) -> QueryParams: - """ - Query parameters to include in the URL when sending requests. - """ - return self._params - - @params.setter - def params(self, params: QueryParamTypes) -> None: - self._params = QueryParams(params) - - def build_request( - self, - method: str, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Request: - """ - Build and return a request instance. - - * The `params`, `headers` and `cookies` arguments - are merged with any values set on the client. - * The `url` argument is merged with any `base_url` set on the client. - - See also: [Request instances][0] - - [0]: /advanced/clients/#request-instances - """ - url = self._merge_url(url) - headers = self._merge_headers(headers) - cookies = self._merge_cookies(cookies) - params = self._merge_queryparams(params) - extensions = {} if extensions is None else extensions - if "timeout" not in extensions: - timeout = ( - self.timeout - if isinstance(timeout, UseClientDefault) - else Timeout(timeout) - ) - extensions = dict(**extensions, timeout=timeout.as_dict()) - return Request( - method, - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - extensions=extensions, - ) - - def _merge_url(self, url: URL | str) -> URL: - """ - Merge a URL argument together with any 'base_url' on the client, - to create the URL used for the outgoing request. - """ - merge_url = URL(url) - if merge_url.is_relative_url: - # To merge URLs we always append to the base URL. To get this - # behaviour correct we always ensure the base URL ends in a '/' - # separator, and strip any leading '/' from the merge URL. - # - # So, eg... - # - # >>> client = Client(base_url="https://www.example.com/subpath") - # >>> client.base_url - # URL('https://www.example.com/subpath/') - # >>> client.build_request("GET", "/path").url - # URL('https://www.example.com/subpath/path') - merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b"/") - return self.base_url.copy_with(raw_path=merge_raw_path) - return merge_url - - def _merge_cookies(self, cookies: CookieTypes | None = None) -> CookieTypes | None: - """ - Merge a cookies argument together with any cookies on the client, - to create the cookies used for the outgoing request. - """ - if cookies or self.cookies: - merged_cookies = Cookies(self.cookies) - merged_cookies.update(cookies) - return merged_cookies - return cookies - - def _merge_headers(self, headers: HeaderTypes | None = None) -> HeaderTypes | None: - """ - Merge a headers argument together with any headers on the client, - to create the headers used for the outgoing request. - """ - merged_headers = Headers(self.headers) - merged_headers.update(headers) - return merged_headers - - def _merge_queryparams( - self, params: QueryParamTypes | None = None - ) -> QueryParamTypes | None: - """ - Merge a queryparams argument together with any queryparams on the client, - to create the queryparams used for the outgoing request. - """ - if params or self.params: - merged_queryparams = QueryParams(self.params) - return merged_queryparams.merge(params) - return params - - def _build_auth(self, auth: AuthTypes | None) -> Auth | None: - if auth is None: - return None - elif isinstance(auth, tuple): - return BasicAuth(username=auth[0], password=auth[1]) - elif isinstance(auth, Auth): - return auth - elif callable(auth): - return FunctionAuth(func=auth) - else: - raise TypeError(f'Invalid "auth" argument: {auth!r}') - - def _build_request_auth( - self, - request: Request, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - ) -> Auth: - auth = ( - self._auth if isinstance(auth, UseClientDefault) else self._build_auth(auth) - ) - - if auth is not None: - return auth - - username, password = request.url.username, request.url.password - if username or password: - return BasicAuth(username=username, password=password) - - return Auth() - - def _build_redirect_request(self, request: Request, response: Response) -> Request: - """ - Given a request and a redirect response, return a new request that - should be used to effect the redirect. - """ - method = self._redirect_method(request, response) - url = self._redirect_url(request, response) - headers = self._redirect_headers(request, url, method) - stream = self._redirect_stream(request, method) - cookies = Cookies(self.cookies) - return Request( - method=method, - url=url, - headers=headers, - cookies=cookies, - stream=stream, - extensions=request.extensions, - ) - - def _redirect_method(self, request: Request, response: Response) -> str: - """ - When being redirected we may want to change the method of the request - based on certain specs or browser behavior. - """ - method = request.method - - # https://tools.ietf.org/html/rfc7231#section-6.4.4 - if response.status_code == codes.SEE_OTHER and method != "HEAD": - method = "GET" - - # Do what the browsers do, despite standards... - # Turn 302s into GETs. - if response.status_code == codes.FOUND and method != "HEAD": - method = "GET" - - # If a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in 'requests' issue 1704. - if response.status_code == codes.MOVED_PERMANENTLY and method == "POST": - method = "GET" - - return method - - def _redirect_url(self, request: Request, response: Response) -> URL: - """ - Return the URL for the redirect to follow. - """ - location = response.headers["Location"] - - try: - url = URL(location) - except InvalidURL as exc: - raise RemoteProtocolError( - f"Invalid URL in location header: {exc}.", request=request - ) from None - - # Handle malformed 'Location' headers that are "absolute" form, have no host. - # See: https://github.com/encode/httpx/issues/771 - if url.scheme and not url.host: - url = url.copy_with(host=request.url.host) - - # Facilitate relative 'Location' headers, as allowed by RFC 7231. - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - if url.is_relative_url: - url = request.url.join(url) - - # Attach previous fragment if needed (RFC 7231 7.1.2) - if request.url.fragment and not url.fragment: - url = url.copy_with(fragment=request.url.fragment) - - return url - - def _redirect_headers(self, request: Request, url: URL, method: str) -> Headers: - """ - Return the headers that should be used for the redirect request. - """ - headers = Headers(request.headers) - - if not same_origin(url, request.url): - if not is_https_redirect(request.url, url): - # Strip Authorization headers when responses are redirected - # away from the origin. (Except for direct HTTP to HTTPS redirects.) - headers.pop("Authorization", None) - - # Update the Host header. - headers["Host"] = url.netloc.decode("ascii") - - if method != request.method and method == "GET": - # If we've switch to a 'GET' request, then strip any headers which - # are only relevant to the request body. - headers.pop("Content-Length", None) - headers.pop("Transfer-Encoding", None) - - # We should use the client cookie store to determine any cookie header, - # rather than whatever was on the original outgoing request. - headers.pop("Cookie", None) - - return headers - - def _redirect_stream( - self, request: Request, method: str - ) -> SyncByteStream | AsyncByteStream | None: - """ - Return the body that should be used for the redirect request. - """ - if method != request.method and method == "GET": - return None - - return request.stream - - def _set_timeout(self, request: Request) -> None: - if "timeout" not in request.extensions: - timeout = ( - self.timeout - if isinstance(self.timeout, UseClientDefault) - else Timeout(self.timeout) - ) - request.extensions = dict(**request.extensions, timeout=timeout.as_dict()) - - -class Client(BaseClient): - """ - An HTTP client, with connection pooling, HTTP/2, redirects, cookie persistence, etc. - - It can be shared between threads. - - Usage: - - ```python - >>> client = httpx.Client() - >>> response = client.get('https://example.org') - ``` - - **Parameters:** - - * **auth** - *(optional)* An authentication class to use when sending - requests. - * **params** - *(optional)* Query parameters to include in request URLs, as - a string, dictionary, or sequence of two-tuples. - * **headers** - *(optional)* Dictionary of HTTP headers to include when - sending requests. - * **cookies** - *(optional)* Dictionary of Cookie items to include when - sending requests. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). - * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be - enabled. Defaults to `False`. - * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. - * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy - URLs. - * **timeout** - *(optional)* The timeout configuration to use when sending - requests. - * **limits** - *(optional)* The limits configuration to use. - * **max_redirects** - *(optional)* The maximum number of redirect responses - that should be followed. - * **base_url** - *(optional)* A URL to use as the base when building - request URLs. - * **transport** - *(optional)* A transport class to use for sending requests - over the network. - * **app** - *(optional)* An WSGI application to send requests to, - rather than sending actual network requests. - * **trust_env** - *(optional)* Enables or disables usage of environment - variables for configuration. - * **default_encoding** - *(optional)* The default encoding to use for decoding - response text, if no charset information is included in a response Content-Type - header. Set to a callable for automatic character set detection. Default: "utf-8". - """ - - def __init__( - self, - *, - auth: AuthTypes | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - mounts: None | (typing.Mapping[str, BaseTransport | None]) = None, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - follow_redirects: bool = False, - limits: Limits = DEFAULT_LIMITS, - max_redirects: int = DEFAULT_MAX_REDIRECTS, - event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, - base_url: URL | str = "", - transport: BaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, - trust_env: bool = True, - default_encoding: str | typing.Callable[[bytes], str] = "utf-8", - ) -> None: - super().__init__( - auth=auth, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - follow_redirects=follow_redirects, - max_redirects=max_redirects, - event_hooks=event_hooks, - base_url=base_url, - trust_env=trust_env, - default_encoding=default_encoding, - ) - - if http2: - try: - import h2 # noqa - except ImportError: # pragma: no cover - raise ImportError( - "Using http2=True, but the 'h2' package is not installed. " - "Make sure to install httpx using `pip install httpx[http2]`." - ) from None - - if proxies: - message = ( - "The 'proxies' argument is now deprecated." - " Use 'proxy' or 'mounts' instead." - ) - warnings.warn(message, DeprecationWarning) - if proxy: - raise RuntimeError("Use either `proxy` or 'proxies', not both.") - - if app: - message = ( - "The 'app' shortcut is now deprecated." - " Use the explicit style 'transport=WSGITransport(app=...)' instead." - ) - warnings.warn(message, DeprecationWarning) - - allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) - - self._transport = self._init_transport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - transport=transport, - app=app, - trust_env=trust_env, - ) - self._mounts: dict[URLPattern, BaseTransport | None] = { - URLPattern(key): None - if proxy is None - else self._init_proxy_transport( - proxy, - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - ) - for key, proxy in proxy_map.items() - } - if mounts is not None: - self._mounts.update( - {URLPattern(key): transport for key, transport in mounts.items()} - ) - - self._mounts = dict(sorted(self._mounts.items())) - - def _init_transport( - self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - transport: BaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, - trust_env: bool = True, - ) -> BaseTransport: - if transport is not None: - return transport - - if app is not None: - return WSGITransport(app=app) - - return HTTPTransport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - ) - - def _init_proxy_transport( - self, - proxy: Proxy, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, - ) -> BaseTransport: - return HTTPTransport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - proxy=proxy, - ) - - def _transport_for_url(self, url: URL) -> BaseTransport: - """ - Returns the transport instance that should be used for a given URL. - This will either be the standard connection pool, or a proxy. - """ - for pattern, transport in self._mounts.items(): - if pattern.matches(url): - return self._transport if transport is None else transport - - return self._transport - - def request( - self, - method: str, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Build and send a request. - - Equivalent to: - - ```python - request = client.build_request(...) - response = client.send(request, ...) - ``` - - See `Client.build_request()`, `Client.send()` and - [Merging of configuration][0] for how the various parameters - are merged with client-level configuration. - - [0]: /advanced/clients/#merging-of-configuration - """ - if cookies is not None: - message = ( - "Setting per-request cookies=<...> is being deprecated, because " - "the expected behaviour on cookie persistence is ambiguous. Set " - "cookies directly on the client instance instead." - ) - warnings.warn(message, DeprecationWarning) - - request = self.build_request( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - extensions=extensions, - ) - return self.send(request, auth=auth, follow_redirects=follow_redirects) - - @contextmanager - def stream( - self, - method: str, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> typing.Iterator[Response]: - """ - Alternative to `httpx.request()` that streams the response body - instead of loading it into memory at once. - - **Parameters**: See `httpx.request`. - - See also: [Streaming Responses][0] - - [0]: /quickstart#streaming-responses - """ - request = self.build_request( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - extensions=extensions, - ) - response = self.send( - request=request, - auth=auth, - follow_redirects=follow_redirects, - stream=True, - ) - try: - yield response - finally: - response.close() - - def send( - self, - request: Request, - *, - stream: bool = False, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - ) -> Response: - """ - Send a request. - - The request is sent as-is, unmodified. - - Typically you'll want to build one with `Client.build_request()` - so that any client-level configuration is merged into the request, - but passing an explicit `httpx.Request()` is supported as well. - - See also: [Request instances][0] - - [0]: /advanced/clients/#request-instances - """ - if self._state == ClientState.CLOSED: - raise RuntimeError("Cannot send a request, as the client has been closed.") - - self._state = ClientState.OPENED - follow_redirects = ( - self.follow_redirects - if isinstance(follow_redirects, UseClientDefault) - else follow_redirects - ) - - self._set_timeout(request) - - auth = self._build_request_auth(request, auth) - - response = self._send_handling_auth( - request, - auth=auth, - follow_redirects=follow_redirects, - history=[], - ) - try: - if not stream: - response.read() - - return response - - except BaseException as exc: - response.close() - raise exc - - def _send_handling_auth( - self, - request: Request, - auth: Auth, - follow_redirects: bool, - history: list[Response], - ) -> Response: - auth_flow = auth.sync_auth_flow(request) - try: - request = next(auth_flow) - - while True: - response = self._send_handling_redirects( - request, - follow_redirects=follow_redirects, - history=history, - ) - try: - try: - next_request = auth_flow.send(response) - except StopIteration: - return response - - response.history = list(history) - response.read() - request = next_request - history.append(response) - - except BaseException as exc: - response.close() - raise exc - finally: - auth_flow.close() - - def _send_handling_redirects( - self, - request: Request, - follow_redirects: bool, - history: list[Response], - ) -> Response: - while True: - if len(history) > self.max_redirects: - raise TooManyRedirects( - "Exceeded maximum allowed redirects.", request=request - ) - - for hook in self._event_hooks["request"]: - hook(request) - - response = self._send_single_request(request) - try: - for hook in self._event_hooks["response"]: - hook(response) - response.history = list(history) - - if not response.has_redirect_location: - return response - - request = self._build_redirect_request(request, response) - history = history + [response] - - if follow_redirects: - response.read() - else: - response.next_request = request - return response - - except BaseException as exc: - response.close() - raise exc - - def _send_single_request(self, request: Request) -> Response: - """ - Sends a single request, without handling any redirections. - """ - transport = self._transport_for_url(request.url) - timer = Timer() - timer.sync_start() - - if not isinstance(request.stream, SyncByteStream): - raise RuntimeError( - "Attempted to send an async request with a sync Client instance." - ) - - with request_context(request=request): - response = transport.handle_request(request) - - assert isinstance(response.stream, SyncByteStream) - - response.request = request - response.stream = BoundSyncStream( - response.stream, response=response, timer=timer - ) - self.cookies.extract_cookies(response) - response.default_encoding = self._default_encoding - - logger.info( - 'HTTP Request: %s %s "%s %d %s"', - request.method, - request.url, - response.http_version, - response.status_code, - response.reason_phrase, - ) - - return response - - def get( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `GET` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "GET", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def options( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send an `OPTIONS` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "OPTIONS", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def head( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `HEAD` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "HEAD", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def post( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `POST` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "POST", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def put( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `PUT` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "PUT", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def patch( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `PATCH` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "PATCH", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def delete( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `DELETE` request. - - **Parameters**: See `httpx.request`. - """ - return self.request( - "DELETE", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def close(self) -> None: - """ - Close transport and proxies. - """ - if self._state != ClientState.CLOSED: - self._state = ClientState.CLOSED - - self._transport.close() - for transport in self._mounts.values(): - if transport is not None: - transport.close() - - def __enter__(self: T) -> T: - if self._state != ClientState.UNOPENED: - msg = { - ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: ( - "Cannot reopen a client instance, once it has been closed." - ), - }[self._state] - raise RuntimeError(msg) - - self._state = ClientState.OPENED - - self._transport.__enter__() - for transport in self._mounts.values(): - if transport is not None: - transport.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - self._state = ClientState.CLOSED - - self._transport.__exit__(exc_type, exc_value, traceback) - for transport in self._mounts.values(): - if transport is not None: - transport.__exit__(exc_type, exc_value, traceback) - - -class AsyncClient(BaseClient): - """ - An asynchronous HTTP client, with connection pooling, HTTP/2, redirects, - cookie persistence, etc. - - It can be shared between tasks. - - Usage: - - ```python - >>> async with httpx.AsyncClient() as client: - >>> response = await client.get('https://example.org') - ``` - - **Parameters:** - - * **auth** - *(optional)* An authentication class to use when sending - requests. - * **params** - *(optional)* Query parameters to include in request URLs, as - a string, dictionary, or sequence of two-tuples. - * **headers** - *(optional)* Dictionary of HTTP headers to include when - sending requests. - * **cookies** - *(optional)* Dictionary of Cookie items to include when - sending requests. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). - * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be - enabled. Defaults to `False`. - * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. - * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy - URLs. - * **timeout** - *(optional)* The timeout configuration to use when sending - requests. - * **limits** - *(optional)* The limits configuration to use. - * **max_redirects** - *(optional)* The maximum number of redirect responses - that should be followed. - * **base_url** - *(optional)* A URL to use as the base when building - request URLs. - * **transport** - *(optional)* A transport class to use for sending requests - over the network. - * **app** - *(optional)* An ASGI application to send requests to, - rather than sending actual network requests. - * **trust_env** - *(optional)* Enables or disables usage of environment - variables for configuration. - * **default_encoding** - *(optional)* The default encoding to use for decoding - response text, if no charset information is included in a response Content-Type - header. Set to a callable for automatic character set detection. Default: "utf-8". - """ - - def __init__( - self, - *, - auth: AuthTypes | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, - mounts: None | (typing.Mapping[str, AsyncBaseTransport | None]) = None, - timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - follow_redirects: bool = False, - limits: Limits = DEFAULT_LIMITS, - max_redirects: int = DEFAULT_MAX_REDIRECTS, - event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, - base_url: URL | str = "", - transport: AsyncBaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, - trust_env: bool = True, - default_encoding: str | typing.Callable[[bytes], str] = "utf-8", - ) -> None: - super().__init__( - auth=auth, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - follow_redirects=follow_redirects, - max_redirects=max_redirects, - event_hooks=event_hooks, - base_url=base_url, - trust_env=trust_env, - default_encoding=default_encoding, - ) - - if http2: - try: - import h2 # noqa - except ImportError: # pragma: no cover - raise ImportError( - "Using http2=True, but the 'h2' package is not installed. " - "Make sure to install httpx using `pip install httpx[http2]`." - ) from None - - if proxies: - message = ( - "The 'proxies' argument is now deprecated." - " Use 'proxy' or 'mounts' instead." - ) - warnings.warn(message, DeprecationWarning) - if proxy: - raise RuntimeError("Use either `proxy` or 'proxies', not both.") - - if app: - message = ( - "The 'app' shortcut is now deprecated." - " Use the explicit style 'transport=ASGITransport(app=...)' instead." - ) - warnings.warn(message, DeprecationWarning) - - allow_env_proxies = trust_env and app is None and transport is None - proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) - - self._transport = self._init_transport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - transport=transport, - app=app, - trust_env=trust_env, - ) - - self._mounts: dict[URLPattern, AsyncBaseTransport | None] = { - URLPattern(key): None - if proxy is None - else self._init_proxy_transport( - proxy, - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - ) - for key, proxy in proxy_map.items() - } - if mounts is not None: - self._mounts.update( - {URLPattern(key): transport for key, transport in mounts.items()} - ) - self._mounts = dict(sorted(self._mounts.items())) - - def _init_transport( - self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - transport: AsyncBaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, - trust_env: bool = True, - ) -> AsyncBaseTransport: - if transport is not None: - return transport - - if app is not None: - return ASGITransport(app=app) - - return AsyncHTTPTransport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - ) - - def _init_proxy_transport( - self, - proxy: Proxy, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, - ) -> AsyncBaseTransport: - return AsyncHTTPTransport( - verify=verify, - cert=cert, - http1=http1, - http2=http2, - limits=limits, - trust_env=trust_env, - proxy=proxy, - ) - - def _transport_for_url(self, url: URL) -> AsyncBaseTransport: - """ - Returns the transport instance that should be used for a given URL. - This will either be the standard connection pool, or a proxy. - """ - for pattern, transport in self._mounts.items(): - if pattern.matches(url): - return self._transport if transport is None else transport - - return self._transport - - async def request( - self, - method: str, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Build and send a request. - - Equivalent to: - - ```python - request = client.build_request(...) - response = await client.send(request, ...) - ``` - - See `AsyncClient.build_request()`, `AsyncClient.send()` - and [Merging of configuration][0] for how the various parameters - are merged with client-level configuration. - - [0]: /advanced/clients/#merging-of-configuration - """ - - if cookies is not None: # pragma: no cover - message = ( - "Setting per-request cookies=<...> is being deprecated, because " - "the expected behaviour on cookie persistence is ambiguous. Set " - "cookies directly on the client instance instead." - ) - warnings.warn(message, DeprecationWarning) - - request = self.build_request( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - extensions=extensions, - ) - return await self.send(request, auth=auth, follow_redirects=follow_redirects) - - @asynccontextmanager - async def stream( - self, - method: str, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> typing.AsyncIterator[Response]: - """ - Alternative to `httpx.request()` that streams the response body - instead of loading it into memory at once. - - **Parameters**: See `httpx.request`. - - See also: [Streaming Responses][0] - - [0]: /quickstart#streaming-responses - """ - request = self.build_request( - method=method, - url=url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - timeout=timeout, - extensions=extensions, - ) - response = await self.send( - request=request, - auth=auth, - follow_redirects=follow_redirects, - stream=True, - ) - try: - yield response - finally: - await response.aclose() - - async def send( - self, - request: Request, - *, - stream: bool = False, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - ) -> Response: - """ - Send a request. - - The request is sent as-is, unmodified. - - Typically you'll want to build one with `AsyncClient.build_request()` - so that any client-level configuration is merged into the request, - but passing an explicit `httpx.Request()` is supported as well. - - See also: [Request instances][0] - - [0]: /advanced/clients/#request-instances - """ - if self._state == ClientState.CLOSED: - raise RuntimeError("Cannot send a request, as the client has been closed.") - - self._state = ClientState.OPENED - follow_redirects = ( - self.follow_redirects - if isinstance(follow_redirects, UseClientDefault) - else follow_redirects - ) - - self._set_timeout(request) - - auth = self._build_request_auth(request, auth) - - response = await self._send_handling_auth( - request, - auth=auth, - follow_redirects=follow_redirects, - history=[], - ) - try: - if not stream: - await response.aread() - - return response - - except BaseException as exc: - await response.aclose() - raise exc - - async def _send_handling_auth( - self, - request: Request, - auth: Auth, - follow_redirects: bool, - history: list[Response], - ) -> Response: - auth_flow = auth.async_auth_flow(request) - try: - request = await auth_flow.__anext__() - - while True: - response = await self._send_handling_redirects( - request, - follow_redirects=follow_redirects, - history=history, - ) - try: - try: - next_request = await auth_flow.asend(response) - except StopAsyncIteration: - return response - - response.history = list(history) - await response.aread() - request = next_request - history.append(response) - - except BaseException as exc: - await response.aclose() - raise exc - finally: - await auth_flow.aclose() - - async def _send_handling_redirects( - self, - request: Request, - follow_redirects: bool, - history: list[Response], - ) -> Response: - while True: - if len(history) > self.max_redirects: - raise TooManyRedirects( - "Exceeded maximum allowed redirects.", request=request - ) - - for hook in self._event_hooks["request"]: - await hook(request) - - response = await self._send_single_request(request) - try: - for hook in self._event_hooks["response"]: - await hook(response) - - response.history = list(history) - - if not response.has_redirect_location: - return response - - request = self._build_redirect_request(request, response) - history = history + [response] - - if follow_redirects: - await response.aread() - else: - response.next_request = request - return response - - except BaseException as exc: - await response.aclose() - raise exc - - async def _send_single_request(self, request: Request) -> Response: - """ - Sends a single request, without handling any redirections. - """ - transport = self._transport_for_url(request.url) - timer = Timer() - await timer.async_start() - - if not isinstance(request.stream, AsyncByteStream): - raise RuntimeError( - "Attempted to send an sync request with an AsyncClient instance." - ) - - with request_context(request=request): - response = await transport.handle_async_request(request) - - assert isinstance(response.stream, AsyncByteStream) - response.request = request - response.stream = BoundAsyncStream( - response.stream, response=response, timer=timer - ) - self.cookies.extract_cookies(response) - response.default_encoding = self._default_encoding - - logger.info( - 'HTTP Request: %s %s "%s %d %s"', - request.method, - request.url, - response.http_version, - response.status_code, - response.reason_phrase, - ) - - return response - - async def get( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `GET` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "GET", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def options( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send an `OPTIONS` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "OPTIONS", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def head( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `HEAD` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "HEAD", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def post( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `POST` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "POST", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def put( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `PUT` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "PUT", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def patch( - self, - url: URL | str, - *, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `PATCH` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "PATCH", - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def delete( - self, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, - follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, - timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, - extensions: RequestExtensions | None = None, - ) -> Response: - """ - Send a `DELETE` request. - - **Parameters**: See `httpx.request`. - """ - return await self.request( - "DELETE", - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - async def aclose(self) -> None: - """ - Close transport and proxies. - """ - if self._state != ClientState.CLOSED: - self._state = ClientState.CLOSED - - await self._transport.aclose() - for proxy in self._mounts.values(): - if proxy is not None: - await proxy.aclose() - - async def __aenter__(self: U) -> U: - if self._state != ClientState.UNOPENED: - msg = { - ClientState.OPENED: "Cannot open a client instance more than once.", - ClientState.CLOSED: ( - "Cannot reopen a client instance, once it has been closed." - ), - }[self._state] - raise RuntimeError(msg) - - self._state = ClientState.OPENED - - await self._transport.__aenter__() - for proxy in self._mounts.values(): - if proxy is not None: - await proxy.__aenter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - self._state = ClientState.CLOSED - - await self._transport.__aexit__(exc_type, exc_value, traceback) - for proxy in self._mounts.values(): - if proxy is not None: - await proxy.__aexit__(exc_type, exc_value, traceback) diff --git a/contrib/python/httpx/httpx/_compat.py b/contrib/python/httpx/httpx/_compat.py deleted file mode 100644 index 7d86dced46f..00000000000 --- a/contrib/python/httpx/httpx/_compat.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -The _compat module is used for code which requires branching between different -Python environments. It is excluded from the code coverage checks. -""" - -import re -import ssl -import sys -from types import ModuleType -from typing import Optional - -# Brotli support is optional -# The C bindings in `brotli` are recommended for CPython. -# The CFFI bindings in `brotlicffi` are recommended for PyPy and everything else. -try: - import brotlicffi as brotli -except ImportError: # pragma: no cover - try: - import brotli - except ImportError: - brotli = None - -# Zstandard support is optional -zstd: Optional[ModuleType] = None -try: - import zstandard as zstd -except (AttributeError, ImportError, ValueError): # Defensive: - zstd = None -else: - # The package 'zstandard' added the 'eof' property starting - # in v0.18.0 which we require to ensure a complete and - # valid zstd stream was fed into the ZstdDecoder. - # See: https://github.com/urllib3/urllib3/pull/2624 - _zstd_version = tuple( - map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] - ) - if _zstd_version < (0, 18): # Defensive: - zstd = None - - -if sys.version_info >= (3, 10) or ssl.OPENSSL_VERSION_INFO >= (1, 1, 0, 7): - - def set_minimum_tls_version_1_2(context: ssl.SSLContext) -> None: - # The OP_NO_SSL* and OP_NO_TLS* become deprecated in favor of - # 'SSLContext.minimum_version' from Python 3.7 onwards, however - # this attribute is not available unless the ssl module is compiled - # with OpenSSL 1.1.0g or newer. - # https://docs.python.org/3.10/library/ssl.html#ssl.SSLContext.minimum_version - # https://docs.python.org/3.7/library/ssl.html#ssl.SSLContext.minimum_version - context.minimum_version = ssl.TLSVersion.TLSv1_2 - -else: - - def set_minimum_tls_version_1_2(context: ssl.SSLContext) -> None: - # If 'minimum_version' isn't available, we configure these options with - # the older deprecated variants. - context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 - context.options |= ssl.OP_NO_TLSv1 - context.options |= ssl.OP_NO_TLSv1_1 - - -__all__ = ["brotli", "set_minimum_tls_version_1_2"] diff --git a/contrib/python/httpx/httpx/_config.py b/contrib/python/httpx/httpx/_config.py deleted file mode 100644 index f9fbf917e0f..00000000000 --- a/contrib/python/httpx/httpx/_config.py +++ /dev/null @@ -1,379 +0,0 @@ -from __future__ import annotations - -import logging -import os -import ssl -import typing -from pathlib import Path - -import certifi - -from ._compat import set_minimum_tls_version_1_2 -from ._models import Headers -from ._types import CertTypes, HeaderTypes, TimeoutTypes, VerifyTypes -from ._urls import URL -from ._utils import get_ca_bundle_from_env - -__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] - -DEFAULT_CIPHERS = ":".join( - [ - "ECDHE+AESGCM", - "ECDHE+CHACHA20", - "DHE+AESGCM", - "DHE+CHACHA20", - "ECDH+AESGCM", - "DH+AESGCM", - "ECDH+AES", - "DH+AES", - "RSA+AESGCM", - "RSA+AES", - "!aNULL", - "!eNULL", - "!MD5", - "!DSS", - ] -) - - -logger = logging.getLogger("httpx") - - -class UnsetType: - pass # pragma: no cover - - -UNSET = UnsetType() - - -def create_ssl_context( - cert: CertTypes | None = None, - verify: VerifyTypes = True, - trust_env: bool = True, - http2: bool = False, -) -> ssl.SSLContext: - return SSLConfig( - cert=cert, verify=verify, trust_env=trust_env, http2=http2 - ).ssl_context - - -class SSLConfig: - """ - SSL Configuration. - """ - - DEFAULT_CA_BUNDLE_PATH = certifi.where() - if callable(DEFAULT_CA_BUNDLE_PATH): - DEFAULT_CA_BUNDLE_PATH = staticmethod(DEFAULT_CA_BUNDLE_PATH) - else: - DEFAULT_CA_BUNDLE_PATH = Path(DEFAULT_CA_BUNDLE_PATH) - - def __init__( - self, - *, - cert: CertTypes | None = None, - verify: VerifyTypes = True, - trust_env: bool = True, - http2: bool = False, - ) -> None: - self.cert = cert - self.verify = verify - self.trust_env = trust_env - self.http2 = http2 - self.ssl_context = self.load_ssl_context() - - def load_ssl_context(self) -> ssl.SSLContext: - logger.debug( - "load_ssl_context verify=%r cert=%r trust_env=%r http2=%r", - self.verify, - self.cert, - self.trust_env, - self.http2, - ) - - if self.verify: - return self.load_ssl_context_verify() - return self.load_ssl_context_no_verify() - - def load_ssl_context_no_verify(self) -> ssl.SSLContext: - """ - Return an SSL context for unverified connections. - """ - context = self._create_default_ssl_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - self._load_client_certs(context) - return context - - def load_ssl_context_verify(self) -> ssl.SSLContext: - """ - Return an SSL context for verified connections. - """ - if self.trust_env and self.verify is True: - ca_bundle = get_ca_bundle_from_env() - if ca_bundle is not None: - self.verify = ca_bundle - - if isinstance(self.verify, ssl.SSLContext): - # Allow passing in our own SSLContext object that's pre-configured. - context = self.verify - self._load_client_certs(context) - return context - elif isinstance(self.verify, bool): - ca_bundle_path = self.DEFAULT_CA_BUNDLE_PATH - elif Path(self.verify).exists(): - ca_bundle_path = Path(self.verify) - else: - raise IOError( - "Could not find a suitable TLS CA certificate bundle, " - "invalid path: {}".format(self.verify) - ) - - context = self._create_default_ssl_context() - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True - - # Signal to server support for PHA in TLS 1.3. Raises an - # AttributeError if only read-only access is implemented. - try: - context.post_handshake_auth = True - except AttributeError: # pragma: no cover - pass - - # Disable using 'commonName' for SSLContext.check_hostname - # when the 'subjectAltName' extension isn't available. - try: - context.hostname_checks_common_name = False - except AttributeError: # pragma: no cover - pass - - if callable(ca_bundle_path): - logger.debug("load_verify_locations cafile=%r", ca_bundle_path) - context.load_verify_locations(cafile=ca_bundle_path) - elif ca_bundle_path.is_file(): - cafile = str(ca_bundle_path) - logger.debug("load_verify_locations cafile=%r", cafile) - context.load_verify_locations(cafile=cafile) - elif ca_bundle_path.is_dir(): - capath = str(ca_bundle_path) - logger.debug("load_verify_locations capath=%r", capath) - context.load_verify_locations(capath=capath) - - self._load_client_certs(context) - - return context - - def _create_default_ssl_context(self) -> ssl.SSLContext: - """ - Creates the default SSLContext object that's used for both verified - and unverified connections. - """ - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - set_minimum_tls_version_1_2(context) - context.options |= ssl.OP_NO_COMPRESSION - context.set_ciphers(DEFAULT_CIPHERS) - - if ssl.HAS_ALPN: - alpn_idents = ["http/1.1", "h2"] if self.http2 else ["http/1.1"] - context.set_alpn_protocols(alpn_idents) - - keylogfile = os.environ.get("SSLKEYLOGFILE") - if keylogfile and self.trust_env: - context.keylog_filename = keylogfile - - return context - - def _load_client_certs(self, ssl_context: ssl.SSLContext) -> None: - """ - Loads client certificates into our SSLContext object - """ - if self.cert is not None: - if isinstance(self.cert, str): - ssl_context.load_cert_chain(certfile=self.cert) - elif isinstance(self.cert, tuple) and len(self.cert) == 2: - ssl_context.load_cert_chain(certfile=self.cert[0], keyfile=self.cert[1]) - elif isinstance(self.cert, tuple) and len(self.cert) == 3: - ssl_context.load_cert_chain( - certfile=self.cert[0], - keyfile=self.cert[1], - password=self.cert[2], - ) - - -class Timeout: - """ - Timeout configuration. - - **Usage**: - - Timeout(None) # No timeouts. - Timeout(5.0) # 5s timeout on all operations. - Timeout(None, connect=5.0) # 5s timeout on connect, no other timeouts. - Timeout(5.0, connect=10.0) # 10s timeout on connect. 5s timeout elsewhere. - Timeout(5.0, pool=None) # No timeout on acquiring connection from pool. - # 5s timeout elsewhere. - """ - - def __init__( - self, - timeout: TimeoutTypes | UnsetType = UNSET, - *, - connect: None | float | UnsetType = UNSET, - read: None | float | UnsetType = UNSET, - write: None | float | UnsetType = UNSET, - pool: None | float | UnsetType = UNSET, - ) -> None: - if isinstance(timeout, Timeout): - # Passed as a single explicit Timeout. - assert connect is UNSET - assert read is UNSET - assert write is UNSET - assert pool is UNSET - self.connect = timeout.connect # type: typing.Optional[float] - self.read = timeout.read # type: typing.Optional[float] - self.write = timeout.write # type: typing.Optional[float] - self.pool = timeout.pool # type: typing.Optional[float] - elif isinstance(timeout, tuple): - # Passed as a tuple. - self.connect = timeout[0] - self.read = timeout[1] - self.write = None if len(timeout) < 3 else timeout[2] - self.pool = None if len(timeout) < 4 else timeout[3] - elif not ( - isinstance(connect, UnsetType) - or isinstance(read, UnsetType) - or isinstance(write, UnsetType) - or isinstance(pool, UnsetType) - ): - self.connect = connect - self.read = read - self.write = write - self.pool = pool - else: - if isinstance(timeout, UnsetType): - raise ValueError( - "httpx.Timeout must either include a default, or set all " - "four parameters explicitly." - ) - self.connect = timeout if isinstance(connect, UnsetType) else connect - self.read = timeout if isinstance(read, UnsetType) else read - self.write = timeout if isinstance(write, UnsetType) else write - self.pool = timeout if isinstance(pool, UnsetType) else pool - - def as_dict(self) -> dict[str, float | None]: - return { - "connect": self.connect, - "read": self.read, - "write": self.write, - "pool": self.pool, - } - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, self.__class__) - and self.connect == other.connect - and self.read == other.read - and self.write == other.write - and self.pool == other.pool - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - if len({self.connect, self.read, self.write, self.pool}) == 1: - return f"{class_name}(timeout={self.connect})" - return ( - f"{class_name}(connect={self.connect}, " - f"read={self.read}, write={self.write}, pool={self.pool})" - ) - - -class Limits: - """ - Configuration for limits to various client behaviors. - - **Parameters:** - - * **max_connections** - The maximum number of concurrent connections that may be - established. - * **max_keepalive_connections** - Allow the connection pool to maintain - keep-alive connections below this point. Should be less than or equal - to `max_connections`. - * **keepalive_expiry** - Time limit on idle keep-alive connections in seconds. - """ - - def __init__( - self, - *, - max_connections: int | None = None, - max_keepalive_connections: int | None = None, - keepalive_expiry: float | None = 5.0, - ) -> None: - self.max_connections = max_connections - self.max_keepalive_connections = max_keepalive_connections - self.keepalive_expiry = keepalive_expiry - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, self.__class__) - and self.max_connections == other.max_connections - and self.max_keepalive_connections == other.max_keepalive_connections - and self.keepalive_expiry == other.keepalive_expiry - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return ( - f"{class_name}(max_connections={self.max_connections}, " - f"max_keepalive_connections={self.max_keepalive_connections}, " - f"keepalive_expiry={self.keepalive_expiry})" - ) - - -class Proxy: - def __init__( - self, - url: URL | str, - *, - ssl_context: ssl.SSLContext | None = None, - auth: tuple[str, str] | None = None, - headers: HeaderTypes | None = None, - ) -> None: - url = URL(url) - headers = Headers(headers) - - if url.scheme not in ("http", "https", "socks5"): - raise ValueError(f"Unknown scheme for proxy URL {url!r}") - - if url.username or url.password: - # Remove any auth credentials from the URL. - auth = (url.username, url.password) - url = url.copy_with(username=None, password=None) - - self.url = url - self.auth = auth - self.headers = headers - self.ssl_context = ssl_context - - @property - def raw_auth(self) -> tuple[bytes, bytes] | None: - # The proxy authentication as raw bytes. - return ( - None - if self.auth is None - else (self.auth[0].encode("utf-8"), self.auth[1].encode("utf-8")) - ) - - def __repr__(self) -> str: - # The authentication is represented with the password component masked. - auth = (self.auth[0], "********") if self.auth else None - - # Build a nice concise representation. - url_str = f"{str(self.url)!r}" - auth_str = f", auth={auth!r}" if auth else "" - headers_str = f", headers={dict(self.headers)!r}" if self.headers else "" - return f"Proxy({url_str}{auth_str}{headers_str})" - - -DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) -DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20) -DEFAULT_MAX_REDIRECTS = 20 diff --git a/contrib/python/httpx/httpx/_content.py b/contrib/python/httpx/httpx/_content.py deleted file mode 100644 index 786699f38ff..00000000000 --- a/contrib/python/httpx/httpx/_content.py +++ /dev/null @@ -1,238 +0,0 @@ -from __future__ import annotations - -import inspect -import warnings -from json import dumps as json_dumps -from typing import ( - Any, - AsyncIterable, - AsyncIterator, - Iterable, - Iterator, - Mapping, -) -from urllib.parse import urlencode - -from ._exceptions import StreamClosed, StreamConsumed -from ._multipart import MultipartStream -from ._types import ( - AsyncByteStream, - RequestContent, - RequestData, - RequestFiles, - ResponseContent, - SyncByteStream, -) -from ._utils import peek_filelike_length, primitive_value_to_str - -__all__ = ["ByteStream"] - - -class ByteStream(AsyncByteStream, SyncByteStream): - def __init__(self, stream: bytes) -> None: - self._stream = stream - - def __iter__(self) -> Iterator[bytes]: - yield self._stream - - async def __aiter__(self) -> AsyncIterator[bytes]: - yield self._stream - - -class IteratorByteStream(SyncByteStream): - CHUNK_SIZE = 65_536 - - def __init__(self, stream: Iterable[bytes]) -> None: - self._stream = stream - self._is_stream_consumed = False - self._is_generator = inspect.isgenerator(stream) - - def __iter__(self) -> Iterator[bytes]: - if self._is_stream_consumed and self._is_generator: - raise StreamConsumed() - - self._is_stream_consumed = True - if hasattr(self._stream, "read"): - # File-like interfaces should use 'read' directly. - chunk = self._stream.read(self.CHUNK_SIZE) - while chunk: - yield chunk - chunk = self._stream.read(self.CHUNK_SIZE) - else: - # Otherwise iterate. - for part in self._stream: - yield part - - -class AsyncIteratorByteStream(AsyncByteStream): - CHUNK_SIZE = 65_536 - - def __init__(self, stream: AsyncIterable[bytes]) -> None: - self._stream = stream - self._is_stream_consumed = False - self._is_generator = inspect.isasyncgen(stream) - - async def __aiter__(self) -> AsyncIterator[bytes]: - if self._is_stream_consumed and self._is_generator: - raise StreamConsumed() - - self._is_stream_consumed = True - if hasattr(self._stream, "aread"): - # File-like interfaces should use 'aread' directly. - chunk = await self._stream.aread(self.CHUNK_SIZE) - while chunk: - yield chunk - chunk = await self._stream.aread(self.CHUNK_SIZE) - else: - # Otherwise iterate. - async for part in self._stream: - yield part - - -class UnattachedStream(AsyncByteStream, SyncByteStream): - """ - If a request or response is serialized using pickle, then it is no longer - attached to a stream for I/O purposes. Any stream operations should result - in `httpx.StreamClosed`. - """ - - def __iter__(self) -> Iterator[bytes]: - raise StreamClosed() - - async def __aiter__(self) -> AsyncIterator[bytes]: - raise StreamClosed() - yield b"" # pragma: no cover - - -def encode_content( - content: str | bytes | Iterable[bytes] | AsyncIterable[bytes], -) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: - if isinstance(content, (bytes, str)): - body = content.encode("utf-8") if isinstance(content, str) else content - content_length = len(body) - headers = {"Content-Length": str(content_length)} if body else {} - return headers, ByteStream(body) - - elif isinstance(content, Iterable) and not isinstance(content, dict): - # `not isinstance(content, dict)` is a bit oddly specific, but it - # catches a case that's easy for users to make in error, and would - # otherwise pass through here, like any other bytes-iterable, - # because `dict` happens to be iterable. See issue #2491. - content_length_or_none = peek_filelike_length(content) - - if content_length_or_none is None: - headers = {"Transfer-Encoding": "chunked"} - else: - headers = {"Content-Length": str(content_length_or_none)} - return headers, IteratorByteStream(content) # type: ignore - - elif isinstance(content, AsyncIterable): - headers = {"Transfer-Encoding": "chunked"} - return headers, AsyncIteratorByteStream(content) - - raise TypeError(f"Unexpected type for 'content', {type(content)!r}") - - -def encode_urlencoded_data( - data: RequestData, -) -> tuple[dict[str, str], ByteStream]: - plain_data = [] - for key, value in data.items(): - if isinstance(value, (list, tuple)): - plain_data.extend([(key, primitive_value_to_str(item)) for item in value]) - else: - plain_data.append((key, primitive_value_to_str(value))) - body = urlencode(plain_data, doseq=True).encode("utf-8") - content_length = str(len(body)) - content_type = "application/x-www-form-urlencoded" - headers = {"Content-Length": content_length, "Content-Type": content_type} - return headers, ByteStream(body) - - -def encode_multipart_data( - data: RequestData, files: RequestFiles, boundary: bytes | None -) -> tuple[dict[str, str], MultipartStream]: - multipart = MultipartStream(data=data, files=files, boundary=boundary) - headers = multipart.get_headers() - return headers, multipart - - -def encode_text(text: str) -> tuple[dict[str, str], ByteStream]: - body = text.encode("utf-8") - content_length = str(len(body)) - content_type = "text/plain; charset=utf-8" - headers = {"Content-Length": content_length, "Content-Type": content_type} - return headers, ByteStream(body) - - -def encode_html(html: str) -> tuple[dict[str, str], ByteStream]: - body = html.encode("utf-8") - content_length = str(len(body)) - content_type = "text/html; charset=utf-8" - headers = {"Content-Length": content_length, "Content-Type": content_type} - return headers, ByteStream(body) - - -def encode_json(json: Any) -> tuple[dict[str, str], ByteStream]: - body = json_dumps(json).encode("utf-8") - content_length = str(len(body)) - content_type = "application/json" - headers = {"Content-Length": content_length, "Content-Type": content_type} - return headers, ByteStream(body) - - -def encode_request( - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: Any | None = None, - boundary: bytes | None = None, -) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: - """ - Handles encoding the given `content`, `data`, `files`, and `json`, - returning a two-tuple of (<headers>, <stream>). - """ - if data is not None and not isinstance(data, Mapping): - # We prefer to separate `content=<bytes|str|byte iterator|bytes aiterator>` - # for raw request content, and `data=<form data>` for url encoded or - # multipart form content. - # - # However for compat with requests, we *do* still support - # `data=<bytes...>` usages. We deal with that case here, treating it - # as if `content=<...>` had been supplied instead. - message = "Use 'content=<...>' to upload raw bytes/text content." - warnings.warn(message, DeprecationWarning) - return encode_content(data) - - if content is not None: - return encode_content(content) - elif files: - return encode_multipart_data(data or {}, files, boundary) - elif data: - return encode_urlencoded_data(data) - elif json is not None: - return encode_json(json) - - return {}, ByteStream(b"") - - -def encode_response( - content: ResponseContent | None = None, - text: str | None = None, - html: str | None = None, - json: Any | None = None, -) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: - """ - Handles encoding the given `content`, returning a two-tuple of - (<headers>, <stream>). - """ - if content is not None: - return encode_content(content) - elif text is not None: - return encode_text(text) - elif html is not None: - return encode_html(html) - elif json is not None: - return encode_json(json) - - return {}, ByteStream(b"") diff --git a/contrib/python/httpx/httpx/_decoders.py b/contrib/python/httpx/httpx/_decoders.py deleted file mode 100644 index 62f2c0b911a..00000000000 --- a/contrib/python/httpx/httpx/_decoders.py +++ /dev/null @@ -1,371 +0,0 @@ -""" -Handlers for Content-Encoding. - -See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding -""" - -from __future__ import annotations - -import codecs -import io -import typing -import zlib - -from ._compat import brotli, zstd -from ._exceptions import DecodingError - - -class ContentDecoder: - def decode(self, data: bytes) -> bytes: - raise NotImplementedError() # pragma: no cover - - def flush(self) -> bytes: - raise NotImplementedError() # pragma: no cover - - -class IdentityDecoder(ContentDecoder): - """ - Handle unencoded data. - """ - - def decode(self, data: bytes) -> bytes: - return data - - def flush(self) -> bytes: - return b"" - - -class DeflateDecoder(ContentDecoder): - """ - Handle 'deflate' decoding. - - See: https://stackoverflow.com/questions/1838699 - """ - - def __init__(self) -> None: - self.first_attempt = True - self.decompressor = zlib.decompressobj() - - def decode(self, data: bytes) -> bytes: - was_first_attempt = self.first_attempt - self.first_attempt = False - try: - return self.decompressor.decompress(data) - except zlib.error as exc: - if was_first_attempt: - self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS) - return self.decode(data) - raise DecodingError(str(exc)) from exc - - def flush(self) -> bytes: - try: - return self.decompressor.flush() - except zlib.error as exc: # pragma: no cover - raise DecodingError(str(exc)) from exc - - -class GZipDecoder(ContentDecoder): - """ - Handle 'gzip' decoding. - - See: https://stackoverflow.com/questions/1838699 - """ - - def __init__(self) -> None: - self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) - - def decode(self, data: bytes) -> bytes: - try: - return self.decompressor.decompress(data) - except zlib.error as exc: - raise DecodingError(str(exc)) from exc - - def flush(self) -> bytes: - try: - return self.decompressor.flush() - except zlib.error as exc: # pragma: no cover - raise DecodingError(str(exc)) from exc - - -class BrotliDecoder(ContentDecoder): - """ - Handle 'brotli' decoding. - - Requires `pip install brotlipy`. See: https://brotlipy.readthedocs.io/ - or `pip install brotli`. See https://github.com/google/brotli - Supports both 'brotlipy' and 'Brotli' packages since they share an import - name. The top branches are for 'brotlipy' and bottom branches for 'Brotli' - """ - - def __init__(self) -> None: - if brotli is None: # pragma: no cover - raise ImportError( - "Using 'BrotliDecoder', but neither of the 'brotlicffi' or 'brotli' " - "packages have been installed. " - "Make sure to install httpx using `pip install httpx[brotli]`." - ) from None - - self.decompressor = brotli.Decompressor() - self.seen_data = False - self._decompress: typing.Callable[[bytes], bytes] - if hasattr(self.decompressor, "decompress"): - # The 'brotlicffi' package. - self._decompress = self.decompressor.decompress # pragma: no cover - else: - # The 'brotli' package. - self._decompress = self.decompressor.process # pragma: no cover - - def decode(self, data: bytes) -> bytes: - if not data: - return b"" - self.seen_data = True - try: - return self._decompress(data) - except brotli.error as exc: - raise DecodingError(str(exc)) from exc - - def flush(self) -> bytes: - if not self.seen_data: - return b"" - try: - if hasattr(self.decompressor, "finish"): - # Only available in the 'brotlicffi' package. - - # As the decompressor decompresses eagerly, this - # will never actually emit any data. However, it will potentially throw - # errors if a truncated or damaged data stream has been used. - self.decompressor.finish() # pragma: no cover - return b"" - except brotli.error as exc: # pragma: no cover - raise DecodingError(str(exc)) from exc - - -class ZStandardDecoder(ContentDecoder): - """ - Handle 'zstd' RFC 8878 decoding. - - Requires `pip install zstandard`. - Can be installed as a dependency of httpx using `pip install httpx[zstd]`. - """ - - # inspired by the ZstdDecoder implementation in urllib3 - def __init__(self) -> None: - if zstd is None: # pragma: no cover - raise ImportError( - "Using 'ZStandardDecoder', ..." - "Make sure to install httpx using `pip install httpx[zstd]`." - ) from None - - self.decompressor = zstd.ZstdDecompressor().decompressobj() - - def decode(self, data: bytes) -> bytes: - assert zstd is not None - output = io.BytesIO() - try: - output.write(self.decompressor.decompress(data)) - while self.decompressor.eof and self.decompressor.unused_data: - unused_data = self.decompressor.unused_data - self.decompressor = zstd.ZstdDecompressor().decompressobj() - output.write(self.decompressor.decompress(unused_data)) - except zstd.ZstdError as exc: - raise DecodingError(str(exc)) from exc - return output.getvalue() - - def flush(self) -> bytes: - ret = self.decompressor.flush() # note: this is a no-op - if not self.decompressor.eof: - raise DecodingError("Zstandard data is incomplete") # pragma: no cover - return bytes(ret) - - -class MultiDecoder(ContentDecoder): - """ - Handle the case where multiple encodings have been applied. - """ - - def __init__(self, children: typing.Sequence[ContentDecoder]) -> None: - """ - 'children' should be a sequence of decoders in the order in which - each was applied. - """ - # Note that we reverse the order for decoding. - self.children = list(reversed(children)) - - def decode(self, data: bytes) -> bytes: - for child in self.children: - data = child.decode(data) - return data - - def flush(self) -> bytes: - data = b"" - for child in self.children: - data = child.decode(data) + child.flush() - return data - - -class ByteChunker: - """ - Handles returning byte content in fixed-size chunks. - """ - - def __init__(self, chunk_size: int | None = None) -> None: - self._buffer = io.BytesIO() - self._chunk_size = chunk_size - - def decode(self, content: bytes) -> list[bytes]: - if self._chunk_size is None: - return [content] if content else [] - - self._buffer.write(content) - if self._buffer.tell() >= self._chunk_size: - value = self._buffer.getvalue() - chunks = [ - value[i : i + self._chunk_size] - for i in range(0, len(value), self._chunk_size) - ] - if len(chunks[-1]) == self._chunk_size: - self._buffer.seek(0) - self._buffer.truncate() - return chunks - else: - self._buffer.seek(0) - self._buffer.write(chunks[-1]) - self._buffer.truncate() - return chunks[:-1] - else: - return [] - - def flush(self) -> list[bytes]: - value = self._buffer.getvalue() - self._buffer.seek(0) - self._buffer.truncate() - return [value] if value else [] - - -class TextChunker: - """ - Handles returning text content in fixed-size chunks. - """ - - def __init__(self, chunk_size: int | None = None) -> None: - self._buffer = io.StringIO() - self._chunk_size = chunk_size - - def decode(self, content: str) -> list[str]: - if self._chunk_size is None: - return [content] if content else [] - - self._buffer.write(content) - if self._buffer.tell() >= self._chunk_size: - value = self._buffer.getvalue() - chunks = [ - value[i : i + self._chunk_size] - for i in range(0, len(value), self._chunk_size) - ] - if len(chunks[-1]) == self._chunk_size: - self._buffer.seek(0) - self._buffer.truncate() - return chunks - else: - self._buffer.seek(0) - self._buffer.write(chunks[-1]) - self._buffer.truncate() - return chunks[:-1] - else: - return [] - - def flush(self) -> list[str]: - value = self._buffer.getvalue() - self._buffer.seek(0) - self._buffer.truncate() - return [value] if value else [] - - -class TextDecoder: - """ - Handles incrementally decoding bytes into text - """ - - def __init__(self, encoding: str = "utf-8") -> None: - self.decoder = codecs.getincrementaldecoder(encoding)(errors="replace") - - def decode(self, data: bytes) -> str: - return self.decoder.decode(data) - - def flush(self) -> str: - return self.decoder.decode(b"", True) - - -class LineDecoder: - """ - Handles incrementally reading lines from text. - - Has the same behaviour as the stdllib splitlines, - but handling the input iteratively. - """ - - def __init__(self) -> None: - self.buffer: list[str] = [] - self.trailing_cr: bool = False - - def decode(self, text: str) -> list[str]: - # See https://docs.python.org/3/library/stdtypes.html#str.splitlines - NEWLINE_CHARS = "\n\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029" - - # We always push a trailing `\r` into the next decode iteration. - if self.trailing_cr: - text = "\r" + text - self.trailing_cr = False - if text.endswith("\r"): - self.trailing_cr = True - text = text[:-1] - - if not text: - # NOTE: the edge case input of empty text doesn't occur in practice, - # because other httpx internals filter out this value - return [] # pragma: no cover - - trailing_newline = text[-1] in NEWLINE_CHARS - lines = text.splitlines() - - if len(lines) == 1 and not trailing_newline: - # No new lines, buffer the input and continue. - self.buffer.append(lines[0]) - return [] - - if self.buffer: - # Include any existing buffer in the first portion of the - # splitlines result. - lines = ["".join(self.buffer) + lines[0]] + lines[1:] - self.buffer = [] - - if not trailing_newline: - # If the last segment of splitlines is not newline terminated, - # then drop it from our output and start a new buffer. - self.buffer = [lines.pop()] - - return lines - - def flush(self) -> list[str]: - if not self.buffer and not self.trailing_cr: - return [] - - lines = ["".join(self.buffer)] - self.buffer = [] - self.trailing_cr = False - return lines - - -SUPPORTED_DECODERS = { - "identity": IdentityDecoder, - "gzip": GZipDecoder, - "deflate": DeflateDecoder, - "br": BrotliDecoder, - "zstd": ZStandardDecoder, -} - - -if brotli is None: - SUPPORTED_DECODERS.pop("br") # pragma: no cover -if zstd is None: - SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/contrib/python/httpx/httpx/_exceptions.py b/contrib/python/httpx/httpx/_exceptions.py deleted file mode 100644 index 77f45a6d398..00000000000 --- a/contrib/python/httpx/httpx/_exceptions.py +++ /dev/null @@ -1,379 +0,0 @@ -""" -Our exception hierarchy: - -* HTTPError - x RequestError - + TransportError - - TimeoutException - · ConnectTimeout - · ReadTimeout - · WriteTimeout - · PoolTimeout - - NetworkError - · ConnectError - · ReadError - · WriteError - · CloseError - - ProtocolError - · LocalProtocolError - · RemoteProtocolError - - ProxyError - - UnsupportedProtocol - + DecodingError - + TooManyRedirects - x HTTPStatusError -* InvalidURL -* CookieConflict -* StreamError - x StreamConsumed - x StreamClosed - x ResponseNotRead - x RequestNotRead -""" - -from __future__ import annotations - -import contextlib -import typing - -if typing.TYPE_CHECKING: - from ._models import Request, Response # pragma: no cover - -__all__ = [ - "CloseError", - "ConnectError", - "ConnectTimeout", - "CookieConflict", - "DecodingError", - "HTTPError", - "HTTPStatusError", - "InvalidURL", - "LocalProtocolError", - "NetworkError", - "PoolTimeout", - "ProtocolError", - "ProxyError", - "ReadError", - "ReadTimeout", - "RemoteProtocolError", - "RequestError", - "RequestNotRead", - "ResponseNotRead", - "StreamClosed", - "StreamConsumed", - "StreamError", - "TimeoutException", - "TooManyRedirects", - "TransportError", - "UnsupportedProtocol", - "WriteError", - "WriteTimeout", -] - - -class HTTPError(Exception): - """ - Base class for `RequestError` and `HTTPStatusError`. - - Useful for `try...except` blocks when issuing a request, - and then calling `.raise_for_status()`. - - For example: - - ``` - try: - response = httpx.get("https://www.example.com") - response.raise_for_status() - except httpx.HTTPError as exc: - print(f"HTTP Exception for {exc.request.url} - {exc}") - ``` - """ - - def __init__(self, message: str) -> None: - super().__init__(message) - self._request: Request | None = None - - @property - def request(self) -> Request: - if self._request is None: - raise RuntimeError("The .request property has not been set.") - return self._request - - @request.setter - def request(self, request: Request) -> None: - self._request = request - - -class RequestError(HTTPError): - """ - Base class for all exceptions that may occur when issuing a `.request()`. - """ - - def __init__(self, message: str, *, request: Request | None = None) -> None: - super().__init__(message) - # At the point an exception is raised we won't typically have a request - # instance to associate it with. - # - # The 'request_context' context manager is used within the Client and - # Response methods in order to ensure that any raised exceptions - # have a `.request` property set on them. - self._request = request - - -class TransportError(RequestError): - """ - Base class for all exceptions that occur at the level of the Transport API. - """ - - -# Timeout exceptions... - - -class TimeoutException(TransportError): - """ - The base class for timeout errors. - - An operation has timed out. - """ - - -class ConnectTimeout(TimeoutException): - """ - Timed out while connecting to the host. - """ - - -class ReadTimeout(TimeoutException): - """ - Timed out while receiving data from the host. - """ - - -class WriteTimeout(TimeoutException): - """ - Timed out while sending data to the host. - """ - - -class PoolTimeout(TimeoutException): - """ - Timed out waiting to acquire a connection from the pool. - """ - - -# Core networking exceptions... - - -class NetworkError(TransportError): - """ - The base class for network-related errors. - - An error occurred while interacting with the network. - """ - - -class ReadError(NetworkError): - """ - Failed to receive data from the network. - """ - - -class WriteError(NetworkError): - """ - Failed to send data through the network. - """ - - -class ConnectError(NetworkError): - """ - Failed to establish a connection. - """ - - -class CloseError(NetworkError): - """ - Failed to close a connection. - """ - - -# Other transport exceptions... - - -class ProxyError(TransportError): - """ - An error occurred while establishing a proxy connection. - """ - - -class UnsupportedProtocol(TransportError): - """ - Attempted to make a request to an unsupported protocol. - - For example issuing a request to `ftp://www.example.com`. - """ - - -class ProtocolError(TransportError): - """ - The protocol was violated. - """ - - -class LocalProtocolError(ProtocolError): - """ - A protocol was violated by the client. - - For example if the user instantiated a `Request` instance explicitly, - failed to include the mandatory `Host:` header, and then issued it directly - using `client.send()`. - """ - - -class RemoteProtocolError(ProtocolError): - """ - The protocol was violated by the server. - - For example, returning malformed HTTP. - """ - - -# Other request exceptions... - - -class DecodingError(RequestError): - """ - Decoding of the response failed, due to a malformed encoding. - """ - - -class TooManyRedirects(RequestError): - """ - Too many redirects. - """ - - -# Client errors - - -class HTTPStatusError(HTTPError): - """ - The response had an error HTTP status of 4xx or 5xx. - - May be raised when calling `response.raise_for_status()` - """ - - def __init__(self, message: str, *, request: Request, response: Response) -> None: - super().__init__(message) - self.request = request - self.response = response - - -class InvalidURL(Exception): - """ - URL is improperly formed or cannot be parsed. - """ - - def __init__(self, message: str) -> None: - super().__init__(message) - - -class CookieConflict(Exception): - """ - Attempted to lookup a cookie by name, but multiple cookies existed. - - Can occur when calling `response.cookies.get(...)`. - """ - - def __init__(self, message: str) -> None: - super().__init__(message) - - -# Stream exceptions... - -# These may occur as the result of a programming error, by accessing -# the request/response stream in an invalid manner. - - -class StreamError(RuntimeError): - """ - The base class for stream exceptions. - - The developer made an error in accessing the request stream in - an invalid way. - """ - - def __init__(self, message: str) -> None: - super().__init__(message) - - -class StreamConsumed(StreamError): - """ - Attempted to read or stream content, but the content has already - been streamed. - """ - - def __init__(self) -> None: - message = ( - "Attempted to read or stream some content, but the content has " - "already been streamed. For requests, this could be due to passing " - "a generator as request content, and then receiving a redirect " - "response or a secondary request as part of an authentication flow." - "For responses, this could be due to attempting to stream the response " - "content more than once." - ) - super().__init__(message) - - -class StreamClosed(StreamError): - """ - Attempted to read or stream response content, but the request has been - closed. - """ - - def __init__(self) -> None: - message = ( - "Attempted to read or stream content, but the stream has " "been closed." - ) - super().__init__(message) - - -class ResponseNotRead(StreamError): - """ - Attempted to access streaming response content, without having called `read()`. - """ - - def __init__(self) -> None: - message = ( - "Attempted to access streaming response content," - " without having called `read()`." - ) - super().__init__(message) - - -class RequestNotRead(StreamError): - """ - Attempted to access streaming request content, without having called `read()`. - """ - - def __init__(self) -> None: - message = ( - "Attempted to access streaming request content," - " without having called `read()`." - ) - super().__init__(message) - - -def request_context( - request: Request | None = None, -) -> typing.Iterator[None]: - """ - A context manager that can be used to attach the given request context - to any `RequestError` exceptions that are raised within the block. - """ - try: - yield - except RequestError as exc: - if request is not None: - exc.request = request - raise exc diff --git a/contrib/python/httpx/httpx/_main.py b/contrib/python/httpx/httpx/_main.py deleted file mode 100644 index 72657f8ca3c..00000000000 --- a/contrib/python/httpx/httpx/_main.py +++ /dev/null @@ -1,509 +0,0 @@ -from __future__ import annotations - -import functools -import json -import sys -import typing - -import click -import httpcore -import pygments.lexers -import pygments.util -import rich.console -import rich.markup -import rich.progress -import rich.syntax -import rich.table - -from ._client import Client -from ._exceptions import RequestError -from ._models import Response -from ._status_codes import codes - - -def print_help() -> None: - console = rich.console.Console() - - console.print("[bold]HTTPX :butterfly:", justify="center") - console.print() - console.print("A next generation HTTP client.", justify="center") - console.print() - console.print( - "Usage: [bold]httpx[/bold] [cyan]<URL> [OPTIONS][/cyan] ", justify="left" - ) - console.print() - - table = rich.table.Table.grid(padding=1, pad_edge=True) - table.add_column("Parameter", no_wrap=True, justify="left", style="bold") - table.add_column("Description") - table.add_row( - "-m, --method [cyan]METHOD", - "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD.\n" - "[Default: GET, or POST if a request body is included]", - ) - table.add_row( - "-p, --params [cyan]<NAME VALUE> ...", - "Query parameters to include in the request URL.", - ) - table.add_row( - "-c, --content [cyan]TEXT", "Byte content to include in the request body." - ) - table.add_row( - "-d, --data [cyan]<NAME VALUE> ...", "Form data to include in the request body." - ) - table.add_row( - "-f, --files [cyan]<NAME FILENAME> ...", - "Form files to include in the request body.", - ) - table.add_row("-j, --json [cyan]TEXT", "JSON data to include in the request body.") - table.add_row( - "-h, --headers [cyan]<NAME VALUE> ...", - "Include additional HTTP headers in the request.", - ) - table.add_row( - "--cookies [cyan]<NAME VALUE> ...", "Cookies to include in the request." - ) - table.add_row( - "--auth [cyan]<USER PASS>", - "Username and password to include in the request. Specify '-' for the password" - " to use a password prompt. Note that using --verbose/-v will expose" - " the Authorization header, including the password encoding" - " in a trivially reversible format.", - ) - - table.add_row( - "--proxy [cyan]URL", - "Send the request via a proxy. Should be the URL giving the proxy address.", - ) - - table.add_row( - "--timeout [cyan]FLOAT", - "Timeout value to use for network operations, such as establishing the" - " connection, reading some data, etc... [Default: 5.0]", - ) - - table.add_row("--follow-redirects", "Automatically follow redirects.") - table.add_row("--no-verify", "Disable SSL verification.") - table.add_row( - "--http2", "Send the request using HTTP/2, if the remote server supports it." - ) - - table.add_row( - "--download [cyan]FILE", - "Save the response content as a file, rather than displaying it.", - ) - - table.add_row("-v, --verbose", "Verbose output. Show request as well as response.") - table.add_row("--help", "Show this message and exit.") - console.print(table) - - -def get_lexer_for_response(response: Response) -> str: - content_type = response.headers.get("Content-Type") - if content_type is not None: - mime_type, _, _ = content_type.partition(";") - try: - return typing.cast( - str, pygments.lexers.get_lexer_for_mimetype(mime_type.strip()).name - ) - except pygments.util.ClassNotFound: # pragma: no cover - pass - return "" # pragma: no cover - - -def format_request_headers(request: httpcore.Request, http2: bool = False) -> str: - version = "HTTP/2" if http2 else "HTTP/1.1" - headers = [ - (name.lower() if http2 else name, value) for name, value in request.headers - ] - method = request.method.decode("ascii") - target = request.url.target.decode("ascii") - lines = [f"{method} {target} {version}"] + [ - f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers - ] - return "\n".join(lines) - - -def format_response_headers( - http_version: bytes, - status: int, - reason_phrase: bytes | None, - headers: list[tuple[bytes, bytes]], -) -> str: - version = http_version.decode("ascii") - reason = ( - codes.get_reason_phrase(status) - if reason_phrase is None - else reason_phrase.decode("ascii") - ) - lines = [f"{version} {status} {reason}"] + [ - f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers - ] - return "\n".join(lines) - - -def print_request_headers(request: httpcore.Request, http2: bool = False) -> None: - console = rich.console.Console() - http_text = format_request_headers(request, http2=http2) - syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) - console.print(syntax) - syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) - console.print(syntax) - - -def print_response_headers( - http_version: bytes, - status: int, - reason_phrase: bytes | None, - headers: list[tuple[bytes, bytes]], -) -> None: - console = rich.console.Console() - http_text = format_response_headers(http_version, status, reason_phrase, headers) - syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) - console.print(syntax) - syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) - console.print(syntax) - - -def print_response(response: Response) -> None: - console = rich.console.Console() - lexer_name = get_lexer_for_response(response) - if lexer_name: - if lexer_name.lower() == "json": - try: - data = response.json() - text = json.dumps(data, indent=4) - except ValueError: # pragma: no cover - text = response.text - else: - text = response.text - - syntax = rich.syntax.Syntax(text, lexer_name, theme="ansi_dark", word_wrap=True) - console.print(syntax) - else: - console.print(f"<{len(response.content)} bytes of binary data>") - - -_PCTRTT = typing.Tuple[typing.Tuple[str, str], ...] -_PCTRTTT = typing.Tuple[_PCTRTT, ...] -_PeerCertRetDictType = typing.Dict[str, typing.Union[str, _PCTRTTT, _PCTRTT]] - - -def format_certificate(cert: _PeerCertRetDictType) -> str: # pragma: no cover - lines = [] - for key, value in cert.items(): - if isinstance(value, (list, tuple)): - lines.append(f"* {key}:") - for item in value: - if key in ("subject", "issuer"): - for sub_item in item: - lines.append(f"* {sub_item[0]}: {sub_item[1]!r}") - elif isinstance(item, tuple) and len(item) == 2: - lines.append(f"* {item[0]}: {item[1]!r}") - else: - lines.append(f"* {item!r}") - else: - lines.append(f"* {key}: {value!r}") - return "\n".join(lines) - - -def trace( - name: str, info: typing.Mapping[str, typing.Any], verbose: bool = False -) -> None: - console = rich.console.Console() - if name == "connection.connect_tcp.started" and verbose: - host = info["host"] - console.print(f"* Connecting to {host!r}") - elif name == "connection.connect_tcp.complete" and verbose: - stream = info["return_value"] - server_addr = stream.get_extra_info("server_addr") - console.print(f"* Connected to {server_addr[0]!r} on port {server_addr[1]}") - elif name == "connection.start_tls.complete" and verbose: # pragma: no cover - stream = info["return_value"] - ssl_object = stream.get_extra_info("ssl_object") - version = ssl_object.version() - cipher = ssl_object.cipher() - server_cert = ssl_object.getpeercert() - alpn = ssl_object.selected_alpn_protocol() - console.print(f"* SSL established using {version!r} / {cipher[0]!r}") - console.print(f"* Selected ALPN protocol: {alpn!r}") - if server_cert: - console.print("* Server certificate:") - console.print(format_certificate(server_cert)) - elif name == "http11.send_request_headers.started" and verbose: - request = info["request"] - print_request_headers(request, http2=False) - elif name == "http2.send_request_headers.started" and verbose: # pragma: no cover - request = info["request"] - print_request_headers(request, http2=True) - elif name == "http11.receive_response_headers.complete": - http_version, status, reason_phrase, headers = info["return_value"] - print_response_headers(http_version, status, reason_phrase, headers) - elif name == "http2.receive_response_headers.complete": # pragma: no cover - status, headers = info["return_value"] - http_version = b"HTTP/2" - reason_phrase = None - print_response_headers(http_version, status, reason_phrase, headers) - - -def download_response(response: Response, download: typing.BinaryIO) -> None: - console = rich.console.Console() - console.print() - content_length = response.headers.get("Content-Length") - with rich.progress.Progress( - "[progress.description]{task.description}", - "[progress.percentage]{task.percentage:>3.0f}%", - rich.progress.BarColumn(bar_width=None), - rich.progress.DownloadColumn(), - rich.progress.TransferSpeedColumn(), - ) as progress: - description = f"Downloading [bold]{rich.markup.escape(download.name)}" - download_task = progress.add_task( - description, - total=int(content_length or 0), - start=content_length is not None, - ) - for chunk in response.iter_bytes(): - download.write(chunk) - progress.update(download_task, completed=response.num_bytes_downloaded) - - -def validate_json( - ctx: click.Context, - param: click.Option | click.Parameter, - value: typing.Any, -) -> typing.Any: - if value is None: - return None - - try: - return json.loads(value) - except json.JSONDecodeError: # pragma: no cover - raise click.BadParameter("Not valid JSON") - - -def validate_auth( - ctx: click.Context, - param: click.Option | click.Parameter, - value: typing.Any, -) -> typing.Any: - if value == (None, None): - return None - - username, password = value - if password == "-": # pragma: no cover - password = click.prompt("Password", hide_input=True) - return (username, password) - - -def handle_help( - ctx: click.Context, - param: click.Option | click.Parameter, - value: typing.Any, -) -> None: - if not value or ctx.resilient_parsing: - return - - print_help() - ctx.exit() - - [email protected](add_help_option=False) [email protected]("url", type=str) - "--method", - "-m", - "method", - type=str, - help=( - "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD. " - "[Default: GET, or POST if a request body is included]" - ), -) - "--params", - "-p", - "params", - type=(str, str), - multiple=True, - help="Query parameters to include in the request URL.", -) - "--content", - "-c", - "content", - type=str, - help="Byte content to include in the request body.", -) - "--data", - "-d", - "data", - type=(str, str), - multiple=True, - help="Form data to include in the request body.", -) - "--files", - "-f", - "files", - type=(str, click.File(mode="rb")), - multiple=True, - help="Form files to include in the request body.", -) - "--json", - "-j", - "json", - type=str, - callback=validate_json, - help="JSON data to include in the request body.", -) - "--headers", - "-h", - "headers", - type=(str, str), - multiple=True, - help="Include additional HTTP headers in the request.", -) - "--cookies", - "cookies", - type=(str, str), - multiple=True, - help="Cookies to include in the request.", -) - "--auth", - "auth", - type=(str, str), - default=(None, None), - callback=validate_auth, - help=( - "Username and password to include in the request. " - "Specify '-' for the password to use a password prompt. " - "Note that using --verbose/-v will expose the Authorization header, " - "including the password encoding in a trivially reversible format." - ), -) - "--proxy", - "proxy", - type=str, - default=None, - help="Send the request via a proxy. Should be the URL giving the proxy address.", -) - "--timeout", - "timeout", - type=float, - default=5.0, - help=( - "Timeout value to use for network operations, such as establishing the " - "connection, reading some data, etc... [Default: 5.0]" - ), -) - "--follow-redirects", - "follow_redirects", - is_flag=True, - default=False, - help="Automatically follow redirects.", -) - "--no-verify", - "verify", - is_flag=True, - default=True, - help="Disable SSL verification.", -) - "--http2", - "http2", - type=bool, - is_flag=True, - default=False, - help="Send the request using HTTP/2, if the remote server supports it.", -) - "--download", - type=click.File("wb"), - help="Save the response content as a file, rather than displaying it.", -) - "--verbose", - "-v", - type=bool, - is_flag=True, - default=False, - help="Verbose. Show request as well as response.", -) - "--help", - is_flag=True, - is_eager=True, - expose_value=False, - callback=handle_help, - help="Show this message and exit.", -) -def main( - url: str, - method: str, - params: list[tuple[str, str]], - content: str, - data: list[tuple[str, str]], - files: list[tuple[str, click.File]], - json: str, - headers: list[tuple[str, str]], - cookies: list[tuple[str, str]], - auth: tuple[str, str] | None, - proxy: str, - timeout: float, - follow_redirects: bool, - verify: bool, - http2: bool, - download: typing.BinaryIO | None, - verbose: bool, -) -> None: - """ - An HTTP command line client. - Sends a request and displays the response. - """ - if not method: - method = "POST" if content or data or files or json else "GET" - - try: - with Client( - proxy=proxy, - timeout=timeout, - verify=verify, - http2=http2, - ) as client: - with client.stream( - method, - url, - params=list(params), - content=content, - data=dict(data), - files=files, # type: ignore - json=json, - headers=headers, - cookies=dict(cookies), - auth=auth, - follow_redirects=follow_redirects, - extensions={"trace": functools.partial(trace, verbose=verbose)}, - ) as response: - if download is not None: - download_response(response, download) - else: - response.read() - if response.content: - print_response(response) - - except RequestError as exc: - console = rich.console.Console() - console.print(f"[red]{type(exc).__name__}[/red]: {exc}") - sys.exit(1) - - sys.exit(0 if response.is_success else 1) diff --git a/contrib/python/httpx/httpx/_models.py b/contrib/python/httpx/httpx/_models.py deleted file mode 100644 index 01d9583bc5e..00000000000 --- a/contrib/python/httpx/httpx/_models.py +++ /dev/null @@ -1,1211 +0,0 @@ -from __future__ import annotations - -import datetime -import email.message -import json as jsonlib -import typing -import urllib.request -from collections.abc import Mapping -from http.cookiejar import Cookie, CookieJar - -from ._content import ByteStream, UnattachedStream, encode_request, encode_response -from ._decoders import ( - SUPPORTED_DECODERS, - ByteChunker, - ContentDecoder, - IdentityDecoder, - LineDecoder, - MultiDecoder, - TextChunker, - TextDecoder, -) -from ._exceptions import ( - CookieConflict, - HTTPStatusError, - RequestNotRead, - ResponseNotRead, - StreamClosed, - StreamConsumed, - request_context, -) -from ._multipart import get_multipart_boundary_from_content_type -from ._status_codes import codes -from ._types import ( - AsyncByteStream, - CookieTypes, - HeaderTypes, - QueryParamTypes, - RequestContent, - RequestData, - RequestExtensions, - RequestFiles, - ResponseContent, - ResponseExtensions, - SyncByteStream, -) -from ._urls import URL -from ._utils import ( - is_known_encoding, - normalize_header_key, - normalize_header_value, - obfuscate_sensitive_headers, - parse_content_type_charset, - parse_header_links, -) - -__all__ = ["Cookies", "Headers", "Request", "Response"] - - -class Headers(typing.MutableMapping[str, str]): - """ - HTTP headers, as a case-insensitive multi-dict. - """ - - def __init__( - self, - headers: HeaderTypes | None = None, - encoding: str | None = None, - ) -> None: - if headers is None: - self._list = [] # type: typing.List[typing.Tuple[bytes, bytes, bytes]] - elif isinstance(headers, Headers): - self._list = list(headers._list) - elif isinstance(headers, Mapping): - self._list = [ - ( - normalize_header_key(k, lower=False, encoding=encoding), - normalize_header_key(k, lower=True, encoding=encoding), - normalize_header_value(v, encoding), - ) - for k, v in headers.items() - ] - else: - self._list = [ - ( - normalize_header_key(k, lower=False, encoding=encoding), - normalize_header_key(k, lower=True, encoding=encoding), - normalize_header_value(v, encoding), - ) - for k, v in headers - ] - - self._encoding = encoding - - @property - def encoding(self) -> str: - """ - Header encoding is mandated as ascii, but we allow fallbacks to utf-8 - or iso-8859-1. - """ - if self._encoding is None: - for encoding in ["ascii", "utf-8"]: - for key, value in self.raw: - try: - key.decode(encoding) - value.decode(encoding) - except UnicodeDecodeError: - break - else: - # The else block runs if 'break' did not occur, meaning - # all values fitted the encoding. - self._encoding = encoding - break - else: - # The ISO-8859-1 encoding covers all 256 code points in a byte, - # so will never raise decode errors. - self._encoding = "iso-8859-1" - return self._encoding - - @encoding.setter - def encoding(self, value: str) -> None: - self._encoding = value - - @property - def raw(self) -> list[tuple[bytes, bytes]]: - """ - Returns a list of the raw header items, as byte pairs. - """ - return [(raw_key, value) for raw_key, _, value in self._list] - - def keys(self) -> typing.KeysView[str]: - return {key.decode(self.encoding): None for _, key, value in self._list}.keys() - - def values(self) -> typing.ValuesView[str]: - values_dict: dict[str, str] = {} - for _, key, value in self._list: - str_key = key.decode(self.encoding) - str_value = value.decode(self.encoding) - if str_key in values_dict: - values_dict[str_key] += f", {str_value}" - else: - values_dict[str_key] = str_value - return values_dict.values() - - def items(self) -> typing.ItemsView[str, str]: - """ - Return `(key, value)` items of headers. Concatenate headers - into a single comma separated value when a key occurs multiple times. - """ - values_dict: dict[str, str] = {} - for _, key, value in self._list: - str_key = key.decode(self.encoding) - str_value = value.decode(self.encoding) - if str_key in values_dict: - values_dict[str_key] += f", {str_value}" - else: - values_dict[str_key] = str_value - return values_dict.items() - - def multi_items(self) -> list[tuple[str, str]]: - """ - Return a list of `(key, value)` pairs of headers. Allow multiple - occurrences of the same key without concatenating into a single - comma separated value. - """ - return [ - (key.decode(self.encoding), value.decode(self.encoding)) - for _, key, value in self._list - ] - - def get(self, key: str, default: typing.Any = None) -> typing.Any: - """ - Return a header value. If multiple occurrences of the header occur - then concatenate them together with commas. - """ - try: - return self[key] - except KeyError: - return default - - def get_list(self, key: str, split_commas: bool = False) -> list[str]: - """ - Return a list of all header values for a given key. - If `split_commas=True` is passed, then any comma separated header - values are split into multiple return strings. - """ - get_header_key = key.lower().encode(self.encoding) - - values = [ - item_value.decode(self.encoding) - for _, item_key, item_value in self._list - if item_key.lower() == get_header_key - ] - - if not split_commas: - return values - - split_values = [] - for value in values: - split_values.extend([item.strip() for item in value.split(",")]) - return split_values - - def update(self, headers: HeaderTypes | None = None) -> None: # type: ignore - headers = Headers(headers) - for key in headers.keys(): - if key in self: - self.pop(key) - self._list.extend(headers._list) - - def copy(self) -> Headers: - return Headers(self, encoding=self.encoding) - - def __getitem__(self, key: str) -> str: - """ - Return a single header value. - - If there are multiple headers with the same key, then we concatenate - them with commas. See: https://tools.ietf.org/html/rfc7230#section-3.2.2 - """ - normalized_key = key.lower().encode(self.encoding) - - items = [ - header_value.decode(self.encoding) - for _, header_key, header_value in self._list - if header_key == normalized_key - ] - - if items: - return ", ".join(items) - - raise KeyError(key) - - def __setitem__(self, key: str, value: str) -> None: - """ - Set the header `key` to `value`, removing any duplicate entries. - Retains insertion order. - """ - set_key = key.encode(self._encoding or "utf-8") - set_value = value.encode(self._encoding or "utf-8") - lookup_key = set_key.lower() - - found_indexes = [ - idx - for idx, (_, item_key, _) in enumerate(self._list) - if item_key == lookup_key - ] - - for idx in reversed(found_indexes[1:]): - del self._list[idx] - - if found_indexes: - idx = found_indexes[0] - self._list[idx] = (set_key, lookup_key, set_value) - else: - self._list.append((set_key, lookup_key, set_value)) - - def __delitem__(self, key: str) -> None: - """ - Remove the header `key`. - """ - del_key = key.lower().encode(self.encoding) - - pop_indexes = [ - idx - for idx, (_, item_key, _) in enumerate(self._list) - if item_key.lower() == del_key - ] - - if not pop_indexes: - raise KeyError(key) - - for idx in reversed(pop_indexes): - del self._list[idx] - - def __contains__(self, key: typing.Any) -> bool: - header_key = key.lower().encode(self.encoding) - return header_key in [key for _, key, _ in self._list] - - def __iter__(self) -> typing.Iterator[typing.Any]: - return iter(self.keys()) - - def __len__(self) -> int: - return len(self._list) - - def __eq__(self, other: typing.Any) -> bool: - try: - other_headers = Headers(other) - except ValueError: - return False - - self_list = [(key, value) for _, key, value in self._list] - other_list = [(key, value) for _, key, value in other_headers._list] - return sorted(self_list) == sorted(other_list) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - - encoding_str = "" - if self.encoding != "ascii": - encoding_str = f", encoding={self.encoding!r}" - - as_list = list(obfuscate_sensitive_headers(self.multi_items())) - as_dict = dict(as_list) - - no_duplicate_keys = len(as_dict) == len(as_list) - if no_duplicate_keys: - return f"{class_name}({as_dict!r}{encoding_str})" - return f"{class_name}({as_list!r}{encoding_str})" - - -class Request: - def __init__( - self, - method: str | bytes, - url: URL | str, - *, - params: QueryParamTypes | None = None, - headers: HeaderTypes | None = None, - cookies: CookieTypes | None = None, - content: RequestContent | None = None, - data: RequestData | None = None, - files: RequestFiles | None = None, - json: typing.Any | None = None, - stream: SyncByteStream | AsyncByteStream | None = None, - extensions: RequestExtensions | None = None, - ) -> None: - self.method = ( - method.decode("ascii").upper() - if isinstance(method, bytes) - else method.upper() - ) - self.url = URL(url) - if params is not None: - self.url = self.url.copy_merge_params(params=params) - self.headers = Headers(headers) - self.extensions = {} if extensions is None else extensions - - if cookies: - Cookies(cookies).set_cookie_header(self) - - if stream is None: - content_type: str | None = self.headers.get("content-type") - headers, stream = encode_request( - content=content, - data=data, - files=files, - json=json, - boundary=get_multipart_boundary_from_content_type( - content_type=content_type.encode(self.headers.encoding) - if content_type - else None - ), - ) - self._prepare(headers) - self.stream = stream - # Load the request body, except for streaming content. - if isinstance(stream, ByteStream): - self.read() - else: - # There's an important distinction between `Request(content=...)`, - # and `Request(stream=...)`. - # - # Using `content=...` implies automatically populated `Host` and content - # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`. - # - # Using `stream=...` will not automatically include *any* - # auto-populated headers. - # - # As an end-user you don't really need `stream=...`. It's only - # useful when: - # - # * Preserving the request stream when copying requests, eg for redirects. - # * Creating request instances on the *server-side* of the transport API. - self.stream = stream - - def _prepare(self, default_headers: dict[str, str]) -> None: - for key, value in default_headers.items(): - # Ignore Transfer-Encoding if the Content-Length has been set explicitly. - if key.lower() == "transfer-encoding" and "Content-Length" in self.headers: - continue - self.headers.setdefault(key, value) - - auto_headers: list[tuple[bytes, bytes]] = [] - - has_host = "Host" in self.headers - has_content_length = ( - "Content-Length" in self.headers or "Transfer-Encoding" in self.headers - ) - - if not has_host and self.url.host: - auto_headers.append((b"Host", self.url.netloc)) - if not has_content_length and self.method in ("POST", "PUT", "PATCH"): - auto_headers.append((b"Content-Length", b"0")) - - self.headers = Headers(auto_headers + self.headers.raw) - - @property - def content(self) -> bytes: - if not hasattr(self, "_content"): - raise RequestNotRead() - return self._content - - def read(self) -> bytes: - """ - Read and return the request content. - """ - if not hasattr(self, "_content"): - assert isinstance(self.stream, typing.Iterable) - self._content = b"".join(self.stream) - if not isinstance(self.stream, ByteStream): - # If a streaming request has been read entirely into memory, then - # we can replace the stream with a raw bytes implementation, - # to ensure that any non-replayable streams can still be used. - self.stream = ByteStream(self._content) - return self._content - - async def aread(self) -> bytes: - """ - Read and return the request content. - """ - if not hasattr(self, "_content"): - assert isinstance(self.stream, typing.AsyncIterable) - self._content = b"".join([part async for part in self.stream]) - if not isinstance(self.stream, ByteStream): - # If a streaming request has been read entirely into memory, then - # we can replace the stream with a raw bytes implementation, - # to ensure that any non-replayable streams can still be used. - self.stream = ByteStream(self._content) - return self._content - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - url = str(self.url) - return f"<{class_name}({self.method!r}, {url!r})>" - - def __getstate__(self) -> dict[str, typing.Any]: - return { - name: value - for name, value in self.__dict__.items() - if name not in ["extensions", "stream"] - } - - def __setstate__(self, state: dict[str, typing.Any]) -> None: - for name, value in state.items(): - setattr(self, name, value) - self.extensions = {} - self.stream = UnattachedStream() - - -class Response: - def __init__( - self, - status_code: int, - *, - headers: HeaderTypes | None = None, - content: ResponseContent | None = None, - text: str | None = None, - html: str | None = None, - json: typing.Any = None, - stream: SyncByteStream | AsyncByteStream | None = None, - request: Request | None = None, - extensions: ResponseExtensions | None = None, - history: list[Response] | None = None, - default_encoding: str | typing.Callable[[bytes], str] = "utf-8", - ) -> None: - self.status_code = status_code - self.headers = Headers(headers) - - self._request: Request | None = request - - # When follow_redirects=False and a redirect is received, - # the client will set `response.next_request`. - self.next_request: Request | None = None - - self.extensions: ResponseExtensions = {} if extensions is None else extensions - self.history = [] if history is None else list(history) - - self.is_closed = False - self.is_stream_consumed = False - - self.default_encoding = default_encoding - - if stream is None: - headers, stream = encode_response(content, text, html, json) - self._prepare(headers) - self.stream = stream - if isinstance(stream, ByteStream): - # Load the response body, except for streaming content. - self.read() - else: - # There's an important distinction between `Response(content=...)`, - # and `Response(stream=...)`. - # - # Using `content=...` implies automatically populated content headers, - # of either `Content-Length: ...` or `Transfer-Encoding: chunked`. - # - # Using `stream=...` will not automatically include any content headers. - # - # As an end-user you don't really need `stream=...`. It's only - # useful when creating response instances having received a stream - # from the transport API. - self.stream = stream - - self._num_bytes_downloaded = 0 - - def _prepare(self, default_headers: dict[str, str]) -> None: - for key, value in default_headers.items(): - # Ignore Transfer-Encoding if the Content-Length has been set explicitly. - if key.lower() == "transfer-encoding" and "content-length" in self.headers: - continue - self.headers.setdefault(key, value) - - @property - def elapsed(self) -> datetime.timedelta: - """ - Returns the time taken for the complete request/response - cycle to complete. - """ - if not hasattr(self, "_elapsed"): - raise RuntimeError( - "'.elapsed' may only be accessed after the response " - "has been read or closed." - ) - return self._elapsed - - @elapsed.setter - def elapsed(self, elapsed: datetime.timedelta) -> None: - self._elapsed = elapsed - - @property - def request(self) -> Request: - """ - Returns the request instance associated to the current response. - """ - if self._request is None: - raise RuntimeError( - "The request instance has not been set on this response." - ) - return self._request - - @request.setter - def request(self, value: Request) -> None: - self._request = value - - @property - def http_version(self) -> str: - try: - http_version: bytes = self.extensions["http_version"] - except KeyError: - return "HTTP/1.1" - else: - return http_version.decode("ascii", errors="ignore") - - @property - def reason_phrase(self) -> str: - try: - reason_phrase: bytes = self.extensions["reason_phrase"] - except KeyError: - return codes.get_reason_phrase(self.status_code) - else: - return reason_phrase.decode("ascii", errors="ignore") - - @property - def url(self) -> URL: - """ - Returns the URL for which the request was made. - """ - return self.request.url - - @property - def content(self) -> bytes: - if not hasattr(self, "_content"): - raise ResponseNotRead() - return self._content - - @property - def text(self) -> str: - if not hasattr(self, "_text"): - content = self.content - if not content: - self._text = "" - else: - decoder = TextDecoder(encoding=self.encoding or "utf-8") - self._text = "".join([decoder.decode(self.content), decoder.flush()]) - return self._text - - @property - def encoding(self) -> str | None: - """ - Return an encoding to use for decoding the byte content into text. - The priority for determining this is given by... - - * `.encoding = <>` has been set explicitly. - * The encoding as specified by the charset parameter in the Content-Type header. - * The encoding as determined by `default_encoding`, which may either be - a string like "utf-8" indicating the encoding to use, or may be a callable - which enables charset autodetection. - """ - if not hasattr(self, "_encoding"): - encoding = self.charset_encoding - if encoding is None or not is_known_encoding(encoding): - if isinstance(self.default_encoding, str): - encoding = self.default_encoding - elif hasattr(self, "_content"): - encoding = self.default_encoding(self._content) - self._encoding = encoding or "utf-8" - return self._encoding - - @encoding.setter - def encoding(self, value: str) -> None: - """ - Set the encoding to use for decoding the byte content into text. - - If the `text` attribute has been accessed, attempting to set the - encoding will throw a ValueError. - """ - if hasattr(self, "_text"): - raise ValueError( - "Setting encoding after `text` has been accessed is not allowed." - ) - self._encoding = value - - @property - def charset_encoding(self) -> str | None: - """ - Return the encoding, as specified by the Content-Type header. - """ - content_type = self.headers.get("Content-Type") - if content_type is None: - return None - - return parse_content_type_charset(content_type) - - def _get_content_decoder(self) -> ContentDecoder: - """ - Returns a decoder instance which can be used to decode the raw byte - content, depending on the Content-Encoding used in the response. - """ - if not hasattr(self, "_decoder"): - decoders: list[ContentDecoder] = [] - values = self.headers.get_list("content-encoding", split_commas=True) - for value in values: - value = value.strip().lower() - try: - decoder_cls = SUPPORTED_DECODERS[value] - decoders.append(decoder_cls()) - except KeyError: - continue - - if len(decoders) == 1: - self._decoder = decoders[0] - elif len(decoders) > 1: - self._decoder = MultiDecoder(children=decoders) - else: - self._decoder = IdentityDecoder() - - return self._decoder - - @property - def is_informational(self) -> bool: - """ - A property which is `True` for 1xx status codes, `False` otherwise. - """ - return codes.is_informational(self.status_code) - - @property - def is_success(self) -> bool: - """ - A property which is `True` for 2xx status codes, `False` otherwise. - """ - return codes.is_success(self.status_code) - - @property - def is_redirect(self) -> bool: - """ - A property which is `True` for 3xx status codes, `False` otherwise. - - Note that not all responses with a 3xx status code indicate a URL redirect. - - Use `response.has_redirect_location` to determine responses with a properly - formed URL redirection. - """ - return codes.is_redirect(self.status_code) - - @property - def is_client_error(self) -> bool: - """ - A property which is `True` for 4xx status codes, `False` otherwise. - """ - return codes.is_client_error(self.status_code) - - @property - def is_server_error(self) -> bool: - """ - A property which is `True` for 5xx status codes, `False` otherwise. - """ - return codes.is_server_error(self.status_code) - - @property - def is_error(self) -> bool: - """ - A property which is `True` for 4xx and 5xx status codes, `False` otherwise. - """ - return codes.is_error(self.status_code) - - @property - def has_redirect_location(self) -> bool: - """ - Returns True for 3xx responses with a properly formed URL redirection, - `False` otherwise. - """ - return ( - self.status_code - in ( - # 301 (Cacheable redirect. Method may change to GET.) - codes.MOVED_PERMANENTLY, - # 302 (Uncacheable redirect. Method may change to GET.) - codes.FOUND, - # 303 (Client should make a GET or HEAD request.) - codes.SEE_OTHER, - # 307 (Equiv. 302, but retain method) - codes.TEMPORARY_REDIRECT, - # 308 (Equiv. 301, but retain method) - codes.PERMANENT_REDIRECT, - ) - and "Location" in self.headers - ) - - def raise_for_status(self) -> Response: - """ - Raise the `HTTPStatusError` if one occurred. - """ - request = self._request - if request is None: - raise RuntimeError( - "Cannot call `raise_for_status` as the request " - "instance has not been set on this response." - ) - - if self.is_success: - return self - - if self.has_redirect_location: - message = ( - "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" - "Redirect location: '{0.headers[location]}'\n" - "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" - ) - else: - message = ( - "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" - "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" - ) - - status_class = self.status_code // 100 - error_types = { - 1: "Informational response", - 3: "Redirect response", - 4: "Client error", - 5: "Server error", - } - error_type = error_types.get(status_class, "Invalid status code") - message = message.format(self, error_type=error_type) - raise HTTPStatusError(message, request=request, response=self) - - def json(self, **kwargs: typing.Any) -> typing.Any: - return jsonlib.loads(self.content, **kwargs) - - @property - def cookies(self) -> Cookies: - if not hasattr(self, "_cookies"): - self._cookies = Cookies() - self._cookies.extract_cookies(self) - return self._cookies - - @property - def links(self) -> dict[str | None, dict[str, str]]: - """ - Returns the parsed header links of the response, if any - """ - header = self.headers.get("link") - if header is None: - return {} - - return { - (link.get("rel") or link.get("url")): link - for link in parse_header_links(header) - } - - @property - def num_bytes_downloaded(self) -> int: - return self._num_bytes_downloaded - - def __repr__(self) -> str: - return f"<Response [{self.status_code} {self.reason_phrase}]>" - - def __getstate__(self) -> dict[str, typing.Any]: - return { - name: value - for name, value in self.__dict__.items() - if name not in ["extensions", "stream", "is_closed", "_decoder"] - } - - def __setstate__(self, state: dict[str, typing.Any]) -> None: - for name, value in state.items(): - setattr(self, name, value) - self.is_closed = True - self.extensions = {} - self.stream = UnattachedStream() - - def read(self) -> bytes: - """ - Read and return the response content. - """ - if not hasattr(self, "_content"): - self._content = b"".join(self.iter_bytes()) - return self._content - - def iter_bytes(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: - """ - A byte-iterator over the decoded response content. - This allows us to handle gzip, deflate, brotli, and zstd encoded responses. - """ - if hasattr(self, "_content"): - chunk_size = len(self._content) if chunk_size is None else chunk_size - for i in range(0, len(self._content), max(chunk_size, 1)): - yield self._content[i : i + chunk_size] - else: - decoder = self._get_content_decoder() - chunker = ByteChunker(chunk_size=chunk_size) - with request_context(request=self._request): - for raw_bytes in self.iter_raw(): - decoded = decoder.decode(raw_bytes) - for chunk in chunker.decode(decoded): - yield chunk - decoded = decoder.flush() - for chunk in chunker.decode(decoded): - yield chunk # pragma: no cover - for chunk in chunker.flush(): - yield chunk - - def iter_text(self, chunk_size: int | None = None) -> typing.Iterator[str]: - """ - A str-iterator over the decoded response content - that handles both gzip, deflate, etc but also detects the content's - string encoding. - """ - decoder = TextDecoder(encoding=self.encoding or "utf-8") - chunker = TextChunker(chunk_size=chunk_size) - with request_context(request=self._request): - for byte_content in self.iter_bytes(): - text_content = decoder.decode(byte_content) - for chunk in chunker.decode(text_content): - yield chunk - text_content = decoder.flush() - for chunk in chunker.decode(text_content): - yield chunk # pragma: no cover - for chunk in chunker.flush(): - yield chunk - - def iter_lines(self) -> typing.Iterator[str]: - decoder = LineDecoder() - with request_context(request=self._request): - for text in self.iter_text(): - for line in decoder.decode(text): - yield line - for line in decoder.flush(): - yield line - - def iter_raw(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: - """ - A byte-iterator over the raw response content. - """ - if self.is_stream_consumed: - raise StreamConsumed() - if self.is_closed: - raise StreamClosed() - if not isinstance(self.stream, SyncByteStream): - raise RuntimeError("Attempted to call a sync iterator on an async stream.") - - self.is_stream_consumed = True - self._num_bytes_downloaded = 0 - chunker = ByteChunker(chunk_size=chunk_size) - - with request_context(request=self._request): - for raw_stream_bytes in self.stream: - self._num_bytes_downloaded += len(raw_stream_bytes) - for chunk in chunker.decode(raw_stream_bytes): - yield chunk - - for chunk in chunker.flush(): - yield chunk - - self.close() - - def close(self) -> None: - """ - Close the response and release the connection. - Automatically called if the response body is read to completion. - """ - if not isinstance(self.stream, SyncByteStream): - raise RuntimeError("Attempted to call an sync close on an async stream.") - - if not self.is_closed: - self.is_closed = True - with request_context(request=self._request): - self.stream.close() - - async def aread(self) -> bytes: - """ - Read and return the response content. - """ - if not hasattr(self, "_content"): - self._content = b"".join([part async for part in self.aiter_bytes()]) - return self._content - - async def aiter_bytes( - self, chunk_size: int | None = None - ) -> typing.AsyncIterator[bytes]: - """ - A byte-iterator over the decoded response content. - This allows us to handle gzip, deflate, brotli, and zstd encoded responses. - """ - if hasattr(self, "_content"): - chunk_size = len(self._content) if chunk_size is None else chunk_size - for i in range(0, len(self._content), max(chunk_size, 1)): - yield self._content[i : i + chunk_size] - else: - decoder = self._get_content_decoder() - chunker = ByteChunker(chunk_size=chunk_size) - with request_context(request=self._request): - async for raw_bytes in self.aiter_raw(): - decoded = decoder.decode(raw_bytes) - for chunk in chunker.decode(decoded): - yield chunk - decoded = decoder.flush() - for chunk in chunker.decode(decoded): - yield chunk # pragma: no cover - for chunk in chunker.flush(): - yield chunk - - async def aiter_text( - self, chunk_size: int | None = None - ) -> typing.AsyncIterator[str]: - """ - A str-iterator over the decoded response content - that handles both gzip, deflate, etc but also detects the content's - string encoding. - """ - decoder = TextDecoder(encoding=self.encoding or "utf-8") - chunker = TextChunker(chunk_size=chunk_size) - with request_context(request=self._request): - async for byte_content in self.aiter_bytes(): - text_content = decoder.decode(byte_content) - for chunk in chunker.decode(text_content): - yield chunk - text_content = decoder.flush() - for chunk in chunker.decode(text_content): - yield chunk # pragma: no cover - for chunk in chunker.flush(): - yield chunk - - async def aiter_lines(self) -> typing.AsyncIterator[str]: - decoder = LineDecoder() - with request_context(request=self._request): - async for text in self.aiter_text(): - for line in decoder.decode(text): - yield line - for line in decoder.flush(): - yield line - - async def aiter_raw( - self, chunk_size: int | None = None - ) -> typing.AsyncIterator[bytes]: - """ - A byte-iterator over the raw response content. - """ - if self.is_stream_consumed: - raise StreamConsumed() - if self.is_closed: - raise StreamClosed() - if not isinstance(self.stream, AsyncByteStream): - raise RuntimeError("Attempted to call an async iterator on an sync stream.") - - self.is_stream_consumed = True - self._num_bytes_downloaded = 0 - chunker = ByteChunker(chunk_size=chunk_size) - - with request_context(request=self._request): - async for raw_stream_bytes in self.stream: - self._num_bytes_downloaded += len(raw_stream_bytes) - for chunk in chunker.decode(raw_stream_bytes): - yield chunk - - for chunk in chunker.flush(): - yield chunk - - await self.aclose() - - async def aclose(self) -> None: - """ - Close the response and release the connection. - Automatically called if the response body is read to completion. - """ - if not isinstance(self.stream, AsyncByteStream): - raise RuntimeError("Attempted to call an async close on an sync stream.") - - if not self.is_closed: - self.is_closed = True - with request_context(request=self._request): - await self.stream.aclose() - - -class Cookies(typing.MutableMapping[str, str]): - """ - HTTP Cookies, as a mutable mapping. - """ - - def __init__(self, cookies: CookieTypes | None = None) -> None: - if cookies is None or isinstance(cookies, dict): - self.jar = CookieJar() - if isinstance(cookies, dict): - for key, value in cookies.items(): - self.set(key, value) - elif isinstance(cookies, list): - self.jar = CookieJar() - for key, value in cookies: - self.set(key, value) - elif isinstance(cookies, Cookies): - self.jar = CookieJar() - for cookie in cookies.jar: - self.jar.set_cookie(cookie) - else: - self.jar = cookies - - def extract_cookies(self, response: Response) -> None: - """ - Loads any cookies based on the response `Set-Cookie` headers. - """ - urllib_response = self._CookieCompatResponse(response) - urllib_request = self._CookieCompatRequest(response.request) - - self.jar.extract_cookies(urllib_response, urllib_request) # type: ignore - - def set_cookie_header(self, request: Request) -> None: - """ - Sets an appropriate 'Cookie:' HTTP header on the `Request`. - """ - urllib_request = self._CookieCompatRequest(request) - self.jar.add_cookie_header(urllib_request) - - def set(self, name: str, value: str, domain: str = "", path: str = "/") -> None: - """ - Set a cookie value by name. May optionally include domain and path. - """ - kwargs = { - "version": 0, - "name": name, - "value": value, - "port": None, - "port_specified": False, - "domain": domain, - "domain_specified": bool(domain), - "domain_initial_dot": domain.startswith("."), - "path": path, - "path_specified": bool(path), - "secure": False, - "expires": None, - "discard": True, - "comment": None, - "comment_url": None, - "rest": {"HttpOnly": None}, - "rfc2109": False, - } - cookie = Cookie(**kwargs) # type: ignore - self.jar.set_cookie(cookie) - - def get( # type: ignore - self, - name: str, - default: str | None = None, - domain: str | None = None, - path: str | None = None, - ) -> str | None: - """ - Get a cookie by name. May optionally include domain and path - in order to specify exactly which cookie to retrieve. - """ - value = None - for cookie in self.jar: - if cookie.name == name: - if domain is None or cookie.domain == domain: - if path is None or cookie.path == path: - if value is not None: - message = f"Multiple cookies exist with name={name}" - raise CookieConflict(message) - value = cookie.value - - if value is None: - return default - return value - - def delete( - self, - name: str, - domain: str | None = None, - path: str | None = None, - ) -> None: - """ - Delete a cookie by name. May optionally include domain and path - in order to specify exactly which cookie to delete. - """ - if domain is not None and path is not None: - return self.jar.clear(domain, path, name) - - remove = [ - cookie - for cookie in self.jar - if cookie.name == name - and (domain is None or cookie.domain == domain) - and (path is None or cookie.path == path) - ] - - for cookie in remove: - self.jar.clear(cookie.domain, cookie.path, cookie.name) - - def clear(self, domain: str | None = None, path: str | None = None) -> None: - """ - Delete all cookies. Optionally include a domain and path in - order to only delete a subset of all the cookies. - """ - args = [] - if domain is not None: - args.append(domain) - if path is not None: - assert domain is not None - args.append(path) - self.jar.clear(*args) - - def update(self, cookies: CookieTypes | None = None) -> None: # type: ignore - cookies = Cookies(cookies) - for cookie in cookies.jar: - self.jar.set_cookie(cookie) - - def __setitem__(self, name: str, value: str) -> None: - return self.set(name, value) - - def __getitem__(self, name: str) -> str: - value = self.get(name) - if value is None: - raise KeyError(name) - return value - - def __delitem__(self, name: str) -> None: - return self.delete(name) - - def __len__(self) -> int: - return len(self.jar) - - def __iter__(self) -> typing.Iterator[str]: - return (cookie.name for cookie in self.jar) - - def __bool__(self) -> bool: - for _ in self.jar: - return True - return False - - def __repr__(self) -> str: - cookies_repr = ", ".join( - [ - f"<Cookie {cookie.name}={cookie.value} for {cookie.domain} />" - for cookie in self.jar - ] - ) - - return f"<Cookies[{cookies_repr}]>" - - class _CookieCompatRequest(urllib.request.Request): - """ - Wraps a `Request` instance up in a compatibility interface suitable - for use with `CookieJar` operations. - """ - - def __init__(self, request: Request) -> None: - super().__init__( - url=str(request.url), - headers=dict(request.headers), - method=request.method, - ) - self.request = request - - def add_unredirected_header(self, key: str, value: str) -> None: - super().add_unredirected_header(key, value) - self.request.headers[key] = value - - class _CookieCompatResponse: - """ - Wraps a `Request` instance up in a compatibility interface suitable - for use with `CookieJar` operations. - """ - - def __init__(self, response: Response) -> None: - self.response = response - - def info(self) -> email.message.Message: - info = email.message.Message() - for key, value in self.response.headers.multi_items(): - # Note that setting `info[key]` here is an "append" operation, - # not a "replace" operation. - # https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__ - info[key] = value - return info diff --git a/contrib/python/httpx/httpx/_multipart.py b/contrib/python/httpx/httpx/_multipart.py deleted file mode 100644 index 8edb622778a..00000000000 --- a/contrib/python/httpx/httpx/_multipart.py +++ /dev/null @@ -1,269 +0,0 @@ -from __future__ import annotations - -import io -import os -import typing -from pathlib import Path - -from ._types import ( - AsyncByteStream, - FileContent, - FileTypes, - RequestData, - RequestFiles, - SyncByteStream, -) -from ._utils import ( - format_form_param, - guess_content_type, - peek_filelike_length, - primitive_value_to_str, - to_bytes, -) - - -def get_multipart_boundary_from_content_type( - content_type: bytes | None, -) -> bytes | None: - if not content_type or not content_type.startswith(b"multipart/form-data"): - return None - # parse boundary according to - # https://www.rfc-editor.org/rfc/rfc2046#section-5.1.1 - if b";" in content_type: - for section in content_type.split(b";"): - if section.strip().lower().startswith(b"boundary="): - return section.strip()[len(b"boundary=") :].strip(b'"') - return None - - -class DataField: - """ - A single form field item, within a multipart form field. - """ - - def __init__(self, name: str, value: str | bytes | int | float | None) -> None: - if not isinstance(name, str): - raise TypeError( - f"Invalid type for name. Expected str, got {type(name)}: {name!r}" - ) - if value is not None and not isinstance(value, (str, bytes, int, float)): - raise TypeError( - "Invalid type for value. Expected primitive type," - f" got {type(value)}: {value!r}" - ) - self.name = name - self.value: str | bytes = ( - value if isinstance(value, bytes) else primitive_value_to_str(value) - ) - - def render_headers(self) -> bytes: - if not hasattr(self, "_headers"): - name = format_form_param("name", self.name) - self._headers = b"".join( - [b"Content-Disposition: form-data; ", name, b"\r\n\r\n"] - ) - - return self._headers - - def render_data(self) -> bytes: - if not hasattr(self, "_data"): - self._data = to_bytes(self.value) - - return self._data - - def get_length(self) -> int: - headers = self.render_headers() - data = self.render_data() - return len(headers) + len(data) - - def render(self) -> typing.Iterator[bytes]: - yield self.render_headers() - yield self.render_data() - - -class FileField: - """ - A single file field item, within a multipart form field. - """ - - CHUNK_SIZE = 64 * 1024 - - def __init__(self, name: str, value: FileTypes) -> None: - self.name = name - - fileobj: FileContent - - headers: dict[str, str] = {} - content_type: str | None = None - - # This large tuple based API largely mirror's requests' API - # It would be good to think of better APIs for this that we could - # include in httpx 2.0 since variable length tuples(especially of 4 elements) - # are quite unwieldly - if isinstance(value, tuple): - if len(value) == 2: - # neither the 3rd parameter (content_type) nor the 4th (headers) - # was included - filename, fileobj = value - elif len(value) == 3: - filename, fileobj, content_type = value - else: - # all 4 parameters included - filename, fileobj, content_type, headers = value # type: ignore - else: - filename = Path(str(getattr(value, "name", "upload"))).name - fileobj = value - - if content_type is None: - content_type = guess_content_type(filename) - - has_content_type_header = any("content-type" in key.lower() for key in headers) - if content_type is not None and not has_content_type_header: - # note that unlike requests, we ignore the content_type provided in the 3rd - # tuple element if it is also included in the headers requests does - # the opposite (it overwrites the headerwith the 3rd tuple element) - headers["Content-Type"] = content_type - - if isinstance(fileobj, io.StringIO): - raise TypeError( - "Multipart file uploads require 'io.BytesIO', not 'io.StringIO'." - ) - if isinstance(fileobj, io.TextIOBase): - raise TypeError( - "Multipart file uploads must be opened in binary mode, not text mode." - ) - - self.filename = filename - self.file = fileobj - self.headers = headers - - def get_length(self) -> int | None: - headers = self.render_headers() - - if isinstance(self.file, (str, bytes)): - return len(headers) + len(to_bytes(self.file)) - - file_length = peek_filelike_length(self.file) - - # If we can't determine the filesize without reading it into memory, - # then return `None` here, to indicate an unknown file length. - if file_length is None: - return None - - return len(headers) + file_length - - def render_headers(self) -> bytes: - if not hasattr(self, "_headers"): - parts = [ - b"Content-Disposition: form-data; ", - format_form_param("name", self.name), - ] - if self.filename: - filename = format_form_param("filename", self.filename) - parts.extend([b"; ", filename]) - for header_name, header_value in self.headers.items(): - key, val = f"\r\n{header_name}: ".encode(), header_value.encode() - parts.extend([key, val]) - parts.append(b"\r\n\r\n") - self._headers = b"".join(parts) - - return self._headers - - def render_data(self) -> typing.Iterator[bytes]: - if isinstance(self.file, (str, bytes)): - yield to_bytes(self.file) - return - - if hasattr(self.file, "seek"): - try: - self.file.seek(0) - except io.UnsupportedOperation: - pass - - chunk = self.file.read(self.CHUNK_SIZE) - while chunk: - yield to_bytes(chunk) - chunk = self.file.read(self.CHUNK_SIZE) - - def render(self) -> typing.Iterator[bytes]: - yield self.render_headers() - yield from self.render_data() - - -class MultipartStream(SyncByteStream, AsyncByteStream): - """ - Request content as streaming multipart encoded form data. - """ - - def __init__( - self, - data: RequestData, - files: RequestFiles, - boundary: bytes | None = None, - ) -> None: - if boundary is None: - boundary = os.urandom(16).hex().encode("ascii") - - self.boundary = boundary - self.content_type = "multipart/form-data; boundary=%s" % boundary.decode( - "ascii" - ) - self.fields = list(self._iter_fields(data, files)) - - def _iter_fields( - self, data: RequestData, files: RequestFiles - ) -> typing.Iterator[FileField | DataField]: - for name, value in data.items(): - if isinstance(value, (tuple, list)): - for item in value: - yield DataField(name=name, value=item) - else: - yield DataField(name=name, value=value) - - file_items = files.items() if isinstance(files, typing.Mapping) else files - for name, value in file_items: - yield FileField(name=name, value=value) - - def iter_chunks(self) -> typing.Iterator[bytes]: - for field in self.fields: - yield b"--%s\r\n" % self.boundary - yield from field.render() - yield b"\r\n" - yield b"--%s--\r\n" % self.boundary - - def get_content_length(self) -> int | None: - """ - Return the length of the multipart encoded content, or `None` if - any of the files have a length that cannot be determined upfront. - """ - boundary_length = len(self.boundary) - length = 0 - - for field in self.fields: - field_length = field.get_length() - if field_length is None: - return None - - length += 2 + boundary_length + 2 # b"--{boundary}\r\n" - length += field_length - length += 2 # b"\r\n" - - length += 2 + boundary_length + 4 # b"--{boundary}--\r\n" - return length - - # Content stream interface. - - def get_headers(self) -> dict[str, str]: - content_length = self.get_content_length() - content_type = self.content_type - if content_length is None: - return {"Transfer-Encoding": "chunked", "Content-Type": content_type} - return {"Content-Length": str(content_length), "Content-Type": content_type} - - def __iter__(self) -> typing.Iterator[bytes]: - for chunk in self.iter_chunks(): - yield chunk - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - for chunk in self.iter_chunks(): - yield chunk diff --git a/contrib/python/httpx/httpx/_status_codes.py b/contrib/python/httpx/httpx/_status_codes.py deleted file mode 100644 index 133a6231a5b..00000000000 --- a/contrib/python/httpx/httpx/_status_codes.py +++ /dev/null @@ -1,162 +0,0 @@ -from __future__ import annotations - -from enum import IntEnum - -__all__ = ["codes"] - - -class codes(IntEnum): - """HTTP status codes and reason phrases - - Status codes from the following RFCs are all observed: - - * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 - * RFC 6585: Additional HTTP Status Codes - * RFC 3229: Delta encoding in HTTP - * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 - * RFC 5842: Binding Extensions to WebDAV - * RFC 7238: Permanent Redirect - * RFC 2295: Transparent Content Negotiation in HTTP - * RFC 2774: An HTTP Extension Framework - * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) - * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) - * RFC 7725: An HTTP Status Code to Report Legal Obstacles - * RFC 8297: An HTTP Status Code for Indicating Hints - * RFC 8470: Using Early Data in HTTP - """ - - def __new__(cls, value: int, phrase: str = "") -> codes: - obj = int.__new__(cls, value) - obj._value_ = value - - obj.phrase = phrase # type: ignore[attr-defined] - return obj - - def __str__(self) -> str: - return str(self.value) - - @classmethod - def get_reason_phrase(cls, value: int) -> str: - try: - return codes(value).phrase # type: ignore - except ValueError: - return "" - - @classmethod - def is_informational(cls, value: int) -> bool: - """ - Returns `True` for 1xx status codes, `False` otherwise. - """ - return 100 <= value <= 199 - - @classmethod - def is_success(cls, value: int) -> bool: - """ - Returns `True` for 2xx status codes, `False` otherwise. - """ - return 200 <= value <= 299 - - @classmethod - def is_redirect(cls, value: int) -> bool: - """ - Returns `True` for 3xx status codes, `False` otherwise. - """ - return 300 <= value <= 399 - - @classmethod - def is_client_error(cls, value: int) -> bool: - """ - Returns `True` for 4xx status codes, `False` otherwise. - """ - return 400 <= value <= 499 - - @classmethod - def is_server_error(cls, value: int) -> bool: - """ - Returns `True` for 5xx status codes, `False` otherwise. - """ - return 500 <= value <= 599 - - @classmethod - def is_error(cls, value: int) -> bool: - """ - Returns `True` for 4xx or 5xx status codes, `False` otherwise. - """ - return 400 <= value <= 599 - - # informational - CONTINUE = 100, "Continue" - SWITCHING_PROTOCOLS = 101, "Switching Protocols" - PROCESSING = 102, "Processing" - EARLY_HINTS = 103, "Early Hints" - - # success - OK = 200, "OK" - CREATED = 201, "Created" - ACCEPTED = 202, "Accepted" - NON_AUTHORITATIVE_INFORMATION = 203, "Non-Authoritative Information" - NO_CONTENT = 204, "No Content" - RESET_CONTENT = 205, "Reset Content" - PARTIAL_CONTENT = 206, "Partial Content" - MULTI_STATUS = 207, "Multi-Status" - ALREADY_REPORTED = 208, "Already Reported" - IM_USED = 226, "IM Used" - - # redirection - MULTIPLE_CHOICES = 300, "Multiple Choices" - MOVED_PERMANENTLY = 301, "Moved Permanently" - FOUND = 302, "Found" - SEE_OTHER = 303, "See Other" - NOT_MODIFIED = 304, "Not Modified" - USE_PROXY = 305, "Use Proxy" - TEMPORARY_REDIRECT = 307, "Temporary Redirect" - PERMANENT_REDIRECT = 308, "Permanent Redirect" - - # client error - BAD_REQUEST = 400, "Bad Request" - UNAUTHORIZED = 401, "Unauthorized" - PAYMENT_REQUIRED = 402, "Payment Required" - FORBIDDEN = 403, "Forbidden" - NOT_FOUND = 404, "Not Found" - METHOD_NOT_ALLOWED = 405, "Method Not Allowed" - NOT_ACCEPTABLE = 406, "Not Acceptable" - PROXY_AUTHENTICATION_REQUIRED = 407, "Proxy Authentication Required" - REQUEST_TIMEOUT = 408, "Request Timeout" - CONFLICT = 409, "Conflict" - GONE = 410, "Gone" - LENGTH_REQUIRED = 411, "Length Required" - PRECONDITION_FAILED = 412, "Precondition Failed" - REQUEST_ENTITY_TOO_LARGE = 413, "Request Entity Too Large" - REQUEST_URI_TOO_LONG = 414, "Request-URI Too Long" - UNSUPPORTED_MEDIA_TYPE = 415, "Unsupported Media Type" - REQUESTED_RANGE_NOT_SATISFIABLE = 416, "Requested Range Not Satisfiable" - EXPECTATION_FAILED = 417, "Expectation Failed" - IM_A_TEAPOT = 418, "I'm a teapot" - MISDIRECTED_REQUEST = 421, "Misdirected Request" - UNPROCESSABLE_ENTITY = 422, "Unprocessable Entity" - LOCKED = 423, "Locked" - FAILED_DEPENDENCY = 424, "Failed Dependency" - TOO_EARLY = 425, "Too Early" - UPGRADE_REQUIRED = 426, "Upgrade Required" - PRECONDITION_REQUIRED = 428, "Precondition Required" - TOO_MANY_REQUESTS = 429, "Too Many Requests" - REQUEST_HEADER_FIELDS_TOO_LARGE = 431, "Request Header Fields Too Large" - UNAVAILABLE_FOR_LEGAL_REASONS = 451, "Unavailable For Legal Reasons" - - # server errors - INTERNAL_SERVER_ERROR = 500, "Internal Server Error" - NOT_IMPLEMENTED = 501, "Not Implemented" - BAD_GATEWAY = 502, "Bad Gateway" - SERVICE_UNAVAILABLE = 503, "Service Unavailable" - GATEWAY_TIMEOUT = 504, "Gateway Timeout" - HTTP_VERSION_NOT_SUPPORTED = 505, "HTTP Version Not Supported" - VARIANT_ALSO_NEGOTIATES = 506, "Variant Also Negotiates" - INSUFFICIENT_STORAGE = 507, "Insufficient Storage" - LOOP_DETECTED = 508, "Loop Detected" - NOT_EXTENDED = 510, "Not Extended" - NETWORK_AUTHENTICATION_REQUIRED = 511, "Network Authentication Required" - - -# Include lower-case styles for `requests` compatibility. -for code in codes: - setattr(codes, code._name_.lower(), int(code)) diff --git a/contrib/python/httpx/httpx/_transports/__init__.py b/contrib/python/httpx/httpx/_transports/__init__.py deleted file mode 100644 index 7a321053b29..00000000000 --- a/contrib/python/httpx/httpx/_transports/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .asgi import * -from .base import * -from .default import * -from .mock import * -from .wsgi import * - -__all__ = [ - "ASGITransport", - "AsyncBaseTransport", - "BaseTransport", - "AsyncHTTPTransport", - "HTTPTransport", - "MockTransport", - "WSGITransport", -] diff --git a/contrib/python/httpx/httpx/_transports/asgi.py b/contrib/python/httpx/httpx/_transports/asgi.py deleted file mode 100644 index 8578d4aeff2..00000000000 --- a/contrib/python/httpx/httpx/_transports/asgi.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import annotations - -import typing - -import sniffio - -from .._models import Request, Response -from .._types import AsyncByteStream -from .base import AsyncBaseTransport - -if typing.TYPE_CHECKING: # pragma: no cover - import asyncio - - import trio - - Event = typing.Union[asyncio.Event, trio.Event] - - -_Message = typing.MutableMapping[str, typing.Any] -_Receive = typing.Callable[[], typing.Awaitable[_Message]] -_Send = typing.Callable[ - [typing.MutableMapping[str, typing.Any]], typing.Awaitable[None] -] -_ASGIApp = typing.Callable[ - [typing.MutableMapping[str, typing.Any], _Receive, _Send], typing.Awaitable[None] -] - -__all__ = ["ASGITransport"] - - -def create_event() -> Event: - if sniffio.current_async_library() == "trio": - import trio - - return trio.Event() - else: - import asyncio - - return asyncio.Event() - - -class ASGIResponseStream(AsyncByteStream): - def __init__(self, body: list[bytes]) -> None: - self._body = body - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - yield b"".join(self._body) - - -class ASGITransport(AsyncBaseTransport): - """ - A custom AsyncTransport that handles sending requests directly to an ASGI app. - - ```python - transport = httpx.ASGITransport( - app=app, - root_path="/submount", - client=("1.2.3.4", 123) - ) - client = httpx.AsyncClient(transport=transport) - ``` - - Arguments: - - * `app` - The ASGI application. - * `raise_app_exceptions` - Boolean indicating if exceptions in the application - should be raised. Default to `True`. Can be set to `False` for use cases - such as testing the content of a client 500 response. - * `root_path` - The root path on which the ASGI application should be mounted. - * `client` - A two-tuple indicating the client IP and port of incoming requests. - ``` - """ - - def __init__( - self, - app: _ASGIApp, - raise_app_exceptions: bool = True, - root_path: str = "", - client: tuple[str, int] = ("127.0.0.1", 123), - ) -> None: - self.app = app - self.raise_app_exceptions = raise_app_exceptions - self.root_path = root_path - self.client = client - - async def handle_async_request( - self, - request: Request, - ) -> Response: - assert isinstance(request.stream, AsyncByteStream) - - # ASGI scope. - scope = { - "type": "http", - "asgi": {"version": "3.0"}, - "http_version": "1.1", - "method": request.method, - "headers": [(k.lower(), v) for (k, v) in request.headers.raw], - "scheme": request.url.scheme, - "path": request.url.path, - "raw_path": request.url.raw_path.split(b"?")[0], - "query_string": request.url.query, - "server": (request.url.host, request.url.port), - "client": self.client, - "root_path": self.root_path, - } - - # Request. - request_body_chunks = request.stream.__aiter__() - request_complete = False - - # Response. - status_code = None - response_headers = None - body_parts = [] - response_started = False - response_complete = create_event() - - # ASGI callables. - - async def receive() -> dict[str, typing.Any]: - nonlocal request_complete - - if request_complete: - await response_complete.wait() - return {"type": "http.disconnect"} - - try: - body = await request_body_chunks.__anext__() - except StopAsyncIteration: - request_complete = True - return {"type": "http.request", "body": b"", "more_body": False} - return {"type": "http.request", "body": body, "more_body": True} - - async def send(message: typing.MutableMapping[str, typing.Any]) -> None: - nonlocal status_code, response_headers, response_started - - if message["type"] == "http.response.start": - assert not response_started - - status_code = message["status"] - response_headers = message.get("headers", []) - response_started = True - - elif message["type"] == "http.response.body": - assert not response_complete.is_set() - body = message.get("body", b"") - more_body = message.get("more_body", False) - - if body and request.method != "HEAD": - body_parts.append(body) - - if not more_body: - response_complete.set() - - try: - await self.app(scope, receive, send) - except Exception: # noqa: PIE-786 - if self.raise_app_exceptions: - raise - - response_complete.set() - if status_code is None: - status_code = 500 - if response_headers is None: - response_headers = {} - - assert response_complete.is_set() - assert status_code is not None - assert response_headers is not None - - stream = ASGIResponseStream(body_parts) - - return Response(status_code, headers=response_headers, stream=stream) diff --git a/contrib/python/httpx/httpx/_transports/base.py b/contrib/python/httpx/httpx/_transports/base.py deleted file mode 100644 index 66fd99d7024..00000000000 --- a/contrib/python/httpx/httpx/_transports/base.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -import typing -from types import TracebackType - -from .._models import Request, Response - -T = typing.TypeVar("T", bound="BaseTransport") -A = typing.TypeVar("A", bound="AsyncBaseTransport") - -__all__ = ["AsyncBaseTransport", "BaseTransport"] - - -class BaseTransport: - def __enter__(self: T) -> T: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - self.close() - - def handle_request(self, request: Request) -> Response: - """ - Send a single HTTP request and return a response. - - Developers shouldn't typically ever need to call into this API directly, - since the Client class provides all the higher level user-facing API - niceties. - - In order to properly release any network resources, the response - stream should *either* be consumed immediately, with a call to - `response.stream.read()`, or else the `handle_request` call should - be followed with a try/finally block to ensuring the stream is - always closed. - - Example usage: - - with httpx.HTTPTransport() as transport: - req = httpx.Request( - method=b"GET", - url=(b"https", b"www.example.com", 443, b"/"), - headers=[(b"Host", b"www.example.com")], - ) - resp = transport.handle_request(req) - body = resp.stream.read() - print(resp.status_code, resp.headers, body) - - - Takes a `Request` instance as the only argument. - - Returns a `Response` instance. - """ - raise NotImplementedError( - "The 'handle_request' method must be implemented." - ) # pragma: no cover - - def close(self) -> None: - pass - - -class AsyncBaseTransport: - async def __aenter__(self: A) -> A: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - await self.aclose() - - async def handle_async_request( - self, - request: Request, - ) -> Response: - raise NotImplementedError( - "The 'handle_async_request' method must be implemented." - ) # pragma: no cover - - async def aclose(self) -> None: - pass diff --git a/contrib/python/httpx/httpx/_transports/default.py b/contrib/python/httpx/httpx/_transports/default.py deleted file mode 100644 index 33db416dd19..00000000000 --- a/contrib/python/httpx/httpx/_transports/default.py +++ /dev/null @@ -1,389 +0,0 @@ -""" -Custom transports, with nicely configured defaults. - -The following additional keyword arguments are currently supported by httpcore... - -* uds: str -* local_address: str -* retries: int - -Example usages... - -# Disable HTTP/2 on a single specific domain. -mounts = { - "all://": httpx.HTTPTransport(http2=True), - "all://*example.org": httpx.HTTPTransport() -} - -# Using advanced httpcore configuration, with connection retries. -transport = httpx.HTTPTransport(retries=1) -client = httpx.Client(transport=transport) - -# Using advanced httpcore configuration, with unix domain sockets. -transport = httpx.HTTPTransport(uds="socket.uds") -client = httpx.Client(transport=transport) -""" - -from __future__ import annotations - -import contextlib -import typing -from types import TracebackType - -import httpcore - -from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context -from .._exceptions import ( - ConnectError, - ConnectTimeout, - LocalProtocolError, - NetworkError, - PoolTimeout, - ProtocolError, - ProxyError, - ReadError, - ReadTimeout, - RemoteProtocolError, - TimeoutException, - UnsupportedProtocol, - WriteError, - WriteTimeout, -) -from .._models import Request, Response -from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream, VerifyTypes -from .._urls import URL -from .base import AsyncBaseTransport, BaseTransport - -T = typing.TypeVar("T", bound="HTTPTransport") -A = typing.TypeVar("A", bound="AsyncHTTPTransport") - -SOCKET_OPTION = typing.Union[ - typing.Tuple[int, int, int], - typing.Tuple[int, int, typing.Union[bytes, bytearray]], - typing.Tuple[int, int, None, int], -] - -__all__ = ["AsyncHTTPTransport", "HTTPTransport"] - - -def map_httpcore_exceptions() -> typing.Iterator[None]: - try: - yield - except Exception as exc: - mapped_exc = None - - for from_exc, to_exc in HTTPCORE_EXC_MAP.items(): - if not isinstance(exc, from_exc): - continue - # We want to map to the most specific exception we can find. - # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to - # `httpx.ReadTimeout`, not just `httpx.TimeoutException`. - if mapped_exc is None or issubclass(to_exc, mapped_exc): - mapped_exc = to_exc - - if mapped_exc is None: # pragma: no cover - raise - - message = str(exc) - raise mapped_exc(message) from exc - - -HTTPCORE_EXC_MAP = { - httpcore.TimeoutException: TimeoutException, - httpcore.ConnectTimeout: ConnectTimeout, - httpcore.ReadTimeout: ReadTimeout, - httpcore.WriteTimeout: WriteTimeout, - httpcore.PoolTimeout: PoolTimeout, - httpcore.NetworkError: NetworkError, - httpcore.ConnectError: ConnectError, - httpcore.ReadError: ReadError, - httpcore.WriteError: WriteError, - httpcore.ProxyError: ProxyError, - httpcore.UnsupportedProtocol: UnsupportedProtocol, - httpcore.ProtocolError: ProtocolError, - httpcore.LocalProtocolError: LocalProtocolError, - httpcore.RemoteProtocolError: RemoteProtocolError, -} - - -class ResponseStream(SyncByteStream): - def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None: - self._httpcore_stream = httpcore_stream - - def __iter__(self) -> typing.Iterator[bytes]: - with map_httpcore_exceptions(): - for part in self._httpcore_stream: - yield part - - def close(self) -> None: - if hasattr(self._httpcore_stream, "close"): - self._httpcore_stream.close() - - -class HTTPTransport(BaseTransport): - def __init__( - self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, - proxy: ProxyTypes | None = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - - if proxy is None: - self._pool = httpcore.ConnectionPool( - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - uds=uds, - local_address=local_address, - retries=retries, - socket_options=socket_options, - ) - elif proxy.url.scheme in ("http", "https"): - self._pool = httpcore.HTTPProxy( - proxy_url=httpcore.URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - ssl_context=ssl_context, - proxy_ssl_context=proxy.ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - elif proxy.url.scheme == "socks5": - try: - import socksio # noqa - except ImportError: # pragma: no cover - raise ImportError( - "Using SOCKS proxy, but the 'socksio' package is not installed. " - "Make sure to install httpx using `pip install httpx[socks]`." - ) from None - - self._pool = httpcore.SOCKSProxy( - proxy_url=httpcore.URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - ) - else: # pragma: no cover - raise ValueError( - "Proxy protocol must be either 'http', 'https', or 'socks5'," - f" but got {proxy.url.scheme!r}." - ) - - def __enter__(self: T) -> T: # Use generics for subclass support. - self._pool.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - with map_httpcore_exceptions(): - self._pool.__exit__(exc_type, exc_value, traceback) - - def handle_request( - self, - request: Request, - ) -> Response: - assert isinstance(request.stream, SyncByteStream) - - req = httpcore.Request( - method=request.method, - url=httpcore.URL( - scheme=request.url.raw_scheme, - host=request.url.raw_host, - port=request.url.port, - target=request.url.raw_path, - ), - headers=request.headers.raw, - content=request.stream, - extensions=request.extensions, - ) - with map_httpcore_exceptions(): - resp = self._pool.handle_request(req) - - assert isinstance(resp.stream, typing.Iterable) - - return Response( - status_code=resp.status, - headers=resp.headers, - stream=ResponseStream(resp.stream), - extensions=resp.extensions, - ) - - def close(self) -> None: - self._pool.close() - - -class AsyncResponseStream(AsyncByteStream): - def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None: - self._httpcore_stream = httpcore_stream - - async def __aiter__(self) -> typing.AsyncIterator[bytes]: - with map_httpcore_exceptions(): - async for part in self._httpcore_stream: - yield part - - async def aclose(self) -> None: - if hasattr(self._httpcore_stream, "aclose"): - await self._httpcore_stream.aclose() - - -class AsyncHTTPTransport(AsyncBaseTransport): - def __init__( - self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, - http1: bool = True, - http2: bool = False, - limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, - proxy: ProxyTypes | None = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options: typing.Iterable[SOCKET_OPTION] | None = None, - ) -> None: - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - - if proxy is None: - self._pool = httpcore.AsyncConnectionPool( - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - uds=uds, - local_address=local_address, - retries=retries, - socket_options=socket_options, - ) - elif proxy.url.scheme in ("http", "https"): - self._pool = httpcore.AsyncHTTPProxy( - proxy_url=httpcore.URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - proxy_ssl_context=proxy.ssl_context, - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - elif proxy.url.scheme == "socks5": - try: - import socksio # noqa - except ImportError: # pragma: no cover - raise ImportError( - "Using SOCKS proxy, but the 'socksio' package is not installed. " - "Make sure to install httpx using `pip install httpx[socks]`." - ) from None - - self._pool = httpcore.AsyncSOCKSProxy( - proxy_url=httpcore.URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - ) - else: # pragma: no cover - raise ValueError( - "Proxy protocol must be either 'http', 'https', or 'socks5'," - " but got {proxy.url.scheme!r}." - ) - - async def __aenter__(self: A) -> A: # Use generics for subclass support. - await self._pool.__aenter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None = None, - exc_value: BaseException | None = None, - traceback: TracebackType | None = None, - ) -> None: - with map_httpcore_exceptions(): - await self._pool.__aexit__(exc_type, exc_value, traceback) - - async def handle_async_request( - self, - request: Request, - ) -> Response: - assert isinstance(request.stream, AsyncByteStream) - - req = httpcore.Request( - method=request.method, - url=httpcore.URL( - scheme=request.url.raw_scheme, - host=request.url.raw_host, - port=request.url.port, - target=request.url.raw_path, - ), - headers=request.headers.raw, - content=request.stream, - extensions=request.extensions, - ) - with map_httpcore_exceptions(): - resp = await self._pool.handle_async_request(req) - - assert isinstance(resp.stream, typing.AsyncIterable) - - return Response( - status_code=resp.status, - headers=resp.headers, - stream=AsyncResponseStream(resp.stream), - extensions=resp.extensions, - ) - - async def aclose(self) -> None: - await self._pool.aclose() diff --git a/contrib/python/httpx/httpx/_transports/mock.py b/contrib/python/httpx/httpx/_transports/mock.py deleted file mode 100644 index 8c418f59e06..00000000000 --- a/contrib/python/httpx/httpx/_transports/mock.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -import typing - -from .._models import Request, Response -from .base import AsyncBaseTransport, BaseTransport - -SyncHandler = typing.Callable[[Request], Response] -AsyncHandler = typing.Callable[[Request], typing.Coroutine[None, None, Response]] - - -__all__ = ["MockTransport"] - - -class MockTransport(AsyncBaseTransport, BaseTransport): - def __init__(self, handler: SyncHandler | AsyncHandler) -> None: - self.handler = handler - - def handle_request( - self, - request: Request, - ) -> Response: - request.read() - response = self.handler(request) - if not isinstance(response, Response): # pragma: no cover - raise TypeError("Cannot use an async handler in a sync Client") - return response - - async def handle_async_request( - self, - request: Request, - ) -> Response: - await request.aread() - response = self.handler(request) - - # Allow handler to *optionally* be an `async` function. - # If it is, then the `response` variable need to be awaited to actually - # return the result. - - if not isinstance(response, Response): - response = await response - - return response diff --git a/contrib/python/httpx/httpx/_transports/wsgi.py b/contrib/python/httpx/httpx/_transports/wsgi.py deleted file mode 100644 index 8592ffe017a..00000000000 --- a/contrib/python/httpx/httpx/_transports/wsgi.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations - -import io -import itertools -import sys -import typing - -from .._models import Request, Response -from .._types import SyncByteStream -from .base import BaseTransport - -if typing.TYPE_CHECKING: - from _typeshed import OptExcInfo # pragma: no cover - from _typeshed.wsgi import WSGIApplication # pragma: no cover - -_T = typing.TypeVar("_T") - - -__all__ = ["WSGITransport"] - - -def _skip_leading_empty_chunks(body: typing.Iterable[_T]) -> typing.Iterable[_T]: - body = iter(body) - for chunk in body: - if chunk: - return itertools.chain([chunk], body) - return [] - - -class WSGIByteStream(SyncByteStream): - def __init__(self, result: typing.Iterable[bytes]) -> None: - self._close = getattr(result, "close", None) - self._result = _skip_leading_empty_chunks(result) - - def __iter__(self) -> typing.Iterator[bytes]: - for part in self._result: - yield part - - def close(self) -> None: - if self._close is not None: - self._close() - - -class WSGITransport(BaseTransport): - """ - A custom transport that handles sending requests directly to an WSGI app. - The simplest way to use this functionality is to use the `app` argument. - - ``` - client = httpx.Client(app=app) - ``` - - Alternatively, you can setup the transport instance explicitly. - This allows you to include any additional configuration arguments specific - to the WSGITransport class: - - ``` - transport = httpx.WSGITransport( - app=app, - script_name="/submount", - remote_addr="1.2.3.4" - ) - client = httpx.Client(transport=transport) - ``` - - Arguments: - - * `app` - The WSGI application. - * `raise_app_exceptions` - Boolean indicating if exceptions in the application - should be raised. Default to `True`. Can be set to `False` for use cases - such as testing the content of a client 500 response. - * `script_name` - The root path on which the WSGI application should be mounted. - * `remote_addr` - A string indicating the client IP of incoming requests. - ``` - """ - - def __init__( - self, - app: WSGIApplication, - raise_app_exceptions: bool = True, - script_name: str = "", - remote_addr: str = "127.0.0.1", - wsgi_errors: typing.TextIO | None = None, - ) -> None: - self.app = app - self.raise_app_exceptions = raise_app_exceptions - self.script_name = script_name - self.remote_addr = remote_addr - self.wsgi_errors = wsgi_errors - - def handle_request(self, request: Request) -> Response: - request.read() - wsgi_input = io.BytesIO(request.content) - - port = request.url.port or {"http": 80, "https": 443}[request.url.scheme] - environ = { - "wsgi.version": (1, 0), - "wsgi.url_scheme": request.url.scheme, - "wsgi.input": wsgi_input, - "wsgi.errors": self.wsgi_errors or sys.stderr, - "wsgi.multithread": True, - "wsgi.multiprocess": False, - "wsgi.run_once": False, - "REQUEST_METHOD": request.method, - "SCRIPT_NAME": self.script_name, - "PATH_INFO": request.url.path, - "QUERY_STRING": request.url.query.decode("ascii"), - "SERVER_NAME": request.url.host, - "SERVER_PORT": str(port), - "SERVER_PROTOCOL": "HTTP/1.1", - "REMOTE_ADDR": self.remote_addr, - } - for header_key, header_value in request.headers.raw: - key = header_key.decode("ascii").upper().replace("-", "_") - if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): - key = "HTTP_" + key - environ[key] = header_value.decode("ascii") - - seen_status = None - seen_response_headers = None - seen_exc_info = None - - def start_response( - status: str, - response_headers: list[tuple[str, str]], - exc_info: OptExcInfo | None = None, - ) -> typing.Callable[[bytes], typing.Any]: - nonlocal seen_status, seen_response_headers, seen_exc_info - seen_status = status - seen_response_headers = response_headers - seen_exc_info = exc_info - return lambda _: None - - result = self.app(environ, start_response) - - stream = WSGIByteStream(result) - - assert seen_status is not None - assert seen_response_headers is not None - if seen_exc_info and seen_exc_info[0] and self.raise_app_exceptions: - raise seen_exc_info[1] - - status_code = int(seen_status.split()[0]) - headers = [ - (key.encode("ascii"), value.encode("ascii")) - for key, value in seen_response_headers - ] - - return Response(status_code, headers=headers, stream=stream) diff --git a/contrib/python/httpx/httpx/_types.py b/contrib/python/httpx/httpx/_types.py deleted file mode 100644 index 661af262e70..00000000000 --- a/contrib/python/httpx/httpx/_types.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Type definitions for type checking purposes. -""" - -import ssl -from http.cookiejar import CookieJar -from typing import ( - IO, - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - NamedTuple, - Optional, - Sequence, - Tuple, - Union, -) - -if TYPE_CHECKING: # pragma: no cover - from ._auth import Auth # noqa: F401 - from ._config import Proxy, Timeout # noqa: F401 - from ._models import Cookies, Headers, Request # noqa: F401 - from ._urls import URL, QueryParams # noqa: F401 - - -PrimitiveData = Optional[Union[str, int, float, bool]] - -RawURL = NamedTuple( - "RawURL", - [ - ("raw_scheme", bytes), - ("raw_host", bytes), - ("port", Optional[int]), - ("raw_path", bytes), - ], -) - -URLTypes = Union["URL", str] - -QueryParamTypes = Union[ - "QueryParams", - Mapping[str, Union[PrimitiveData, Sequence[PrimitiveData]]], - List[Tuple[str, PrimitiveData]], - Tuple[Tuple[str, PrimitiveData], ...], - str, - bytes, -] - -HeaderTypes = Union[ - "Headers", - Mapping[str, str], - Mapping[bytes, bytes], - Sequence[Tuple[str, str]], - Sequence[Tuple[bytes, bytes]], -] - -CookieTypes = Union["Cookies", CookieJar, Dict[str, str], List[Tuple[str, str]]] - -CertTypes = Union[ - # certfile - str, - # (certfile, keyfile) - Tuple[str, Optional[str]], - # (certfile, keyfile, password) - Tuple[str, Optional[str], Optional[str]], -] -VerifyTypes = Union[str, bool, ssl.SSLContext] -TimeoutTypes = Union[ - Optional[float], - Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], - "Timeout", -] -ProxyTypes = Union["URL", str, "Proxy"] -ProxiesTypes = Union[ProxyTypes, Dict[Union["URL", str], Union[None, ProxyTypes]]] - -AuthTypes = Union[ - Tuple[Union[str, bytes], Union[str, bytes]], - Callable[["Request"], "Request"], - "Auth", -] - -RequestContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] -ResponseContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] -ResponseExtensions = MutableMapping[str, Any] - -RequestData = Mapping[str, Any] - -FileContent = Union[IO[bytes], bytes, str] -FileTypes = Union[ - # file (or bytes) - FileContent, - # (filename, file (or bytes)) - Tuple[Optional[str], FileContent], - # (filename, file (or bytes), content_type) - Tuple[Optional[str], FileContent, Optional[str]], - # (filename, file (or bytes), content_type, headers) - Tuple[Optional[str], FileContent, Optional[str], Mapping[str, str]], -] -RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]] - -RequestExtensions = MutableMapping[str, Any] - -__all__ = ["AsyncByteStream", "SyncByteStream"] - - -class SyncByteStream: - def __iter__(self) -> Iterator[bytes]: - raise NotImplementedError( - "The '__iter__' method must be implemented." - ) # pragma: no cover - yield b"" # pragma: no cover - - def close(self) -> None: - """ - Subclasses can override this method to release any network resources - after a request/response cycle is complete. - """ - - -class AsyncByteStream: - async def __aiter__(self) -> AsyncIterator[bytes]: - raise NotImplementedError( - "The '__aiter__' method must be implemented." - ) # pragma: no cover - yield b"" # pragma: no cover - - async def aclose(self) -> None: - pass diff --git a/contrib/python/httpx/httpx/_urlparse.py b/contrib/python/httpx/httpx/_urlparse.py deleted file mode 100644 index 479c2ef8a1d..00000000000 --- a/contrib/python/httpx/httpx/_urlparse.py +++ /dev/null @@ -1,505 +0,0 @@ -""" -An implementation of `urlparse` that provides URL validation and normalization -as described by RFC3986. - -We rely on this implementation rather than the one in Python's stdlib, because: - -* It provides more complete URL validation. -* It properly differentiates between an empty querystring and an absent querystring, - to distinguish URLs with a trailing '?'. -* It handles scheme, hostname, port, and path normalization. -* It supports IDNA hostnames, normalizing them to their encoded form. -* The API supports passing individual components, as well as the complete URL string. - -Previously we relied on the excellent `rfc3986` package to handle URL parsing and -validation, but this module provides a simpler alternative, with less indirection -required. -""" - -from __future__ import annotations - -import ipaddress -import re -import typing - -import idna - -from ._exceptions import InvalidURL - -MAX_URL_LENGTH = 65536 - -# https://datatracker.ietf.org/doc/html/rfc3986.html#section-2.3 -UNRESERVED_CHARACTERS = ( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" -) -SUB_DELIMS = "!$&'()*+,;=" - -PERCENT_ENCODED_REGEX = re.compile("%[A-Fa-f0-9]{2}") - - -# {scheme}: (optional) -# //{authority} (optional) -# {path} -# ?{query} (optional) -# #{fragment} (optional) -URL_REGEX = re.compile( - ( - r"(?:(?P<scheme>{scheme}):)?" - r"(?://(?P<authority>{authority}))?" - r"(?P<path>{path})" - r"(?:\?(?P<query>{query}))?" - r"(?:#(?P<fragment>{fragment}))?" - ).format( - scheme="([a-zA-Z][a-zA-Z0-9+.-]*)?", - authority="[^/?#]*", - path="[^?#]*", - query="[^#]*", - fragment=".*", - ) -) - -# {userinfo}@ (optional) -# {host} -# :{port} (optional) -AUTHORITY_REGEX = re.compile( - ( - r"(?:(?P<userinfo>{userinfo})@)?" r"(?P<host>{host})" r":?(?P<port>{port})?" - ).format( - userinfo=".*", # Any character sequence. - host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@', - # or an IPv6 address enclosed within square brackets. - port=".*", # Any character sequence. - ) -) - - -# If we call urlparse with an individual component, then we need to regex -# validate that component individually. -# Note that we're duplicating the same strings as above. Shock! Horror!! -COMPONENT_REGEX = { - "scheme": re.compile("([a-zA-Z][a-zA-Z0-9+.-]*)?"), - "authority": re.compile("[^/?#]*"), - "path": re.compile("[^?#]*"), - "query": re.compile("[^#]*"), - "fragment": re.compile(".*"), - "userinfo": re.compile("[^@]*"), - "host": re.compile("(\\[.*\\]|[^:]*)"), - "port": re.compile(".*"), -} - - -# We use these simple regexs as a first pass before handing off to -# the stdlib 'ipaddress' module for IP address validation. -IPv4_STYLE_HOSTNAME = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") -IPv6_STYLE_HOSTNAME = re.compile(r"^\[.*\]$") - - -class ParseResult(typing.NamedTuple): - scheme: str - userinfo: str - host: str - port: int | None - path: str - query: str | None - fragment: str | None - - @property - def authority(self) -> str: - return "".join( - [ - f"{self.userinfo}@" if self.userinfo else "", - f"[{self.host}]" if ":" in self.host else self.host, - f":{self.port}" if self.port is not None else "", - ] - ) - - @property - def netloc(self) -> str: - return "".join( - [ - f"[{self.host}]" if ":" in self.host else self.host, - f":{self.port}" if self.port is not None else "", - ] - ) - - def copy_with(self, **kwargs: str | None) -> ParseResult: - if not kwargs: - return self - - defaults = { - "scheme": self.scheme, - "authority": self.authority, - "path": self.path, - "query": self.query, - "fragment": self.fragment, - } - defaults.update(kwargs) - return urlparse("", **defaults) - - def __str__(self) -> str: - authority = self.authority - return "".join( - [ - f"{self.scheme}:" if self.scheme else "", - f"//{authority}" if authority else "", - self.path, - f"?{self.query}" if self.query is not None else "", - f"#{self.fragment}" if self.fragment is not None else "", - ] - ) - - -def urlparse(url: str = "", **kwargs: str | None) -> ParseResult: - # Initial basic checks on allowable URLs. - # --------------------------------------- - - # Hard limit the maximum allowable URL length. - if len(url) > MAX_URL_LENGTH: - raise InvalidURL("URL too long") - - # If a URL includes any ASCII control characters including \t, \r, \n, - # then treat it as invalid. - if any(char.isascii() and not char.isprintable() for char in url): - char = next(char for char in url if char.isascii() and not char.isprintable()) - idx = url.find(char) - error = ( - f"Invalid non-printable ASCII character in URL, {char!r} at position {idx}." - ) - raise InvalidURL(error) - - # Some keyword arguments require special handling. - # ------------------------------------------------ - - # Coerce "port" to a string, if it is provided as an integer. - if "port" in kwargs: - port = kwargs["port"] - kwargs["port"] = str(port) if isinstance(port, int) else port - - # Replace "netloc" with "host and "port". - if "netloc" in kwargs: - netloc = kwargs.pop("netloc") or "" - kwargs["host"], _, kwargs["port"] = netloc.partition(":") - - # Replace "username" and/or "password" with "userinfo". - if "username" in kwargs or "password" in kwargs: - username = quote(kwargs.pop("username", "") or "") - password = quote(kwargs.pop("password", "") or "") - kwargs["userinfo"] = f"{username}:{password}" if password else username - - # Replace "raw_path" with "path" and "query". - if "raw_path" in kwargs: - raw_path = kwargs.pop("raw_path") or "" - kwargs["path"], seperator, kwargs["query"] = raw_path.partition("?") - if not seperator: - kwargs["query"] = None - - # Ensure that IPv6 "host" addresses are always escaped with "[...]". - if "host" in kwargs: - host = kwargs.get("host") or "" - if ":" in host and not (host.startswith("[") and host.endswith("]")): - kwargs["host"] = f"[{host}]" - - # If any keyword arguments are provided, ensure they are valid. - # ------------------------------------------------------------- - - for key, value in kwargs.items(): - if value is not None: - if len(value) > MAX_URL_LENGTH: - raise InvalidURL(f"URL component '{key}' too long") - - # If a component includes any ASCII control characters including \t, \r, \n, - # then treat it as invalid. - if any(char.isascii() and not char.isprintable() for char in value): - char = next( - char for char in value if char.isascii() and not char.isprintable() - ) - idx = value.find(char) - error = ( - f"Invalid non-printable ASCII character in URL {key} component, " - f"{char!r} at position {idx}." - ) - raise InvalidURL(error) - - # Ensure that keyword arguments match as a valid regex. - if not COMPONENT_REGEX[key].fullmatch(value): - raise InvalidURL(f"Invalid URL component '{key}'") - - # The URL_REGEX will always match, but may have empty components. - url_match = URL_REGEX.match(url) - assert url_match is not None - url_dict = url_match.groupdict() - - # * 'scheme', 'authority', and 'path' may be empty strings. - # * 'query' may be 'None', indicating no trailing "?" portion. - # Any string including the empty string, indicates a trailing "?". - # * 'fragment' may be 'None', indicating no trailing "#" portion. - # Any string including the empty string, indicates a trailing "#". - scheme = kwargs.get("scheme", url_dict["scheme"]) or "" - authority = kwargs.get("authority", url_dict["authority"]) or "" - path = kwargs.get("path", url_dict["path"]) or "" - query = kwargs.get("query", url_dict["query"]) - fragment = kwargs.get("fragment", url_dict["fragment"]) - - # The AUTHORITY_REGEX will always match, but may have empty components. - authority_match = AUTHORITY_REGEX.match(authority) - assert authority_match is not None - authority_dict = authority_match.groupdict() - - # * 'userinfo' and 'host' may be empty strings. - # * 'port' may be 'None'. - userinfo = kwargs.get("userinfo", authority_dict["userinfo"]) or "" - host = kwargs.get("host", authority_dict["host"]) or "" - port = kwargs.get("port", authority_dict["port"]) - - # Normalize and validate each component. - # We end up with a parsed representation of the URL, - # with components that are plain ASCII bytestrings. - parsed_scheme: str = scheme.lower() - parsed_userinfo: str = quote(userinfo, safe=SUB_DELIMS + ":") - parsed_host: str = encode_host(host) - parsed_port: int | None = normalize_port(port, scheme) - - has_scheme = parsed_scheme != "" - has_authority = ( - parsed_userinfo != "" or parsed_host != "" or parsed_port is not None - ) - validate_path(path, has_scheme=has_scheme, has_authority=has_authority) - if has_scheme or has_authority: - path = normalize_path(path) - - # The GEN_DELIMS set is... : / ? # [ ] @ - # These do not need to be percent-quoted unless they serve as delimiters for the - # specific component. - WHATWG_SAFE = '`{}%|^\\"' - - # For 'path' we need to drop ? and # from the GEN_DELIMS set. - parsed_path: str = quote(path, safe=SUB_DELIMS + WHATWG_SAFE + ":/[]@") - # For 'query' we need to drop '#' from the GEN_DELIMS set. - parsed_query: str | None = ( - None - if query is None - else quote(query, safe=SUB_DELIMS + WHATWG_SAFE + ":/?[]@") - ) - # For 'fragment' we can include all of the GEN_DELIMS set. - parsed_fragment: str | None = ( - None - if fragment is None - else quote(fragment, safe=SUB_DELIMS + WHATWG_SAFE + ":/?#[]@") - ) - - # The parsed ASCII bytestrings are our canonical form. - # All properties of the URL are derived from these. - return ParseResult( - parsed_scheme, - parsed_userinfo, - parsed_host, - parsed_port, - parsed_path, - parsed_query, - parsed_fragment, - ) - - -def encode_host(host: str) -> str: - if not host: - return "" - - elif IPv4_STYLE_HOSTNAME.match(host): - # Validate IPv4 hostnames like #.#.#.# - # - # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 - # - # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet - try: - ipaddress.IPv4Address(host) - except ipaddress.AddressValueError: - raise InvalidURL(f"Invalid IPv4 address: {host!r}") - return host - - elif IPv6_STYLE_HOSTNAME.match(host): - # Validate IPv6 hostnames like [...] - # - # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 - # - # "A host identified by an Internet Protocol literal address, version 6 - # [RFC3513] or later, is distinguished by enclosing the IP literal - # within square brackets ("[" and "]"). This is the only place where - # square bracket characters are allowed in the URI syntax." - try: - ipaddress.IPv6Address(host[1:-1]) - except ipaddress.AddressValueError: - raise InvalidURL(f"Invalid IPv6 address: {host!r}") - return host[1:-1] - - elif host.isascii(): - # Regular ASCII hostnames - # - # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 - # - # reg-name = *( unreserved / pct-encoded / sub-delims ) - WHATWG_SAFE = '"`{}%|\\' - return quote(host.lower(), safe=SUB_DELIMS + WHATWG_SAFE) - - # IDNA hostnames - try: - return idna.encode(host.lower()).decode("ascii") - except idna.IDNAError: - raise InvalidURL(f"Invalid IDNA hostname: {host!r}") - - -def normalize_port(port: str | int | None, scheme: str) -> int | None: - # From https://tools.ietf.org/html/rfc3986#section-3.2.3 - # - # "A scheme may define a default port. For example, the "http" scheme - # defines a default port of "80", corresponding to its reserved TCP - # port number. The type of port designated by the port number (e.g., - # TCP, UDP, SCTP) is defined by the URI scheme. URI producers and - # normalizers should omit the port component and its ":" delimiter if - # port is empty or if its value would be the same as that of the - # scheme's default." - if port is None or port == "": - return None - - try: - port_as_int = int(port) - except ValueError: - raise InvalidURL(f"Invalid port: {port!r}") - - # See https://url.spec.whatwg.org/#url-miscellaneous - default_port = {"ftp": 21, "http": 80, "https": 443, "ws": 80, "wss": 443}.get( - scheme - ) - if port_as_int == default_port: - return None - return port_as_int - - -def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None: - """ - Path validation rules that depend on if the URL contains - a scheme or authority component. - - See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3 - """ - if has_authority: - # If a URI contains an authority component, then the path component - # must either be empty or begin with a slash ("/") character." - if path and not path.startswith("/"): - raise InvalidURL("For absolute URLs, path must be empty or begin with '/'") - - if not has_scheme and not has_authority: - # If a URI does not contain an authority component, then the path cannot begin - # with two slash characters ("//"). - if path.startswith("//"): - raise InvalidURL("Relative URLs cannot have a path starting with '//'") - - # In addition, a URI reference (Section 4.1) may be a relative-path reference, - # in which case the first path segment cannot contain a colon (":") character. - if path.startswith(":"): - raise InvalidURL("Relative URLs cannot have a path starting with ':'") - - -def normalize_path(path: str) -> str: - """ - Drop "." and ".." segments from a URL path. - - For example: - - normalize_path("/path/./to/somewhere/..") == "/path/to" - """ - # Fast return when no '.' characters in the path. - if "." not in path: - return path - - components = path.split("/") - - # Fast return when no '.' or '..' components in the path. - if "." not in components and ".." not in components: - return path - - # https://datatracker.ietf.org/doc/html/rfc3986#section-5.2.4 - output: list[str] = [] - for component in components: - if component == ".": - pass - elif component == "..": - if output and output != [""]: - output.pop() - else: - output.append(component) - return "/".join(output) - - -def PERCENT(string: str) -> str: - return "".join([f"%{byte:02X}" for byte in string.encode("utf-8")]) - - -def percent_encoded(string: str, safe: str = "/") -> str: - """ - Use percent-encoding to quote a string. - """ - NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe - - # Fast path for strings that don't need escaping. - if not string.rstrip(NON_ESCAPED_CHARS): - return string - - return "".join( - [char if char in NON_ESCAPED_CHARS else PERCENT(char) for char in string] - ) - - -def quote(string: str, safe: str = "/") -> str: - """ - Use percent-encoding to quote a string, omitting existing '%xx' escape sequences. - - See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1 - - * `string`: The string to be percent-escaped. - * `safe`: A string containing characters that may be treated as safe, and do not - need to be escaped. Unreserved characters are always treated as safe. - See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3 - """ - parts = [] - current_position = 0 - for match in re.finditer(PERCENT_ENCODED_REGEX, string): - start_position, end_position = match.start(), match.end() - matched_text = match.group(0) - # Add any text up to the '%xx' escape sequence. - if start_position != current_position: - leading_text = string[current_position:start_position] - parts.append(percent_encoded(leading_text, safe=safe)) - - # Add the '%xx' escape sequence. - parts.append(matched_text) - current_position = end_position - - # Add any text after the final '%xx' escape sequence. - if current_position != len(string): - trailing_text = string[current_position:] - parts.append(percent_encoded(trailing_text, safe=safe)) - - return "".join(parts) - - -def urlencode(items: list[tuple[str, str]]) -> str: - """ - We can use a much simpler version of the stdlib urlencode here because - we don't need to handle a bunch of different typing cases, such as bytes vs str. - - https://github.com/python/cpython/blob/b2f7b2ef0b5421e01efb8c7bee2ef95d3bab77eb/Lib/urllib/parse.py#L926 - - Note that we use '%20' encoding for spaces. and '%2F for '/'. - This is slightly different than `requests`, but is the behaviour that browsers use. - - See - - https://github.com/encode/httpx/issues/2536 - - https://github.com/encode/httpx/issues/2721 - - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode - """ - return "&".join( - [ - percent_encoded(k, safe="") + "=" + percent_encoded(v, safe="") - for k, v in items - ] - ) diff --git a/contrib/python/httpx/httpx/_urls.py b/contrib/python/httpx/httpx/_urls.py deleted file mode 100644 index ec4ea6b399e..00000000000 --- a/contrib/python/httpx/httpx/_urls.py +++ /dev/null @@ -1,648 +0,0 @@ -from __future__ import annotations - -import typing -from urllib.parse import parse_qs, unquote - -import idna - -from ._types import QueryParamTypes, RawURL -from ._urlparse import urlencode, urlparse -from ._utils import primitive_value_to_str - -__all__ = ["URL", "QueryParams"] - - -class URL: - """ - url = httpx.URL("HTTPS://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink") - - assert url.scheme == "https" - assert url.username == "[email protected]" - assert url.password == "a secret" - assert url.userinfo == b"jo%40email.com:a%20secret" - assert url.host == "müller.de" - assert url.raw_host == b"xn--mller-kva.de" - assert url.port == 1234 - assert url.netloc == b"xn--mller-kva.de:1234" - assert url.path == "/pa th" - assert url.query == b"?search=ab" - assert url.raw_path == b"/pa%20th?search=ab" - assert url.fragment == "anchorlink" - - The components of a URL are broken down like this: - - https://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink - [scheme] [ username ] [password] [ host ][port][ path ] [ query ] [fragment] - [ userinfo ] [ netloc ][ raw_path ] - - Note that: - - * `url.scheme` is normalized to always be lowercased. - - * `url.host` is normalized to always be lowercased. Internationalized domain - names are represented in unicode, without IDNA encoding applied. For instance: - - url = httpx.URL("http://中国.icom.museum") - assert url.host == "中国.icom.museum" - url = httpx.URL("http://xn--fiqs8s.icom.museum") - assert url.host == "中国.icom.museum" - - * `url.raw_host` is normalized to always be lowercased, and is IDNA encoded. - - url = httpx.URL("http://中国.icom.museum") - assert url.raw_host == b"xn--fiqs8s.icom.museum" - url = httpx.URL("http://xn--fiqs8s.icom.museum") - assert url.raw_host == b"xn--fiqs8s.icom.museum" - - * `url.port` is either None or an integer. URLs that include the default port for - "http", "https", "ws", "wss", and "ftp" schemes have their port - normalized to `None`. - - assert httpx.URL("http://example.com") == httpx.URL("http://example.com:80") - assert httpx.URL("http://example.com").port is None - assert httpx.URL("http://example.com:80").port is None - - * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work - with `url.username` and `url.password` instead, which handle the URL escaping. - - * `url.raw_path` is raw bytes of both the path and query, without URL escaping. - This portion is used as the target when constructing HTTP requests. Usually you'll - want to work with `url.path` instead. - - * `url.query` is raw bytes, without URL escaping. A URL query string portion can - only be properly URL escaped when decoding the parameter names and values - themselves. - """ - - def __init__(self, url: URL | str = "", **kwargs: typing.Any) -> None: - if kwargs: - allowed = { - "scheme": str, - "username": str, - "password": str, - "userinfo": bytes, - "host": str, - "port": int, - "netloc": bytes, - "path": str, - "query": bytes, - "raw_path": bytes, - "fragment": str, - "params": object, - } - - # Perform type checking for all supported keyword arguments. - for key, value in kwargs.items(): - if key not in allowed: - message = f"{key!r} is an invalid keyword argument for URL()" - raise TypeError(message) - if value is not None and not isinstance(value, allowed[key]): - expected = allowed[key].__name__ - seen = type(value).__name__ - message = f"Argument {key!r} must be {expected} but got {seen}" - raise TypeError(message) - if isinstance(value, bytes): - kwargs[key] = value.decode("ascii") - - if "params" in kwargs: - # Replace any "params" keyword with the raw "query" instead. - # - # Ensure that empty params use `kwargs["query"] = None` rather - # than `kwargs["query"] = ""`, so that generated URLs do not - # include an empty trailing "?". - params = kwargs.pop("params") - kwargs["query"] = None if not params else str(QueryParams(params)) - - if isinstance(url, str): - self._uri_reference = urlparse(url, **kwargs) - elif isinstance(url, URL): - self._uri_reference = url._uri_reference.copy_with(**kwargs) - else: - raise TypeError( - "Invalid type for url. Expected str or httpx.URL," - f" got {type(url)}: {url!r}" - ) - - @property - def scheme(self) -> str: - """ - The URL scheme, such as "http", "https". - Always normalised to lowercase. - """ - return self._uri_reference.scheme - - @property - def raw_scheme(self) -> bytes: - """ - The raw bytes representation of the URL scheme, such as b"http", b"https". - Always normalised to lowercase. - """ - return self._uri_reference.scheme.encode("ascii") - - @property - def userinfo(self) -> bytes: - """ - The URL userinfo as a raw bytestring. - For example: b"jo%40email.com:a%20secret". - """ - return self._uri_reference.userinfo.encode("ascii") - - @property - def username(self) -> str: - """ - The URL username as a string, with URL decoding applied. - For example: "[email protected]" - """ - userinfo = self._uri_reference.userinfo - return unquote(userinfo.partition(":")[0]) - - @property - def password(self) -> str: - """ - The URL password as a string, with URL decoding applied. - For example: "a secret" - """ - userinfo = self._uri_reference.userinfo - return unquote(userinfo.partition(":")[2]) - - @property - def host(self) -> str: - """ - The URL host as a string. - Always normalized to lowercase, with IDNA hosts decoded into unicode. - - Examples: - - url = httpx.URL("http://www.EXAMPLE.org") - assert url.host == "www.example.org" - - url = httpx.URL("http://中国.icom.museum") - assert url.host == "中国.icom.museum" - - url = httpx.URL("http://xn--fiqs8s.icom.museum") - assert url.host == "中国.icom.museum" - - url = httpx.URL("https://[::ffff:192.168.0.1]") - assert url.host == "::ffff:192.168.0.1" - """ - host: str = self._uri_reference.host - - if host.startswith("xn--"): - host = idna.decode(host) - - return host - - @property - def raw_host(self) -> bytes: - """ - The raw bytes representation of the URL host. - Always normalized to lowercase, and IDNA encoded. - - Examples: - - url = httpx.URL("http://www.EXAMPLE.org") - assert url.raw_host == b"www.example.org" - - url = httpx.URL("http://中国.icom.museum") - assert url.raw_host == b"xn--fiqs8s.icom.museum" - - url = httpx.URL("http://xn--fiqs8s.icom.museum") - assert url.raw_host == b"xn--fiqs8s.icom.museum" - - url = httpx.URL("https://[::ffff:192.168.0.1]") - assert url.raw_host == b"::ffff:192.168.0.1" - """ - return self._uri_reference.host.encode("ascii") - - @property - def port(self) -> int | None: - """ - The URL port as an integer. - - Note that the URL class performs port normalization as per the WHATWG spec. - Default ports for "http", "https", "ws", "wss", and "ftp" schemes are always - treated as `None`. - - For example: - - assert httpx.URL("http://www.example.com") == httpx.URL("http://www.example.com:80") - assert httpx.URL("http://www.example.com:80").port is None - """ - return self._uri_reference.port - - @property - def netloc(self) -> bytes: - """ - Either `<host>` or `<host>:<port>` as bytes. - Always normalized to lowercase, and IDNA encoded. - - This property may be used for generating the value of a request - "Host" header. - """ - return self._uri_reference.netloc.encode("ascii") - - @property - def path(self) -> str: - """ - The URL path as a string. Excluding the query string, and URL decoded. - - For example: - - url = httpx.URL("https://example.com/pa%20th") - assert url.path == "/pa th" - """ - path = self._uri_reference.path or "/" - return unquote(path) - - @property - def query(self) -> bytes: - """ - The URL query string, as raw bytes, excluding the leading b"?". - - This is necessarily a bytewise interface, because we cannot - perform URL decoding of this representation until we've parsed - the keys and values into a QueryParams instance. - - For example: - - url = httpx.URL("https://example.com/?filter=some%20search%20terms") - assert url.query == b"filter=some%20search%20terms" - """ - query = self._uri_reference.query or "" - return query.encode("ascii") - - @property - def params(self) -> QueryParams: - """ - The URL query parameters, neatly parsed and packaged into an immutable - multidict representation. - """ - return QueryParams(self._uri_reference.query) - - @property - def raw_path(self) -> bytes: - """ - The complete URL path and query string as raw bytes. - Used as the target when constructing HTTP requests. - - For example: - - GET /users?search=some%20text HTTP/1.1 - Host: www.example.org - Connection: close - """ - path = self._uri_reference.path or "/" - if self._uri_reference.query is not None: - path += "?" + self._uri_reference.query - return path.encode("ascii") - - @property - def fragment(self) -> str: - """ - The URL fragments, as used in HTML anchors. - As a string, without the leading '#'. - """ - return unquote(self._uri_reference.fragment or "") - - @property - def raw(self) -> RawURL: - """ - Provides the (scheme, host, port, target) for the outgoing request. - - In older versions of `httpx` this was used in the low-level transport API. - We no longer use `RawURL`, and this property will be deprecated - in a future release. - """ - return RawURL( - self.raw_scheme, - self.raw_host, - self.port, - self.raw_path, - ) - - @property - def is_absolute_url(self) -> bool: - """ - Return `True` for absolute URLs such as 'http://example.com/path', - and `False` for relative URLs such as '/path'. - """ - # We don't use `.is_absolute` from `rfc3986` because it treats - # URLs with a fragment portion as not absolute. - # What we actually care about is if the URL provides - # a scheme and hostname to which connections should be made. - return bool(self._uri_reference.scheme and self._uri_reference.host) - - @property - def is_relative_url(self) -> bool: - """ - Return `False` for absolute URLs such as 'http://example.com/path', - and `True` for relative URLs such as '/path'. - """ - return not self.is_absolute_url - - def copy_with(self, **kwargs: typing.Any) -> URL: - """ - Copy this URL, returning a new URL with some components altered. - Accepts the same set of parameters as the components that are made - available via properties on the `URL` class. - - For example: - - url = httpx.URL("https://www.example.com").copy_with( - username="[email protected]", password="a secret" - ) - assert url == "https://jo%40email.com:a%[email protected]" - """ - return URL(self, **kwargs) - - def copy_set_param(self, key: str, value: typing.Any = None) -> URL: - return self.copy_with(params=self.params.set(key, value)) - - def copy_add_param(self, key: str, value: typing.Any = None) -> URL: - return self.copy_with(params=self.params.add(key, value)) - - def copy_remove_param(self, key: str) -> URL: - return self.copy_with(params=self.params.remove(key)) - - def copy_merge_params(self, params: QueryParamTypes) -> URL: - return self.copy_with(params=self.params.merge(params)) - - def join(self, url: URL | str) -> URL: - """ - Return an absolute URL, using this URL as the base. - - Eg. - - url = httpx.URL("https://www.example.com/test") - url = url.join("/new/path") - assert url == "https://www.example.com/new/path" - """ - from urllib.parse import urljoin - - return URL(urljoin(str(self), str(URL(url)))) - - def __hash__(self) -> int: - return hash(str(self)) - - def __eq__(self, other: typing.Any) -> bool: - return isinstance(other, (URL, str)) and str(self) == str(URL(other)) - - def __str__(self) -> str: - return str(self._uri_reference) - - def __repr__(self) -> str: - scheme, userinfo, host, port, path, query, fragment = self._uri_reference - - if ":" in userinfo: - # Mask any password component. - userinfo = f'{userinfo.split(":")[0]}:[secure]' - - authority = "".join( - [ - f"{userinfo}@" if userinfo else "", - f"[{host}]" if ":" in host else host, - f":{port}" if port is not None else "", - ] - ) - url = "".join( - [ - f"{self.scheme}:" if scheme else "", - f"//{authority}" if authority else "", - path, - f"?{query}" if query is not None else "", - f"#{fragment}" if fragment is not None else "", - ] - ) - - return f"{self.__class__.__name__}({url!r})" - - -class QueryParams(typing.Mapping[str, str]): - """ - URL query parameters, as a multi-dict. - """ - - def __init__(self, *args: QueryParamTypes | None, **kwargs: typing.Any) -> None: - assert len(args) < 2, "Too many arguments." - assert not (args and kwargs), "Cannot mix named and unnamed arguments." - - value = args[0] if args else kwargs - - if value is None or isinstance(value, (str, bytes)): - value = value.decode("ascii") if isinstance(value, bytes) else value - self._dict = parse_qs(value, keep_blank_values=True) - elif isinstance(value, QueryParams): - self._dict = {k: list(v) for k, v in value._dict.items()} - else: - dict_value: dict[typing.Any, list[typing.Any]] = {} - if isinstance(value, (list, tuple)): - # Convert list inputs like: - # [("a", "123"), ("a", "456"), ("b", "789")] - # To a dict representation, like: - # {"a": ["123", "456"], "b": ["789"]} - for item in value: - dict_value.setdefault(item[0], []).append(item[1]) - else: - # Convert dict inputs like: - # {"a": "123", "b": ["456", "789"]} - # To dict inputs where values are always lists, like: - # {"a": ["123"], "b": ["456", "789"]} - dict_value = { - k: list(v) if isinstance(v, (list, tuple)) else [v] - for k, v in value.items() - } - - # Ensure that keys and values are neatly coerced to strings. - # We coerce values `True` and `False` to JSON-like "true" and "false" - # representations, and coerce `None` values to the empty string. - self._dict = { - str(k): [primitive_value_to_str(item) for item in v] - for k, v in dict_value.items() - } - - def keys(self) -> typing.KeysView[str]: - """ - Return all the keys in the query params. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert list(q.keys()) == ["a", "b"] - """ - return self._dict.keys() - - def values(self) -> typing.ValuesView[str]: - """ - Return all the values in the query params. If a key occurs more than once - only the first item for that key is returned. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert list(q.values()) == ["123", "789"] - """ - return {k: v[0] for k, v in self._dict.items()}.values() - - def items(self) -> typing.ItemsView[str, str]: - """ - Return all items in the query params. If a key occurs more than once - only the first item for that key is returned. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert list(q.items()) == [("a", "123"), ("b", "789")] - """ - return {k: v[0] for k, v in self._dict.items()}.items() - - def multi_items(self) -> list[tuple[str, str]]: - """ - Return all items in the query params. Allow duplicate keys to occur. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert list(q.multi_items()) == [("a", "123"), ("a", "456"), ("b", "789")] - """ - multi_items: list[tuple[str, str]] = [] - for k, v in self._dict.items(): - multi_items.extend([(k, i) for i in v]) - return multi_items - - def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: - """ - Get a value from the query param for a given key. If the key occurs - more than once, then only the first value is returned. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert q.get("a") == "123" - """ - if key in self._dict: - return self._dict[str(key)][0] - return default - - def get_list(self, key: str) -> list[str]: - """ - Get all values from the query param for a given key. - - Usage: - - q = httpx.QueryParams("a=123&a=456&b=789") - assert q.get_list("a") == ["123", "456"] - """ - return list(self._dict.get(str(key), [])) - - def set(self, key: str, value: typing.Any = None) -> QueryParams: - """ - Return a new QueryParams instance, setting the value of a key. - - Usage: - - q = httpx.QueryParams("a=123") - q = q.set("a", "456") - assert q == httpx.QueryParams("a=456") - """ - q = QueryParams() - q._dict = dict(self._dict) - q._dict[str(key)] = [primitive_value_to_str(value)] - return q - - def add(self, key: str, value: typing.Any = None) -> QueryParams: - """ - Return a new QueryParams instance, setting or appending the value of a key. - - Usage: - - q = httpx.QueryParams("a=123") - q = q.add("a", "456") - assert q == httpx.QueryParams("a=123&a=456") - """ - q = QueryParams() - q._dict = dict(self._dict) - q._dict[str(key)] = q.get_list(key) + [primitive_value_to_str(value)] - return q - - def remove(self, key: str) -> QueryParams: - """ - Return a new QueryParams instance, removing the value of a key. - - Usage: - - q = httpx.QueryParams("a=123") - q = q.remove("a") - assert q == httpx.QueryParams("") - """ - q = QueryParams() - q._dict = dict(self._dict) - q._dict.pop(str(key), None) - return q - - def merge(self, params: QueryParamTypes | None = None) -> QueryParams: - """ - Return a new QueryParams instance, updated with. - - Usage: - - q = httpx.QueryParams("a=123") - q = q.merge({"b": "456"}) - assert q == httpx.QueryParams("a=123&b=456") - - q = httpx.QueryParams("a=123") - q = q.merge({"a": "456", "b": "789"}) - assert q == httpx.QueryParams("a=456&b=789") - """ - q = QueryParams(params) - q._dict = {**self._dict, **q._dict} - return q - - def __getitem__(self, key: typing.Any) -> str: - return self._dict[key][0] - - def __contains__(self, key: typing.Any) -> bool: - return key in self._dict - - def __iter__(self) -> typing.Iterator[typing.Any]: - return iter(self.keys()) - - def __len__(self) -> int: - return len(self._dict) - - def __bool__(self) -> bool: - return bool(self._dict) - - def __hash__(self) -> int: - return hash(str(self)) - - def __eq__(self, other: typing.Any) -> bool: - if not isinstance(other, self.__class__): - return False - return sorted(self.multi_items()) == sorted(other.multi_items()) - - def __str__(self) -> str: - """ - Note that we use '%20' encoding for spaces, and treat '/' as a safe - character. - - See https://github.com/encode/httpx/issues/2536 and - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode - """ - return urlencode(self.multi_items()) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - query_string = str(self) - return f"{class_name}({query_string!r})" - - def update(self, params: QueryParamTypes | None = None) -> None: - raise RuntimeError( - "QueryParams are immutable since 0.18.0. " - "Use `q = q.merge(...)` to create an updated copy." - ) - - def __setitem__(self, key: str, value: str) -> None: - raise RuntimeError( - "QueryParams are immutable since 0.18.0. " - "Use `q = q.set(key, value)` to create an updated copy." - ) diff --git a/contrib/python/httpx/httpx/_utils.py b/contrib/python/httpx/httpx/_utils.py deleted file mode 100644 index a9ece194387..00000000000 --- a/contrib/python/httpx/httpx/_utils.py +++ /dev/null @@ -1,440 +0,0 @@ -from __future__ import annotations - -import codecs -import email.message -import ipaddress -import mimetypes -import os -import re -import time -import typing -from pathlib import Path -from urllib.request import getproxies - -import sniffio - -from ._types import PrimitiveData - -if typing.TYPE_CHECKING: # pragma: no cover - from ._urls import URL - - -_HTML5_FORM_ENCODING_REPLACEMENTS = {'"': "%22", "\\": "\\\\"} -_HTML5_FORM_ENCODING_REPLACEMENTS.update( - {chr(c): "%{:02X}".format(c) for c in range(0x1F + 1) if c != 0x1B} -) -_HTML5_FORM_ENCODING_RE = re.compile( - r"|".join([re.escape(c) for c in _HTML5_FORM_ENCODING_REPLACEMENTS.keys()]) -) - - -def normalize_header_key( - value: str | bytes, - lower: bool, - encoding: str | None = None, -) -> bytes: - """ - Coerce str/bytes into a strictly byte-wise HTTP header key. - """ - if isinstance(value, bytes): - bytes_value = value - else: - bytes_value = value.encode(encoding or "ascii") - - return bytes_value.lower() if lower else bytes_value - - -def normalize_header_value(value: str | bytes, encoding: str | None = None) -> bytes: - """ - Coerce str/bytes into a strictly byte-wise HTTP header value. - """ - if isinstance(value, bytes): - return value - return value.encode(encoding or "ascii") - - -def primitive_value_to_str(value: PrimitiveData) -> str: - """ - Coerce a primitive data type into a string value. - - Note that we prefer JSON-style 'true'/'false' for boolean values here. - """ - if value is True: - return "true" - elif value is False: - return "false" - elif value is None: - return "" - return str(value) - - -def is_known_encoding(encoding: str) -> bool: - """ - Return `True` if `encoding` is a known codec. - """ - try: - codecs.lookup(encoding) - except LookupError: - return False - return True - - -def format_form_param(name: str, value: str) -> bytes: - """ - Encode a name/value pair within a multipart form. - """ - - def replacer(match: typing.Match[str]) -> str: - return _HTML5_FORM_ENCODING_REPLACEMENTS[match.group(0)] - - value = _HTML5_FORM_ENCODING_RE.sub(replacer, value) - return f'{name}="{value}"'.encode() - - -def get_ca_bundle_from_env() -> str | None: - if "SSL_CERT_FILE" in os.environ: - ssl_file = Path(os.environ["SSL_CERT_FILE"]) - if ssl_file.is_file(): - return str(ssl_file) - if "SSL_CERT_DIR" in os.environ: - ssl_path = Path(os.environ["SSL_CERT_DIR"]) - if ssl_path.is_dir(): - return str(ssl_path) - return None - - -def parse_header_links(value: str) -> list[dict[str, str]]: - """ - Returns a list of parsed link headers, for more info see: - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link - The generic syntax of those is: - Link: < uri-reference >; param1=value1; param2="value2" - So for instance: - Link; '<http:/.../front.jpeg>; type="image/jpeg",<http://.../back.jpeg>;' - would return - [ - {"url": "http:/.../front.jpeg", "type": "image/jpeg"}, - {"url": "http://.../back.jpeg"}, - ] - :param value: HTTP Link entity-header field - :return: list of parsed link headers - """ - links: list[dict[str, str]] = [] - replace_chars = " '\"" - value = value.strip(replace_chars) - if not value: - return links - for val in re.split(", *<", value): - try: - url, params = val.split(";", 1) - except ValueError: - url, params = val, "" - link = {"url": url.strip("<> '\"")} - for param in params.split(";"): - try: - key, value = param.split("=") - except ValueError: - break - link[key.strip(replace_chars)] = value.strip(replace_chars) - links.append(link) - return links - - -def parse_content_type_charset(content_type: str) -> str | None: - # We used to use `cgi.parse_header()` here, but `cgi` became a dead battery. - # See: https://peps.python.org/pep-0594/#cgi - msg = email.message.Message() - msg["content-type"] = content_type - return msg.get_content_charset(failobj=None) - - -SENSITIVE_HEADERS = {"authorization", "proxy-authorization"} - - -def obfuscate_sensitive_headers( - items: typing.Iterable[tuple[typing.AnyStr, typing.AnyStr]], -) -> typing.Iterator[tuple[typing.AnyStr, typing.AnyStr]]: - for k, v in items: - if to_str(k.lower()) in SENSITIVE_HEADERS: - v = to_bytes_or_str("[secure]", match_type_of=v) - yield k, v - - -def port_or_default(url: URL) -> int | None: - if url.port is not None: - return url.port - return {"http": 80, "https": 443}.get(url.scheme) - - -def same_origin(url: URL, other: URL) -> bool: - """ - Return 'True' if the given URLs share the same origin. - """ - return ( - url.scheme == other.scheme - and url.host == other.host - and port_or_default(url) == port_or_default(other) - ) - - -def is_https_redirect(url: URL, location: URL) -> bool: - """ - Return 'True' if 'location' is a HTTPS upgrade of 'url' - """ - if url.host != location.host: - return False - - return ( - url.scheme == "http" - and port_or_default(url) == 80 - and location.scheme == "https" - and port_or_default(location) == 443 - ) - - -def get_environment_proxies() -> dict[str, str | None]: - """Gets proxy information from the environment""" - - # urllib.request.getproxies() falls back on System - # Registry and Config for proxies on Windows and macOS. - # We don't want to propagate non-HTTP proxies into - # our configuration such as 'TRAVIS_APT_PROXY'. - proxy_info = getproxies() - mounts: dict[str, str | None] = {} - - for scheme in ("http", "https", "all"): - if proxy_info.get(scheme): - hostname = proxy_info[scheme] - mounts[f"{scheme}://"] = ( - hostname if "://" in hostname else f"http://{hostname}" - ) - - no_proxy_hosts = [host.strip() for host in proxy_info.get("no", "").split(",")] - for hostname in no_proxy_hosts: - # See https://curl.haxx.se/libcurl/c/CURLOPT_NOPROXY.html for details - # on how names in `NO_PROXY` are handled. - if hostname == "*": - # If NO_PROXY=* is used or if "*" occurs as any one of the comma - # separated hostnames, then we should just bypass any information - # from HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, and always ignore - # proxies. - return {} - elif hostname: - # NO_PROXY=.google.com is marked as "all://*.google.com, - # which disables "www.google.com" but not "google.com" - # NO_PROXY=google.com is marked as "all://*google.com, - # which disables "www.google.com" and "google.com". - # (But not "wwwgoogle.com") - # NO_PROXY can include domains, IPv6, IPv4 addresses and "localhost" - # NO_PROXY=example.com,::1,localhost,192.168.0.0/16 - if "://" in hostname: - mounts[hostname] = None - elif is_ipv4_hostname(hostname): - mounts[f"all://{hostname}"] = None - elif is_ipv6_hostname(hostname): - mounts[f"all://[{hostname}]"] = None - elif hostname.lower() == "localhost": - mounts[f"all://{hostname}"] = None - else: - mounts[f"all://*{hostname}"] = None - - return mounts - - -def to_bytes(value: str | bytes, encoding: str = "utf-8") -> bytes: - return value.encode(encoding) if isinstance(value, str) else value - - -def to_str(value: str | bytes, encoding: str = "utf-8") -> str: - return value if isinstance(value, str) else value.decode(encoding) - - -def to_bytes_or_str(value: str, match_type_of: typing.AnyStr) -> typing.AnyStr: - return value if isinstance(match_type_of, str) else value.encode() - - -def unquote(value: str) -> str: - return value[1:-1] if value[0] == value[-1] == '"' else value - - -def guess_content_type(filename: str | None) -> str | None: - if filename: - return mimetypes.guess_type(filename)[0] or "application/octet-stream" - return None - - -def peek_filelike_length(stream: typing.Any) -> int | None: - """ - Given a file-like stream object, return its length in number of bytes - without reading it into memory. - """ - try: - # Is it an actual file? - fd = stream.fileno() - # Yup, seems to be an actual file. - length = os.fstat(fd).st_size - except (AttributeError, OSError): - # No... Maybe it's something that supports random access, like `io.BytesIO`? - try: - # Assuming so, go to end of stream to figure out its length, - # then put it back in place. - offset = stream.tell() - length = stream.seek(0, os.SEEK_END) - stream.seek(offset) - except (AttributeError, OSError): - # Not even that? Sorry, we're doomed... - return None - - return length - - -class Timer: - async def _get_time(self) -> float: - library = sniffio.current_async_library() - if library == "trio": - import trio - - return trio.current_time() - else: - import asyncio - - return asyncio.get_event_loop().time() - - def sync_start(self) -> None: - self.started = time.perf_counter() - - async def async_start(self) -> None: - self.started = await self._get_time() - - def sync_elapsed(self) -> float: - now = time.perf_counter() - return now - self.started - - async def async_elapsed(self) -> float: - now = await self._get_time() - return now - self.started - - -class URLPattern: - """ - A utility class currently used for making lookups against proxy keys... - - # Wildcard matching... - >>> pattern = URLPattern("all://") - >>> pattern.matches(httpx.URL("http://example.com")) - True - - # Witch scheme matching... - >>> pattern = URLPattern("https://") - >>> pattern.matches(httpx.URL("https://example.com")) - True - >>> pattern.matches(httpx.URL("http://example.com")) - False - - # With domain matching... - >>> pattern = URLPattern("https://example.com") - >>> pattern.matches(httpx.URL("https://example.com")) - True - >>> pattern.matches(httpx.URL("http://example.com")) - False - >>> pattern.matches(httpx.URL("https://other.com")) - False - - # Wildcard scheme, with domain matching... - >>> pattern = URLPattern("all://example.com") - >>> pattern.matches(httpx.URL("https://example.com")) - True - >>> pattern.matches(httpx.URL("http://example.com")) - True - >>> pattern.matches(httpx.URL("https://other.com")) - False - - # With port matching... - >>> pattern = URLPattern("https://example.com:1234") - >>> pattern.matches(httpx.URL("https://example.com:1234")) - True - >>> pattern.matches(httpx.URL("https://example.com")) - False - """ - - def __init__(self, pattern: str) -> None: - from ._urls import URL - - if pattern and ":" not in pattern: - raise ValueError( - f"Proxy keys should use proper URL forms rather " - f"than plain scheme strings. " - f'Instead of "{pattern}", use "{pattern}://"' - ) - - url = URL(pattern) - self.pattern = pattern - self.scheme = "" if url.scheme == "all" else url.scheme - self.host = "" if url.host == "*" else url.host - self.port = url.port - if not url.host or url.host == "*": - self.host_regex: typing.Pattern[str] | None = None - elif url.host.startswith("*."): - # *.example.com should match "www.example.com", but not "example.com" - domain = re.escape(url.host[2:]) - self.host_regex = re.compile(f"^.+\\.{domain}$") - elif url.host.startswith("*"): - # *example.com should match "www.example.com" and "example.com" - domain = re.escape(url.host[1:]) - self.host_regex = re.compile(f"^(.+\\.)?{domain}$") - else: - # example.com should match "example.com" but not "www.example.com" - domain = re.escape(url.host) - self.host_regex = re.compile(f"^{domain}$") - - def matches(self, other: URL) -> bool: - if self.scheme and self.scheme != other.scheme: - return False - if ( - self.host - and self.host_regex is not None - and not self.host_regex.match(other.host) - ): - return False - if self.port is not None and self.port != other.port: - return False - return True - - @property - def priority(self) -> tuple[int, int, int]: - """ - The priority allows URLPattern instances to be sortable, so that - we can match from most specific to least specific. - """ - # URLs with a port should take priority over URLs without a port. - port_priority = 0 if self.port is not None else 1 - # Longer hostnames should match first. - host_priority = -len(self.host) - # Longer schemes should match first. - scheme_priority = -len(self.scheme) - return (port_priority, host_priority, scheme_priority) - - def __hash__(self) -> int: - return hash(self.pattern) - - def __lt__(self, other: URLPattern) -> bool: - return self.priority < other.priority - - def __eq__(self, other: typing.Any) -> bool: - return isinstance(other, URLPattern) and self.pattern == other.pattern - - -def is_ipv4_hostname(hostname: str) -> bool: - try: - ipaddress.IPv4Address(hostname.split("/")[0]) - except Exception: - return False - return True - - -def is_ipv6_hostname(hostname: str) -> bool: - try: - ipaddress.IPv6Address(hostname.split("/")[0]) - except Exception: - return False - return True diff --git a/contrib/python/httpx/httpx/py.typed b/contrib/python/httpx/httpx/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/httpx/httpx/py.typed +++ /dev/null diff --git a/contrib/python/httpx/patches/01-arcadia.patch b/contrib/python/httpx/patches/01-arcadia.patch deleted file mode 100644 index 6f4c9b85d31..00000000000 --- a/contrib/python/httpx/patches/01-arcadia.patch +++ /dev/null @@ -1,27 +0,0 @@ ---- contrib/python/httpx/httpx/_config.py (499810bf8a30b39c38d23cdf60c243417fdf81ee) -+++ contrib/python/httpx/httpx/_config.py (2e6e05a77528bac894ef025fa9cc552ac6cd0751) -@@ -56,7 +56,11 @@ class SSLConfig: - SSL Configuration. - """ - -- DEFAULT_CA_BUNDLE_PATH = Path(certifi.where()) -+ DEFAULT_CA_BUNDLE_PATH = certifi.where() -+ if callable(DEFAULT_CA_BUNDLE_PATH): -+ DEFAULT_CA_BUNDLE_PATH = staticmethod(DEFAULT_CA_BUNDLE_PATH) -+ else: -+ DEFAULT_CA_BUNDLE_PATH = Path(DEFAULT_CA_BUNDLE_PATH) - - def __init__( - self, -@@ -137,7 +141,10 @@ class SSLConfig: - except AttributeError: # pragma: nocover - pass - -- if ca_bundle_path.is_file(): -+ if callable(ca_bundle_path): -+ logger.debug("load_verify_locations cafile=%r", ca_bundle_path) -+ context.load_verify_locations(cafile=ca_bundle_path) -+ elif ca_bundle_path.is_file(): - cafile = str(ca_bundle_path) - logger.debug("load_verify_locations cafile=%r", cafile) - context.load_verify_locations(cafile=cafile) diff --git a/contrib/python/httpx/ya.make b/contrib/python/httpx/ya.make deleted file mode 100644 index 26977a0c76f..00000000000 --- a/contrib/python/httpx/ya.make +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(0.27.2) - -LICENSE(BSD-3-Clause) - -PEERDIR( - contrib/python/anyio - contrib/python/certifi - contrib/python/httpcore - contrib/python/idna - contrib/python/sniffio -) - -NO_LINT() - -NO_CHECK_IMPORTS( - httpx._main -) - -PY_SRCS( - TOP_LEVEL - httpx/__init__.py - httpx/__version__.py - httpx/_api.py - httpx/_auth.py - httpx/_client.py - httpx/_compat.py - httpx/_config.py - httpx/_content.py - httpx/_decoders.py - httpx/_exceptions.py - httpx/_main.py - httpx/_models.py - httpx/_multipart.py - httpx/_status_codes.py - httpx/_transports/__init__.py - httpx/_transports/asgi.py - httpx/_transports/base.py - httpx/_transports/default.py - httpx/_transports/mock.py - httpx/_transports/wsgi.py - httpx/_types.py - httpx/_urlparse.py - httpx/_urls.py - httpx/_utils.py -) - -RESOURCE_FILES( - PREFIX contrib/python/httpx/ - .dist-info/METADATA - .dist-info/entry_points.txt - .dist-info/top_level.txt - httpx/py.typed -) - -END() diff --git a/contrib/python/sniffio/.dist-info/METADATA b/contrib/python/sniffio/.dist-info/METADATA deleted file mode 100644 index 88968aed169..00000000000 --- a/contrib/python/sniffio/.dist-info/METADATA +++ /dev/null @@ -1,104 +0,0 @@ -Metadata-Version: 2.1 -Name: sniffio -Version: 1.3.1 -Summary: Sniff out which async library your code is running under -Author-email: "Nathaniel J. Smith" <[email protected]> -License: MIT OR Apache-2.0 -Project-URL: Homepage, https://github.com/python-trio/sniffio -Project-URL: Documentation, https://sniffio.readthedocs.io/ -Project-URL: Changelog, https://sniffio.readthedocs.io/en/latest/history.html -Keywords: async,trio,asyncio -Classifier: License :: OSI Approved :: MIT License -Classifier: License :: OSI Approved :: Apache Software License -Classifier: Framework :: Trio -Classifier: Framework :: AsyncIO -Classifier: Operating System :: POSIX :: Linux -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Operating System :: Microsoft :: Windows -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Intended Audience :: Developers -Classifier: Development Status :: 5 - Production/Stable -Requires-Python: >=3.7 -Description-Content-Type: text/x-rst -License-File: LICENSE -License-File: LICENSE.APACHE2 -License-File: LICENSE.MIT - -.. image:: https://img.shields.io/badge/chat-join%20now-blue.svg - :target: https://gitter.im/python-trio/general - :alt: Join chatroom - -.. image:: https://img.shields.io/badge/docs-read%20now-blue.svg - :target: https://sniffio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -.. image:: https://img.shields.io/pypi/v/sniffio.svg - :target: https://pypi.org/project/sniffio - :alt: Latest PyPi version - -.. image:: https://img.shields.io/conda/vn/conda-forge/sniffio.svg - :target: https://anaconda.org/conda-forge/sniffio - :alt: Latest conda-forge version - -.. image:: https://travis-ci.org/python-trio/sniffio.svg?branch=master - :target: https://travis-ci.org/python-trio/sniffio - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-trio/sniffio/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-trio/sniffio - :alt: Test coverage - -================================================================= -sniffio: Sniff out which async library your code is running under -================================================================= - -You're writing a library. You've decided to be ambitious, and support -multiple async I/O packages, like `Trio -<https://trio.readthedocs.io>`__, and `asyncio -<https://docs.python.org/3/library/asyncio.html>`__, and ... You've -written a bunch of clever code to handle all the differences. But... -how do you know *which* piece of clever code to run? - -This is a tiny package whose only purpose is to let you detect which -async library your code is running under. - -* Documentation: https://sniffio.readthedocs.io - -* Bug tracker and source code: https://github.com/python-trio/sniffio - -* License: MIT or Apache License 2.0, your choice - -* Contributor guide: https://trio.readthedocs.io/en/latest/contributing.html - -* Code of conduct: Contributors are requested to follow our `code of - conduct - <https://trio.readthedocs.io/en/latest/code-of-conduct.html>`_ - in all project spaces. - -This library is maintained by the Trio project, as a service to the -async Python community as a whole. - - -Quickstart ----------- - -.. code-block:: python3 - - from sniffio import current_async_library - import trio - import asyncio - - async def print_library(): - library = current_async_library() - print("This is:", library) - - # Prints "This is trio" - trio.run(print_library) - - # Prints "This is asyncio" - asyncio.run(print_library()) - -For more details, including how to add support to new async libraries, -`please peruse our fine manual <https://sniffio.readthedocs.io>`__. diff --git a/contrib/python/sniffio/.dist-info/top_level.txt b/contrib/python/sniffio/.dist-info/top_level.txt deleted file mode 100644 index 01c650244d0..00000000000 --- a/contrib/python/sniffio/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -sniffio diff --git a/contrib/python/sniffio/.yandex_meta/yamaker.yaml b/contrib/python/sniffio/.yandex_meta/yamaker.yaml deleted file mode 100644 index 1a9504527f4..00000000000 --- a/contrib/python/sniffio/.yandex_meta/yamaker.yaml +++ /dev/null @@ -1,2 +0,0 @@ -mark_as_tests: - - sniffio/_tests/* diff --git a/contrib/python/sniffio/LICENSE b/contrib/python/sniffio/LICENSE deleted file mode 100644 index 51f34429178..00000000000 --- a/contrib/python/sniffio/LICENSE +++ /dev/null @@ -1,3 +0,0 @@ -This software is made available under the terms of *either* of the -licenses found in LICENSE.APACHE2 or LICENSE.MIT. Contributions to are -made under the terms of *both* these licenses. diff --git a/contrib/python/sniffio/LICENSE.APACHE2 b/contrib/python/sniffio/LICENSE.APACHE2 deleted file mode 100644 index d6456956733..00000000000 --- a/contrib/python/sniffio/LICENSE.APACHE2 +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/contrib/python/sniffio/LICENSE.MIT b/contrib/python/sniffio/LICENSE.MIT deleted file mode 100644 index b8bb9718592..00000000000 --- a/contrib/python/sniffio/LICENSE.MIT +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/sniffio/README.rst b/contrib/python/sniffio/README.rst deleted file mode 100644 index 2a62cea5a0a..00000000000 --- a/contrib/python/sniffio/README.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. image:: https://img.shields.io/badge/chat-join%20now-blue.svg - :target: https://gitter.im/python-trio/general - :alt: Join chatroom - -.. image:: https://img.shields.io/badge/docs-read%20now-blue.svg - :target: https://sniffio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -.. image:: https://img.shields.io/pypi/v/sniffio.svg - :target: https://pypi.org/project/sniffio - :alt: Latest PyPi version - -.. image:: https://img.shields.io/conda/vn/conda-forge/sniffio.svg - :target: https://anaconda.org/conda-forge/sniffio - :alt: Latest conda-forge version - -.. image:: https://travis-ci.org/python-trio/sniffio.svg?branch=master - :target: https://travis-ci.org/python-trio/sniffio - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-trio/sniffio/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-trio/sniffio - :alt: Test coverage - -================================================================= -sniffio: Sniff out which async library your code is running under -================================================================= - -You're writing a library. You've decided to be ambitious, and support -multiple async I/O packages, like `Trio -<https://trio.readthedocs.io>`__, and `asyncio -<https://docs.python.org/3/library/asyncio.html>`__, and ... You've -written a bunch of clever code to handle all the differences. But... -how do you know *which* piece of clever code to run? - -This is a tiny package whose only purpose is to let you detect which -async library your code is running under. - -* Documentation: https://sniffio.readthedocs.io - -* Bug tracker and source code: https://github.com/python-trio/sniffio - -* License: MIT or Apache License 2.0, your choice - -* Contributor guide: https://trio.readthedocs.io/en/latest/contributing.html - -* Code of conduct: Contributors are requested to follow our `code of - conduct - <https://trio.readthedocs.io/en/latest/code-of-conduct.html>`_ - in all project spaces. - -This library is maintained by the Trio project, as a service to the -async Python community as a whole. - - -Quickstart ----------- - -.. code-block:: python3 - - from sniffio import current_async_library - import trio - import asyncio - - async def print_library(): - library = current_async_library() - print("This is:", library) - - # Prints "This is trio" - trio.run(print_library) - - # Prints "This is asyncio" - asyncio.run(print_library()) - -For more details, including how to add support to new async libraries, -`please peruse our fine manual <https://sniffio.readthedocs.io>`__. diff --git a/contrib/python/sniffio/sniffio/__init__.py b/contrib/python/sniffio/sniffio/__init__.py deleted file mode 100644 index 63f2f19e409..00000000000 --- a/contrib/python/sniffio/sniffio/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Top-level package for sniffio.""" - -__all__ = [ - "current_async_library", - "AsyncLibraryNotFoundError", - "current_async_library_cvar", - "thread_local", -] - -from ._version import __version__ - -from ._impl import ( - current_async_library, - AsyncLibraryNotFoundError, - current_async_library_cvar, - thread_local, -) diff --git a/contrib/python/sniffio/sniffio/_impl.py b/contrib/python/sniffio/sniffio/_impl.py deleted file mode 100644 index c1a7bbf218b..00000000000 --- a/contrib/python/sniffio/sniffio/_impl.py +++ /dev/null @@ -1,95 +0,0 @@ -from contextvars import ContextVar -from typing import Optional -import sys -import threading - -current_async_library_cvar = ContextVar( - "current_async_library_cvar", default=None -) # type: ContextVar[Optional[str]] - - -class _ThreadLocal(threading.local): - # Since threading.local provides no explicit mechanism is for setting - # a default for a value, a custom class with a class attribute is used - # instead. - name = None # type: Optional[str] - - -thread_local = _ThreadLocal() - - -class AsyncLibraryNotFoundError(RuntimeError): - pass - - -def current_async_library() -> str: - """Detect which async library is currently running. - - The following libraries are currently supported: - - ================ =========== ============================ - Library Requires Magic string - ================ =========== ============================ - **Trio** Trio v0.6+ ``"trio"`` - **Curio** - ``"curio"`` - **asyncio** ``"asyncio"`` - **Trio-asyncio** v0.8.2+ ``"trio"`` or ``"asyncio"``, - depending on current mode - ================ =========== ============================ - - Returns: - A string like ``"trio"``. - - Raises: - AsyncLibraryNotFoundError: if called from synchronous context, - or if the current async library was not recognized. - - Examples: - - .. code-block:: python3 - - from sniffio import current_async_library - - async def generic_sleep(seconds): - library = current_async_library() - if library == "trio": - import trio - await trio.sleep(seconds) - elif library == "asyncio": - import asyncio - await asyncio.sleep(seconds) - # ... and so on ... - else: - raise RuntimeError(f"Unsupported library {library!r}") - - """ - value = thread_local.name - if value is not None: - return value - - value = current_async_library_cvar.get() - if value is not None: - return value - - # Need to sniff for asyncio - if "asyncio" in sys.modules: - import asyncio - try: - current_task = asyncio.current_task # type: ignore[attr-defined] - except AttributeError: - current_task = asyncio.Task.current_task # type: ignore[attr-defined] - try: - if current_task() is not None: - return "asyncio" - except RuntimeError: - pass - - # Sniff for curio (for now) - if 'curio' in sys.modules: - from curio.meta import curio_running - if curio_running(): - return 'curio' - - raise AsyncLibraryNotFoundError( - "unknown async library, or not in async context" - ) diff --git a/contrib/python/sniffio/sniffio/_version.py b/contrib/python/sniffio/sniffio/_version.py deleted file mode 100644 index 0495d10545c..00000000000 --- a/contrib/python/sniffio/sniffio/_version.py +++ /dev/null @@ -1,3 +0,0 @@ -# This file is imported from __init__.py and exec'd from setup.py - -__version__ = "1.3.1" diff --git a/contrib/python/sniffio/sniffio/py.typed b/contrib/python/sniffio/sniffio/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/sniffio/sniffio/py.typed +++ /dev/null diff --git a/contrib/python/sniffio/ya.make b/contrib/python/sniffio/ya.make deleted file mode 100644 index 165b99c587e..00000000000 --- a/contrib/python/sniffio/ya.make +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(1.3.1) - -LICENSE(Apache-2.0 AND MIT) - -NO_LINT() - -PY_SRCS( - TOP_LEVEL - sniffio/__init__.py - sniffio/_impl.py - sniffio/_version.py -) - -RESOURCE_FILES( - PREFIX contrib/python/sniffio/ - .dist-info/METADATA - .dist-info/top_level.txt - sniffio/py.typed -) - -END() diff --git a/contrib/python/tzdata/.dist-info/METADATA b/contrib/python/tzdata/.dist-info/METADATA index 4feec22ac49..393adbb254b 100644 --- a/contrib/python/tzdata/.dist-info/METADATA +++ b/contrib/python/tzdata/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: tzdata -Version: 2024.2 +Version: 2025.1 Summary: Provider of IANA time zone data Home-page: https://github.com/python/tzdata Author: Python Software Foundation diff --git a/contrib/python/tzdata/tzdata/__init__.py b/contrib/python/tzdata/tzdata/__init__.py index e558a8a536c..87dba5a778c 100644 --- a/contrib/python/tzdata/tzdata/__init__.py +++ b/contrib/python/tzdata/tzdata/__init__.py @@ -1,6 +1,6 @@ # IANA versions like 2020a are not valid PEP 440 identifiers; the recommended # way to translate the version is to use YYYY.n where `n` is a 0-based index. -__version__ = "2024.2" +__version__ = "2025.1" # This exposes the original IANA version number. -IANA_VERSION = "2024b" +IANA_VERSION = "2025a" diff --git a/contrib/python/tzdata/tzdata/zoneinfo/America/Asuncion b/contrib/python/tzdata/tzdata/zoneinfo/America/Asuncion Binary files differindex 62250367422..f056047f058 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/America/Asuncion +++ b/contrib/python/tzdata/tzdata/zoneinfo/America/Asuncion diff --git a/contrib/python/tzdata/tzdata/zoneinfo/Asia/Manila b/contrib/python/tzdata/tzdata/zoneinfo/Asia/Manila Binary files differindex 3c3584e09ae..145bb6fb162 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/Asia/Manila +++ b/contrib/python/tzdata/tzdata/zoneinfo/Asia/Manila diff --git a/contrib/python/tzdata/tzdata/zoneinfo/leapseconds b/contrib/python/tzdata/tzdata/zoneinfo/leapseconds index 6c715cb20b0..76f771427f2 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/leapseconds +++ b/contrib/python/tzdata/tzdata/zoneinfo/leapseconds @@ -69,11 +69,11 @@ Leap 2016 Dec 31 23:59:60 + S # Any additional leap seconds will come after this. # This Expires line is commented out for now, # so that pre-2020a zic implementations do not reject this file. -#Expires 2025 Jun 28 00:00:00 +#Expires 2025 Dec 28 00:00:00 # POSIX timestamps for the data in this file: -#updated 1720104763 (2024-07-04 14:52:43 UTC) -#expires 1751068800 (2025-06-28 00:00:00 UTC) +#updated 1736208000 (2025-01-07 00:00:00 UTC) +#expires 1766880000 (2025-12-28 00:00:00 UTC) # Updated through IERS Bulletin C (https://hpiers.obspm.fr/iers/bul/bulc/bulletinc.dat) -# File expires on 28 June 2025 +# File expires on 28 December 2025 diff --git a/contrib/python/tzdata/tzdata/zoneinfo/tzdata.zi b/contrib/python/tzdata/tzdata/zoneinfo/tzdata.zi index 62e78bb826c..db6ba4af2b0 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/tzdata.zi +++ b/contrib/python/tzdata/tzdata/zoneinfo/tzdata.zi @@ -1,4 +1,4 @@ -# version 2024b +# version 2025a # This zic input file is in the public domain. R d 1916 o - Jun 14 23s 1 S R d 1916 1919 - O Su>=1 23s 0 - @@ -721,12 +721,16 @@ R P 2085 o - Ap 21 2 0 - R P 2085 o - Jun 9 2 1 S R P 2086 o - Ap 13 2 0 - R P 2086 o - May 25 2 1 S -R PH 1936 o - N 1 0 1 D -R PH 1937 o - F 1 0 0 S -R PH 1954 o - Ap 12 0 1 D -R PH 1954 o - Jul 1 0 0 S -R PH 1978 o - Mar 22 0 1 D -R PH 1978 o - S 21 0 0 S +R PH 1936 o - O 31 24 1 D +R PH 1937 o - Ja 15 24 0 S +R PH 1941 o - D 15 24 1 D +R PH 1945 o - N 30 24 0 S +R PH 1954 o - Ap 11 24 1 D +R PH 1954 o - Jun 4 24 0 S +R PH 1977 o - Mar 27 24 1 D +R PH 1977 o - S 21 24 0 S +R PH 1990 o - May 21 0 1 D +R PH 1990 o - Jul 28 24 0 S R S 1920 1923 - Ap Su>=15 2 1 S R S 1920 1923 - O Su>=1 2 0 - R S 1962 o - Ap 29 2 1 S @@ -1725,7 +1729,7 @@ R Y 1972 2006 - O lastSu 2 0 S R Y 1987 2006 - Ap Su>=1 2 1 D R Yu 1965 o - Ap lastSu 0 2 DD R Yu 1965 o - O lastSu 2 0 S -R m 1931 o - April 30 0 1 D +R m 1931 o - Ap 30 0 1 D R m 1931 o - O 1 0 0 S R m 1939 o - F 5 0 1 D R m 1939 o - Jun 25 0 0 S @@ -2019,9 +2023,9 @@ R y 2002 2004 - Ap Su>=1 0 0 - R y 2002 2003 - S Su>=1 0 1 - R y 2004 2009 - O Su>=15 0 1 - R y 2005 2009 - Mar Su>=8 0 0 - -R y 2010 ma - O Su>=1 0 1 - +R y 2010 2024 - O Su>=1 0 1 - R y 2010 2012 - Ap Su>=8 0 0 - -R y 2013 ma - Mar Su>=22 0 0 - +R y 2013 2024 - Mar Su>=22 0 0 - R PE 1938 o - Ja 1 0 1 - R PE 1938 o - Ap 1 0 0 - R PE 1938 1939 - S lastSu 0 1 - @@ -2336,7 +2340,8 @@ Z America/Asuncion -3:50:40 - LMT 1890 -3:50:40 - AMT 1931 O 10 -4 - %z 1972 O -3 - %z 1974 Ap --4 y %z +-4 y %z 2024 O 15 +-3 - %z Z America/Bahia -2:34:4 - LMT 1914 -3 B %z 2003 S 24 -3 - %z 2011 O 16 @@ -3268,10 +3273,10 @@ Z Asia/Makassar 7:57:36 - LMT 1920 8 - %z 1942 F 9 9 - %z 1945 S 23 8 - WITA -Z Asia/Manila -15:56 - LMT 1844 D 31 -8:4 - LMT 1899 May 11 -8 PH P%sT 1942 May -9 - JST 1944 N +Z Asia/Manila -15:56:8 - LMT 1844 D 31 +8:3:52 - LMT 1899 S 6 4u +8 PH P%sT 1942 F 11 24 +9 - JST 1945 Mar 4 8 PH P%sT Z Asia/Nicosia 2:13:28 - LMT 1921 N 14 2 CY EE%sT 1998 S diff --git a/contrib/python/tzdata/tzdata/zoneinfo/zone.tab b/contrib/python/tzdata/tzdata/zoneinfo/zone.tab index bfc0b593304..d2be66359f3 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/zone.tab +++ b/contrib/python/tzdata/tzdata/zoneinfo/zone.tab @@ -310,7 +310,7 @@ PF -0900-13930 Pacific/Marquesas Marquesas Islands PF -2308-13457 Pacific/Gambier Gambier Islands PG -0930+14710 Pacific/Port_Moresby most of Papua New Guinea PG -0613+15534 Pacific/Bougainville Bougainville -PH +1435+12100 Asia/Manila +PH +143512+1205804 Asia/Manila PK +2452+06703 Asia/Karachi PL +5215+02100 Europe/Warsaw PM +4703-05620 America/Miquelon diff --git a/contrib/python/tzdata/tzdata/zoneinfo/zone1970.tab b/contrib/python/tzdata/tzdata/zoneinfo/zone1970.tab index 7726f39a092..5ded0565ebf 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/zone1970.tab +++ b/contrib/python/tzdata/tzdata/zoneinfo/zone1970.tab @@ -183,7 +183,7 @@ IR +3540+05126 Asia/Tehran IT,SM,VA +4154+01229 Europe/Rome JM +175805-0764736 America/Jamaica JO +3157+03556 Asia/Amman -JP +353916+1394441 Asia/Tokyo +JP,AU +353916+1394441 Asia/Tokyo Eyre Bird Observatory KE,DJ,ER,ET,KM,MG,SO,TZ,UG,YT -0117+03649 Africa/Nairobi KG +4254+07436 Asia/Bishkek KI,MH,TV,UM,WF +0125+17300 Pacific/Tarawa Gilberts, Marshalls, Wake @@ -246,7 +246,7 @@ PF -0900-13930 Pacific/Marquesas Marquesas Islands PF -2308-13457 Pacific/Gambier Gambier Islands PG,AQ,FM -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas), Chuuk, Yap, Dumont d'Urville PG -0613+15534 Pacific/Bougainville Bougainville -PH +1435+12100 Asia/Manila +PH +143512+1205804 Asia/Manila PK +2452+06703 Asia/Karachi PL +5215+02100 Europe/Warsaw PM +4703-05620 America/Miquelon @@ -293,7 +293,7 @@ RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea SA,AQ,KW,YE +2438+04643 Asia/Riyadh Syowa SB,FM -0932+16012 Pacific/Guadalcanal Pohnpei SD +1536+03232 Africa/Khartoum -SG,MY +0117+10351 Asia/Singapore peninsular Malaysia +SG,AQ,MY +0117+10351 Asia/Singapore peninsular Malaysia, Concordia SR +0550-05510 America/Paramaribo SS +0451+03137 Africa/Juba ST +0020+00644 Africa/Sao_Tome diff --git a/contrib/python/tzdata/tzdata/zoneinfo/zonenow.tab b/contrib/python/tzdata/tzdata/zoneinfo/zonenow.tab index 01f536b3ba3..d2c1e48584f 100644 --- a/contrib/python/tzdata/tzdata/zoneinfo/zonenow.tab +++ b/contrib/python/tzdata/tzdata/zoneinfo/zonenow.tab @@ -97,9 +97,6 @@ XX +1828-06954 America/Santo_Domingo Atlantic Standard ("AST") - eastern Caribbe # -04/-03 (Chile DST) XX -3327-07040 America/Santiago most of Chile # -# -04/-03 (Paraguay DST) -XX -2516-05740 America/Asuncion Paraguay -# # -04/-03 - AST/ADT (North America DST) XX +4439-06336 America/Halifax Atlantic ("AST/ADT") - Canada; Bermuda # @@ -224,7 +221,7 @@ XX +1345+10031 Asia/Bangkok Russia; Indochina; Christmas Island XX -0610+10648 Asia/Jakarta Indonesia ("WIB") # # +08 -XX +0117+10351 Asia/Singapore Russia; Brunei; Malaysia; Singapore +XX +0117+10351 Asia/Singapore Russia; Brunei; Malaysia; Singapore; Concordia # # +08 - AWST XX -3157+11551 Australia/Perth Western Australia ("AWST") @@ -236,7 +233,7 @@ XX +3114+12128 Asia/Shanghai China ("CST") XX +2217+11409 Asia/Hong_Kong Hong Kong ("HKT") # # +08 - PHT -XX +1435+12100 Asia/Manila Philippines ("PHT") +XX +143512+1205804 Asia/Manila Philippines ("PHT") # # +08 - WITA XX -0507+11924 Asia/Makassar Indonesia ("WITA") @@ -248,7 +245,7 @@ XX -3143+12852 Australia/Eucla Eucla XX +5203+11328 Asia/Chita Russia; Palau; East Timor # # +09 - JST -XX +353916+1394441 Asia/Tokyo Japan ("JST") +XX +353916+1394441 Asia/Tokyo Japan ("JST"); Eyre Bird Observatory # # +09 - KST XX +3733+12658 Asia/Seoul Korea ("KST") diff --git a/contrib/python/tzdata/ya.make b/contrib/python/tzdata/ya.make index 6fbf62e9d82..cd028a674f1 100644 --- a/contrib/python/tzdata/ya.make +++ b/contrib/python/tzdata/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2024.2) +VERSION(2025.1) LICENSE(Apache-2.0) diff --git a/contrib/restricted/google/benchmark/.yandex_meta/__init__.py b/contrib/restricted/google/benchmark/.yandex_meta/__init__.py index 4a5ae3d8d4f..353b20c9875 100644 --- a/contrib/restricted/google/benchmark/.yandex_meta/__init__.py +++ b/contrib/restricted/google/benchmark/.yandex_meta/__init__.py @@ -77,7 +77,6 @@ benchmark = CMakeNinjaNixProject( nixattr="gbenchmark", addincl_global={".": {"./include"}}, disable_includes=[ - "emscripten.h", "kstat.h", "qurt.h", "perfmon/", diff --git a/contrib/restricted/google/benchmark/src/cycleclock.h b/contrib/restricted/google/benchmark/src/cycleclock.h index 0c7f0408a4c..bd62f5d7e7a 100644 --- a/contrib/restricted/google/benchmark/src/cycleclock.h +++ b/contrib/restricted/google/benchmark/src/cycleclock.h @@ -48,7 +48,7 @@ extern "C" uint64_t __rdtsc(); #endif #ifdef BENCHMARK_OS_EMSCRIPTEN -#error #include <emscripten.h> +#include <emscripten.h> #endif namespace benchmark { diff --git a/contrib/restricted/google/benchmark/src/timers.cc b/contrib/restricted/google/benchmark/src/timers.cc index a0543fe3d17..8c2722dca27 100644 --- a/contrib/restricted/google/benchmark/src/timers.cc +++ b/contrib/restricted/google/benchmark/src/timers.cc @@ -44,7 +44,7 @@ #endif #ifdef BENCHMARK_OS_EMSCRIPTEN -#error #include <emscripten.h> +#include <emscripten.h> #endif #include <cerrno> diff --git a/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.cpp b/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.cpp deleted file mode 100644 index 429598e3b16..00000000000 --- a/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "absl_unwinder.h" - -#include <library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h> - -#include <absl/debugging/stacktrace.h> - -namespace NYT::NBacktrace { - -//////////////////////////////////////////////////////////////////////////////// - -namespace { - -int AbslStackUnwinder( - void** frames, - int* /*framesSizes*/, - int maxFrames, - int skipFrames, - const void* /*uc*/, - int* /*minDroppedFrames*/) -{ - NBacktrace::TLibunwindCursor cursor; - - for (int i = 0; i < skipFrames + 1; ++i) { - cursor.MoveNext(); - } - - int count = 0; - for (int i = 0; i < maxFrames; ++i) { - if (cursor.IsFinished()) { - return count; - } - - // IP point's to return address. Subtract 1 to get accurate line information for profiler. - frames[i] = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(cursor.GetCurrentIP()) - 1); - count++; - - cursor.MoveNext(); - } - return count; -} - -} // namespace - -void SetAbslStackUnwinder() -{ - absl::SetStackUnwinder(AbslStackUnwinder); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h b/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h deleted file mode 100644 index 9a23c582c53..00000000000 --- a/library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -namespace NYT::NBacktrace { - -//////////////////////////////////////////////////////////////////////////////// - -//! Configures Abseil to use a libunwind-based stack unwinder. -//! This is useful, e.g., for collecting TCMalloc heap dumps. -void SetAbslStackUnwinder(); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/absl_unwinder/ya.make b/library/cpp/yt/backtrace/absl_unwinder/ya.make deleted file mode 100644 index f6bb0fbb5a6..00000000000 --- a/library/cpp/yt/backtrace/absl_unwinder/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) - -SRCS( - absl_unwinder.cpp -) - -PEERDIR( - library/cpp/yt/backtrace/cursors/libunwind - contrib/libs/tcmalloc/malloc_extension -) - -END() diff --git a/library/cpp/yt/memory/blob.cpp b/library/cpp/yt/memory/blob.cpp index b4de038948c..2517446ec7e 100644 --- a/library/cpp/yt/memory/blob.cpp +++ b/library/cpp/yt/memory/blob.cpp @@ -4,6 +4,8 @@ #include <library/cpp/yt/malloc/malloc.h> +#include <library/cpp/yt/system/exit.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -147,9 +149,17 @@ void TBlob::Reset() char* TBlob::DoAllocate(size_t size) { - return static_cast<char*>(PageAligned_ + auto* allocated = static_cast<char*>(PageAligned_ ? ::aligned_malloc(size, GetPageSize()) : ::malloc(size)); + + if (Y_UNLIKELY(!allocated)) { + AbortProcessDramatically( + EProcessExitCode::OutOfMemory, + "Out-of-memory during TBlob allocation"); + } + + return allocated; } void TBlob::Allocate(size_t newCapacity) diff --git a/library/cpp/yt/mlock/README.md b/library/cpp/yt/mlock/README.md deleted file mode 100644 index b61b6072c4d..00000000000 --- a/library/cpp/yt/mlock/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# mlock - -MlockFileMappings подгружает и лочит в память все страницы исполняемого файла. - -В отличии от вызова mlockall, функция не лочит другие страницы процесса. -mlockall явно выделяет физическую память под все vma. Типичный процесс сначала -стартует и инициализирует аллокатор, а потом уже вызывает функцию для mlock страниц. -Аллокатор при старте выделяет большие диапазоны через mmap, но реально их не использует. -Поэтому mlockall приводит в повышенному потреблению памяти. - -Также, в отличии от mlockall, функция может подгрузить страницы в память сразу. diff --git a/library/cpp/yt/mlock/mlock.h b/library/cpp/yt/mlock/mlock.h deleted file mode 100644 index 035fc47e373..00000000000 --- a/library/cpp/yt/mlock/mlock.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -bool MlockFileMappings(bool populate = true); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/library/cpp/yt/mlock/mlock_linux.cpp b/library/cpp/yt/mlock/mlock_linux.cpp deleted file mode 100644 index 8791869f951..00000000000 --- a/library/cpp/yt/mlock/mlock_linux.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "mlock.h" - -#include <stdio.h> -#include <sys/mman.h> -#include <stdint.h> -#include <inttypes.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -void PopulateFile(void* ptr, size_t size) -{ - constexpr size_t PageSize = 4096; - - auto* begin = static_cast<volatile char*>(ptr); - for (auto* current = begin; current < begin + size; current += PageSize) { - *current; - } -} - -bool MlockFileMappings(bool populate) -{ - auto* file = ::fopen("/proc/self/maps", "r"); - if (!file) { - return false; - } - - // Each line of /proc/<pid>/smaps has the following format: - // address perms offset dev inode path - // E.g. - // 08048000-08056000 r-xp 00000000 03:0c 64593 /usr/sbin/gpm - - bool failed = false; - while (true) { - char line[1024]; - if (!fgets(line, sizeof(line), file)) { - break; - } - - char addressStr[64]; - char permsStr[64]; - char offsetStr[64]; - char devStr[64]; - int inode; - if (sscanf(line, "%s %s %s %s %d", - addressStr, - permsStr, - offsetStr, - devStr, - &inode) != 5) - { - continue; - } - - if (inode == 0) { - continue; - } - - if (permsStr[0] != 'r') { - continue; - } - - uintptr_t startAddress; - uintptr_t endAddress; - if (sscanf(addressStr, "%" PRIx64 "-%" PRIx64, - &startAddress, - &endAddress) != 2) - { - continue; - } - - if (::mlock(reinterpret_cast<const void*>(startAddress), endAddress - startAddress) != 0) { - failed = true; - continue; - } - - if (populate) { - PopulateFile(reinterpret_cast<void*>(startAddress), endAddress - startAddress); - } - } - - ::fclose(file); - return !failed; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/library/cpp/yt/mlock/mlock_other.cpp b/library/cpp/yt/mlock/mlock_other.cpp deleted file mode 100644 index 269c5c3cb98..00000000000 --- a/library/cpp/yt/mlock/mlock_other.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "mlock.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -bool MlockFileMappings(bool /* populate */) -{ - return false; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/library/cpp/yt/mlock/unittests/mlock_ut.cpp b/library/cpp/yt/mlock/unittests/mlock_ut.cpp deleted file mode 100644 index 98386622e8d..00000000000 --- a/library/cpp/yt/mlock/unittests/mlock_ut.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include <gtest/gtest.h> - -#include <library/cpp/yt/mlock/mlock.h> - -namespace NYT { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -TEST(TMlockTest, Call) -{ - ASSERT_TRUE(MlockFileMappings(false)); - ASSERT_TRUE(MlockFileMappings(true)); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT
\ No newline at end of file diff --git a/library/cpp/yt/mlock/unittests/ya.make b/library/cpp/yt/mlock/unittests/ya.make deleted file mode 100644 index f1f956d4681..00000000000 --- a/library/cpp/yt/mlock/unittests/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -GTEST() - -INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) - -SRCS( - mlock_ut.cpp -) - -PEERDIR( - library/cpp/yt/mlock -) - -END() diff --git a/library/cpp/yt/mlock/ya.make b/library/cpp/yt/mlock/ya.make deleted file mode 100644 index 2603d128ed8..00000000000 --- a/library/cpp/yt/mlock/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) - -IF (OS_LINUX AND NOT SANITIZER_TYPE) - SRCS(mlock_linux.cpp) -ELSE() - SRCS(mlock_other.cpp) -ENDIF() - -END() - -IF (OS_LINUX AND NOT SANITIZER_TYPE) - RECURSE(unittests) -ENDIF() - diff --git a/library/cpp/yt/threading/traceless_guard-inl.h b/library/cpp/yt/threading/traceless_guard-inl.h deleted file mode 100644 index e03d5196c43..00000000000 --- a/library/cpp/yt/threading/traceless_guard-inl.h +++ /dev/null @@ -1,115 +0,0 @@ -#pragma once - -#ifndef TRACELESS_GUARD_INL_H_ -#error "Direct inclusion of this file is not allowed, include traceless_guard.h" -#endif -#undef TRACELESS_GUARD_INL_H_ - -#include "rw_spin_lock.h" -#include "spin_lock_count.h" - -#include <util/system/guard.h> - -#include <concepts> - -namespace NYT::NThreading::NPrivate { - -//////////////////////////////////////////////////////////////////////////////// - -#ifdef NDEBUG - -template <class T, class TOps = TCommonLockOps<T>> -using TTracelessGuard = TGuard<T, TOps>; - -template <class T, class TOps = TCommonLockOps<T>> -using TTracelessInverseGuard = TInverseGuard<T, TOps>; - -template <class T, class TOps = TTryLockOps<T>> -using TTracelessTryGuard = TTryGuard<T, TOps>; - -template <class T> -using TTracelessReaderGuard = TReaderGuard<T>; - -template <class T> -using TTracelessWriterGuard = TWriterGuard<T>; - -#else - -template <class TOps> -struct TTracelessOps - : public TOps -{ - template <class T> - static void Acquire(T* t) noexcept - { - TOps::Acquire(t); - NDetail::RecordSpinLockReleased(); - } - - template <class T> - static void Release(T* t) noexcept - { - NDetail::RecordSpinLockAcquired(); - TOps::Release(t); - } - - template <class T> - requires requires (T* t) { { TOps::TryAcquire(t) } -> std::same_as<bool>; } - static bool TryAcquire(T* t) noexcept - { - bool isAcquired = TOps::TryAcquire(t); - - if (isAcquired) { - NDetail::RecordSpinLockReleased(); - } - - return isAcquired; - } -}; - -template <CTrackedSpinLock T, class TOps = TCommonLockOps<T>> -using TTracelessGuard = TGuard<T, TTracelessOps<TOps>>; - -template <CTrackedSpinLock T, class TOps = TCommonLockOps<T>> -using TTracelessInverseGuard = TInverseGuard<T, TTracelessOps<TOps>>; - -template <CTrackedSpinLock T, class TOps = TTryLockOps<T>> -using TTracelessTryGuard = TTryGuard<T, TTracelessOps<TOps>>; - -template <CTrackedSpinLock T> -using TTracelessReaderGuard = TGuard<T, TTracelessOps<TReaderSpinlockTraits<T>>>; - -template <CTrackedSpinLock T> -using TTracelessWriterGuard = TGuard<T, TTracelessOps<TWriterSpinlockTraits<T>>>; - -#endif - -//////////////////////////////////////////////////////////////////////////////// - -template <CTrackedSpinLock T> -TTracelessGuard<T> TracelessGuard(const T& mutex) -{ - return {&mutex}; -} - -template <CTrackedSpinLock T> -TTracelessTryGuard<T> TracelessTryGuard(const T& mutex) -{ - return {&mutex}; -} - -template <CTrackedSpinLock T> -TTracelessReaderGuard<T> TracelessReaderGuard(const T& mutex) -{ - return {&mutex}; -} - -template <CTrackedSpinLock T> -TTracelessWriterGuard<T> TracelessWriterGuard(const T& mutex) -{ - return {&mutex}; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NThreading::NPrivate diff --git a/library/cpp/yt/threading/traceless_guard.h b/library/cpp/yt/threading/traceless_guard.h deleted file mode 100644 index 10ad0ded986..00000000000 --- a/library/cpp/yt/threading/traceless_guard.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#define TRACELESS_GUARD_INL_H_ -#include "traceless_guard-inl.h" -#undef TRACELESS_GUARD_INL_H_ - -namespace NYT::NThreading { - -// This guards are zero-cost replacements for normal ones -// which allow user to avoid spinlocks being tracked. - -//////////////////////////////////////////////////////////////////////////////// - -using NPrivate::TTracelessGuard; -using NPrivate::TTracelessInverseGuard; -using NPrivate::TTracelessTryGuard; -using NPrivate::TTracelessReaderGuard; -using NPrivate::TTracelessWriterGuard; - -//////////////////////////////////////////////////////////////////////////////// - -using NPrivate::TracelessGuard; -using NPrivate::TracelessTryGuard; -using NPrivate::TracelessReaderGuard; -using NPrivate::TracelessWriterGuard; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NThreading diff --git a/yql/essentials/ast/yql_expr.cpp b/yql/essentials/ast/yql_expr.cpp index 173dffd77fe..21d0b9a446c 100644 --- a/yql/essentials/ast/yql_expr.cpp +++ b/yql/essentials/ast/yql_expr.cpp @@ -3884,6 +3884,133 @@ TString NormalizeName(const TStringBuf& name) { return result; } +void TDefaultTypeAnnotationVisitor::Visit(const TUnitExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TMultiExprType& type) { + for (const auto& i : type.GetItems()) { + i->Accept(*this); + } +} + +void TDefaultTypeAnnotationVisitor::Visit(const TTupleExprType& type) { + for (const auto& i : type.GetItems()) { + i->Accept(*this); + } +} + +void TDefaultTypeAnnotationVisitor::Visit(const TStructExprType& type) { + for (const auto& i : type.GetItems()) { + i->Accept(*this); + } +} + +void TDefaultTypeAnnotationVisitor::Visit(const TItemExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TListExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TStreamExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TFlowExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TDataExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TPgExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TWorldExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TOptionalExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TCallableExprType& type) { + type.GetReturnType()->Accept(*this); + for (const auto& arg : type.GetArguments()) { + arg.Type->Accept(*this); + } +} + +void TDefaultTypeAnnotationVisitor::Visit(const TResourceExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TTypeExprType& type) { + type.GetType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TDictExprType& type) { + type.GetKeyType()->Accept(*this); + type.GetPayloadType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TVoidExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TNullExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TGenericExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TTaggedExprType& type) { + type.GetBaseType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TErrorExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TVariantExprType& type) { + type.GetUnderlyingType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TEmptyListExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TEmptyDictExprType& type) { + Y_UNUSED(type); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TBlockExprType& type) { + type.GetItemType()->Accept(*this); +} + +void TDefaultTypeAnnotationVisitor::Visit(const TScalarExprType& type) { + type.GetItemType()->Accept(*this); +} + +TErrorTypeVisitor::TErrorTypeVisitor(TExprContext& ctx) + : Ctx_(ctx) +{} + +void TErrorTypeVisitor::Visit(const TErrorExprType& type) { + HasErrors_ = true; + Ctx_.IssueManager.RaiseIssue(type.GetError()); +} + +bool TErrorTypeVisitor::HasErrors() const { + return HasErrors_; +} + } // namespace NYql template<> diff --git a/yql/essentials/ast/yql_expr.h b/yql/essentials/ast/yql_expr.h index 82bd254a44d..a64a2f9aec6 100644 --- a/yql/essentials/ast/yql_expr.h +++ b/yql/essentials/ast/yql_expr.h @@ -121,6 +121,47 @@ struct TTypeAnnotationVisitor { virtual void Visit(const TScalarExprType& type) = 0; }; +struct TDefaultTypeAnnotationVisitor : public TTypeAnnotationVisitor { + void Visit(const TUnitExprType& type) override; + void Visit(const TMultiExprType& type) override; + void Visit(const TTupleExprType& type) override; + void Visit(const TStructExprType& type) override; + void Visit(const TItemExprType& type) override; + void Visit(const TListExprType& type) override; + void Visit(const TStreamExprType& type) override; + void Visit(const TFlowExprType& type) override; + void Visit(const TDataExprType& type) override; + void Visit(const TPgExprType& type) override; + void Visit(const TWorldExprType& type) override; + void Visit(const TOptionalExprType& type) override; + void Visit(const TCallableExprType& type) override; + void Visit(const TResourceExprType& type) override; + void Visit(const TTypeExprType& type) override; + void Visit(const TDictExprType& type) override; + void Visit(const TVoidExprType& type) override; + void Visit(const TNullExprType& type) override; + void Visit(const TGenericExprType& type) override; + void Visit(const TTaggedExprType& type) override; + void Visit(const TErrorExprType& type) override; + void Visit(const TVariantExprType& type) override; + void Visit(const TEmptyListExprType& type) override; + void Visit(const TEmptyDictExprType& type) override; + void Visit(const TBlockExprType& type) override; + void Visit(const TScalarExprType& type) override; +}; + +class TErrorTypeVisitor : public TDefaultTypeAnnotationVisitor +{ +public: + TErrorTypeVisitor(TExprContext& ctx); + void Visit(const TErrorExprType& type) override; + bool HasErrors() const; + +private: + TExprContext& Ctx_; + bool HasErrors_ = false; +}; + enum ETypeAnnotationFlags : ui32 { TypeNonComposable = 0x01, TypeNonPersistable = 0x02, diff --git a/yql/essentials/minikql/benchmark/pack_num/metrics/ya.make b/yql/essentials/minikql/benchmark/pack_num/metrics/ya.make index 398f74568f8..2573d870dd0 100644 --- a/yql/essentials/minikql/benchmark/pack_num/metrics/ya.make +++ b/yql/essentials/minikql/benchmark/pack_num/metrics/ya.make @@ -16,4 +16,6 @@ DEPENDS( yql/essentials/minikql/benchmark/pack_num ) +INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) + END() diff --git a/yql/essentials/minikql/benchmark/pack_num/ya.make b/yql/essentials/minikql/benchmark/pack_num/ya.make index 2f17895c2c1..32397ee8e13 100644 --- a/yql/essentials/minikql/benchmark/pack_num/ya.make +++ b/yql/essentials/minikql/benchmark/pack_num/ya.make @@ -16,4 +16,6 @@ SRCS( pack_num_bench.cpp ) +INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) + END() diff --git a/yql/essentials/parser/pg_wrapper/test/ya.make b/yql/essentials/parser/pg_wrapper/test/ya.make index 01a9249fd90..22ee755f7c3 100644 --- a/yql/essentials/parser/pg_wrapper/test/ya.make +++ b/yql/essentials/parser/pg_wrapper/test/ya.make @@ -10,6 +10,7 @@ IF (SANITIZER_TYPE OR WITH_VALGRIND) TIMEOUT(1800) SIZE(LARGE) TAG(ya:fat sb:ttl=2) + INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) ELSE() TIMEOUT(600) SIZE(MEDIUM) diff --git a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index e8cad5736f8..0dbc7ce4d26 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -178,11 +178,21 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, try { TType* mkqlUserType = nullptr; if (udf.UserType) { + // scan for error types + TErrorTypeVisitor errorVisitor(ctx); + udf.UserType->Accept(errorVisitor); + if (errorVisitor.HasErrors()) { + hasErrors = true; + continue; + } + TStringStream err; mkqlUserType = BuildType(*udf.UserType, {env}, err);// if (!mkqlUserType) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Invalid user type for function: " - << udf.Name << ", error: " << err.Str())); + auto issue = TIssue(udf.Pos, TStringBuilder() << "Invalid user type for function: " + << udf.Name << ", error: " << err.Str()); + issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL); + ctx.AddError(issue); hasErrors = true; continue; } @@ -216,9 +226,11 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, udf.SupportsBlocks = funcInfo.SupportsBlocks; udf.IsStrict = funcInfo.IsStrict; } catch (const std::exception& e) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() + auto issue = TIssue(udf.Pos, TStringBuilder() << "Internal error was found when udf metadata is loading for function: " << udf.Name - << ", reason: " << e.what())); + << ", reason: " << e.what()); + issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL); + ctx.AddError(issue); hasErrors = true; } } diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in index d28661e68e1..9a8388b3407 100644 --- a/yql/essentials/sql/v1/SQLv1.g.in +++ b/yql/essentials/sql/v1/SQLv1.g.in @@ -79,6 +79,7 @@ sql_stmt_core: | create_transfer_stmt | alter_transfer_stmt | drop_transfer_stmt + | alter_database_stmt ; expr: @@ -691,6 +692,8 @@ table_tablestore: TABLESTORE simple_table_ref_core; table_settings_entry: an_id EQUALS table_setting_value; table_as_source: AS values_source; +alter_database_stmt: ALTER DATABASE an_id_schema OWNER TO role_name; + alter_table_stmt: ALTER TABLE simple_table_ref alter_table_action (COMMA alter_table_action)*; alter_table_action: alter_table_add_column @@ -1430,6 +1433,7 @@ keyword_as_compat: | OTHERS // | OUTER // | OVER + | OWNER | PARALLEL | PARTITION | PASSING @@ -1658,6 +1662,7 @@ keyword_compat: ( | OTHERS | OUTER | OVER + | OWNER | PARALLEL | PARTITION | PASSING @@ -2022,6 +2027,7 @@ ORDER: O R D E R; OTHERS: O T H E R S; OUTER: O U T E R; OVER: O V E R; +OWNER: O W N E R; PARALLEL: P A R A L L E L; PARTITION: P A R T I T I O N; PASSING: P A S S I N G; diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in index 3f4089a93b0..20359450c97 100644 --- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in +++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in @@ -78,6 +78,7 @@ sql_stmt_core: | create_transfer_stmt | alter_transfer_stmt | drop_transfer_stmt + | alter_database_stmt ; expr: @@ -683,6 +684,8 @@ backup_collection_settings_entry: an_id EQUALS table_setting_value; backup_stmt: BACKUP object_ref (INCREMENTAL)?; restore_stmt: RESTORE object_ref (AT STRING_VALUE)?; +alter_database_stmt: ALTER DATABASE an_id_schema OWNER TO role_name; + table_inherits: INHERITS LPAREN simple_table_ref_core (COMMA simple_table_ref_core)* RPAREN; table_partition_by: PARTITION BY HASH pure_column_list; with_table_settings: WITH LPAREN table_settings_entry (COMMA table_settings_entry)* RPAREN; @@ -1430,6 +1433,7 @@ keyword_as_compat: | OTHERS // | OUTER // | OVER + | OWNER | PARALLEL | PARTITION | PASSING @@ -1658,6 +1662,7 @@ keyword_compat: ( | OTHERS | OUTER | OVER + | OWNER | PARALLEL | PARTITION | PASSING @@ -2022,6 +2027,7 @@ ORDER: O R D E R; OTHERS: O T H E R S; OUTER: O U T E R; OVER: O V E R; +OWNER: O W N E R; PARALLEL: P A R A L L E L; PARTITION: P A R T I T I O N; PASSING: P A S S I N G; diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp index 1240e073537..0ede01dd8fd 100644 --- a/yql/essentials/sql/v1/format/sql_format.cpp +++ b/yql/essentials/sql/v1/format/sql_format.cpp @@ -863,6 +863,11 @@ private: ExprLineIndent = 0; } + void VisitAlterDatabase(const TRule_alter_database_stmt& msg) { + NewLine(); + VisitAllFields(TRule_alter_database_stmt::GetDescriptor(), msg); + } + void VisitCreateTable(const TRule_create_table_stmt& msg) { NewLine(); Visit(msg.GetToken1()); @@ -3015,6 +3020,7 @@ TStaticData::TStaticData() {TRule_backup_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitBackup)}, {TRule_restore_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitRestore)}, {TRule_alter_sequence_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterSequence)}, + {TRule_alter_database_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterDatabase)}, }) , ObfuscatingVisitDispatch({ {TToken::GetDescriptor(), MakeObfuscatingFunctor(&TObfuscatingVisitor::VisitToken)}, diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h index b7fbc66f937..15ab334a91e 100644 --- a/yql/essentials/sql/v1/format/sql_format_ut.h +++ b/yql/essentials/sql/v1/format/sql_format_ut.h @@ -20,6 +20,15 @@ Y_UNIT_TEST(DotAfterDigits) { setup.Run(cases); } +Y_UNIT_TEST(AlterDatabase) { + TCases cases { + {"use plato;alter database `/Root/test` owner to user1;", "USE plato;\n\nALTER DATABASE `/Root/test` OWNER TO user1;\n"}, + }; + + TSetup setup; + setup.Run(cases); +} + Y_UNIT_TEST(GrantPermissions) { TCases cases { {"use plato;grant connect, modify tables, list on `/Root` to user;", "USE plato;\n\nGRANT CONNECT, MODIFY TABLES, LIST ON `/Root` TO user;\n"}, diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h index f3e3f5f5808..027b37bb69a 100644 --- a/yql/essentials/sql/v1/node.h +++ b/yql/essentials/sql/v1/node.h @@ -1277,6 +1277,11 @@ namespace NSQLTranslationV1 { bool Temporary = false; }; + struct TAlterDatabaseParameters { + TString DbPath; + std::optional<TDeferredAtom> Owner; + }; + struct TTableRef; struct TAnalyzeParams { std::shared_ptr<TTableRef> Table; diff --git a/yql/essentials/sql/v1/query.cpp b/yql/essentials/sql/v1/query.cpp index f80398b6cee..0b8b7c07a7b 100644 --- a/yql/essentials/sql/v1/query.cpp +++ b/yql/essentials/sql/v1/query.cpp @@ -1314,6 +1314,72 @@ TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, b return new TCreateTableNode(pos, tr, existingOk, replaceIfExists, params, std::move(values), scoped); } +class TAlterDatabaseNode final : public TAstListNode { +public: + TAlterDatabaseNode( + TPosition pos, + const TString& service, + const TDeferredAtom& cluster, + const TAlterDatabaseParameters& params, + TScopedStatePtr scoped + ) + : TAstListNode(pos) + , Params(params) + , Scoped(scoped) + , Cluster(cluster) + , Service(service) + { + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + auto options = Y(Q(Y(Q("mode"), Q("alterDatabase")))); + + options = L(options, Q(Y(Q("dbPath"), Q(Params.DbPath)))); + + if (Params.Owner.has_value()) { + options = L(options, Q(Y(Q("owner"), Q(Params.Owner.value().Build())))); + } + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), + Y("let", "world", Y(TString(WriteName), "world", "sink", Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, src); + } + + TPtr DoClone() const final { + return {}; + } + +private: + const TAlterDatabaseParameters Params; + TScopedStatePtr Scoped; + TDeferredAtom Cluster; + TString Service; +}; + +TNodePtr BuildAlterDatabase( + TPosition pos, + const TString& service, + const TDeferredAtom& cluster, + const TAlterDatabaseParameters& params, + TScopedStatePtr scoped +) { + return new TAlterDatabaseNode( + pos, + service, + cluster, + params, + scoped + ); +} + + class TAlterTableNode final : public TAstListNode { public: TAlterTableNode(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) diff --git a/yql/essentials/sql/v1/source.h b/yql/essentials/sql/v1/source.h index 991657ed155..3d2fefcb43f 100644 --- a/yql/essentials/sql/v1/source.h +++ b/yql/essentials/sql/v1/source.h @@ -304,6 +304,8 @@ namespace NSQLTranslationV1 { TNodePtr BuildBatchDelete(TPosition pos, TScopedStatePtr scoped, const TTableRef& table, TSourcePtr source, TNodePtr options = nullptr); // Implemented in query.cpp + TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); + TNodePtr BuildAlterDatabase(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TAlterDatabaseParameters& params,TScopedStatePtr scoped); TNodePtr BuildTableKey(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TViewDescription& view); TNodePtr BuildTableKeys(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TString& func, const TVector<TTableArg>& args); TNodePtr BuildTopicKey(TPosition pos, const TDeferredAtom& cluster, const TDeferredAtom& name); diff --git a/yql/essentials/sql/v1/sql.cpp b/yql/essentials/sql/v1/sql.cpp index b7a7ef9e8f2..9e7c052b928 100644 --- a/yql/essentials/sql/v1/sql.cpp +++ b/yql/essentials/sql/v1/sql.cpp @@ -181,6 +181,7 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { case TRule_sql_stmt_core::kAltSqlStmtCore58: // create transfer case TRule_sql_stmt_core::kAltSqlStmtCore59: // alter transfer case TRule_sql_stmt_core::kAltSqlStmtCore60: // drop transfer + case TRule_sql_stmt_core::kAltSqlStmtCore61: // alter database return false; } } diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp index 7510acc4679..5db4995768d 100644 --- a/yql/essentials/sql/v1/sql_query.cpp +++ b/yql/essentials/sql/v1/sql_query.cpp @@ -1925,6 +1925,30 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& node.HasBlock4(), context)); break; } + case TRule_sql_stmt_core::kAltSqlStmtCore61: { + // alter_database_stmt: ALTER DATABASE an_id_schema OWNER TO role_name + auto& node = core.GetAlt_sql_stmt_core61().GetRule_alter_database_stmt1(); + + TDeferredAtom roleName; + { + bool allowSystemRoles = true; + if (!RoleNameClause(node.GetRule_role_name6(), roleName, allowSystemRoles)) { + return false; + } + } + + TAlterDatabaseParameters alterDatabaseParams; + alterDatabaseParams.Owner = roleName; + alterDatabaseParams.DbPath = Id(node.GetRule_an_id_schema3(), *this); + + const TPosition pos = Ctx.Pos(); + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + + auto stmt = BuildAlterDatabase(pos, service, cluster, alterDatabaseParams, Ctx.Scoped); + AddStatementToBlocks(blocks, stmt); + break; + } case TRule_sql_stmt_core::ALT_NOT_SET: Ctx.IncrementMonCounter("sql_errors", "UnknownStatement" + internalStatementName); AltNotImplemented("sql_stmt_core", core); diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index bd385c8feb5..d850dac37b5 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -969,6 +969,23 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["primarykey"]); } + Y_UNIT_TEST(AlterDatabaseAst) { + NYql::TAstParseResult request = SqlToYql("USE plato; ALTER DATABASE `/Root/test` OWNER TO user1;"); + UNIT_ASSERT(request.IsOk()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( + R"(let world (Write! world sink '('('mode 'alterDatabase) '('dbPath '/Root/test) '('owner ''"user1"))))" + )); + }; + + TWordCountHive elementStat({TString("\'mode \'alterDatabase")}); + VerifyProgram(request, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alterDatabase"]); + } + Y_UNIT_TEST(CreateTableNonNullableYqlTypeAstCorrect) { NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null);"); UNIT_ASSERT(res.Root); diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 9463408886e..b591fd15a19 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -1087,6 +1087,23 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["primarykey"]); } + Y_UNIT_TEST(AlterDatabaseAst) { + NYql::TAstParseResult request = SqlToYql("USE plato; ALTER DATABASE `/Root/test` OWNER TO user1;"); + UNIT_ASSERT(request.IsOk()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( + R"(let world (Write! world sink '('('mode 'alterDatabase) '('dbPath '/Root/test) '('owner ''"user1"))))" + )); + }; + + TWordCountHive elementStat({TString("\'mode \'alterDatabase")}); + VerifyProgram(request, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alterDatabase"]); + } + Y_UNIT_TEST(CreateTableNonNullableYqlTypeAstCorrect) { NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null);"); UNIT_ASSERT(res.Root); diff --git a/yql/essentials/tests/sql/minirun/pure.make b/yql/essentials/tests/sql/minirun/pure.make index e76272da52d..9045ab914a8 100644 --- a/yql/essentials/tests/sql/minirun/pure.make +++ b/yql/essentials/tests/sql/minirun/pure.make @@ -14,6 +14,7 @@ IF (SANITIZER_TYPE OR WITH_VALGRIND) TIMEOUT(1800) SIZE(LARGE) TAG(ya:fat sb:ttl=2) + INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) ELSE() TIMEOUT(600) SIZE(MEDIUM) diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index 8b6291d7191..cd350352392 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -6950,6 +6950,13 @@ "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_udf-complex_return_type_/sql.yql" } ], + "test_sql2yql.test[udf-error_type]": [ + { + "checksum": "f45c5bc8c840d0fc7d5ff8fb667b3437", + "size": 1125, + "uri": "https://{canondata_backend}/1916746/f97ecdac9f904e3c699edd8e02d8d57adc458252/resource.tar.gz#test_sql2yql.test_udf-error_type_/sql.yql" + } + ], "test_sql2yql.test[udf-generic_udf]": [ { "checksum": "7b52e62f76aceb26c896d539c9d3ece6", @@ -10740,6 +10747,11 @@ "uri": "file://test_sql_format.test_udf-complex_return_type_/formatted.sql" } ], + "test_sql_format.test[udf-error_type]": [ + { + "uri": "file://test_sql_format.test_udf-error_type_/formatted.sql" + } + ], "test_sql_format.test[udf-generic_udf]": [ { "uri": "file://test_sql_format.test_udf-generic_udf_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-error_type_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-error_type_/formatted.sql new file mode 100644 index 00000000000..67c0c1cab61 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-error_type_/formatted.sql @@ -0,0 +1,6 @@ +/* custom error: Member not found: Json */ +SELECT + Yson::From(value, Json) +FROM + AS_TABLE([<|value: 1|>]) +; diff --git a/yql/essentials/tests/sql/sql2yql/ya.make b/yql/essentials/tests/sql/sql2yql/ya.make index cff97cecd9e..fd234dd8219 100644 --- a/yql/essentials/tests/sql/sql2yql/ya.make +++ b/yql/essentials/tests/sql/sql2yql/ya.make @@ -15,6 +15,7 @@ IF (SANITIZER_TYPE) TIMEOUT(1800) SIZE(LARGE) TAG(ya:fat sb:ttl=2) + INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) ELSE() TIMEOUT(600) SIZE(MEDIUM) diff --git a/yql/essentials/tests/sql/suites/udf/error_type.cfg b/yql/essentials/tests/sql/suites/udf/error_type.cfg new file mode 100644 index 00000000000..61364a1d2ce --- /dev/null +++ b/yql/essentials/tests/sql/suites/udf/error_type.cfg @@ -0,0 +1,3 @@ +xfail +udf yson2_udf + diff --git a/yql/essentials/tests/sql/suites/udf/error_type.sql b/yql/essentials/tests/sql/suites/udf/error_type.sql new file mode 100644 index 00000000000..de19bfe556c --- /dev/null +++ b/yql/essentials/tests/sql/suites/udf/error_type.sql @@ -0,0 +1,4 @@ +/* custom error: Member not found: Json */ +SELECT + Yson::From(value, Json) +FROM AS_TABLE([<|value:1|>]) diff --git a/yql/essentials/types/binary_json/ut_benchmark/ya.make b/yql/essentials/types/binary_json/ut_benchmark/ya.make index 8e64578bad9..01e22c5ac05 100644 --- a/yql/essentials/types/binary_json/ut_benchmark/ya.make +++ b/yql/essentials/types/binary_json/ut_benchmark/ya.make @@ -27,4 +27,6 @@ PEERDIR( YQL_LAST_ABI_VERSION() +INCLUDE(${ARCADIA_ROOT}/devtools/large_on_multi_slots.inc) + END() diff --git a/yql/essentials/udfs/common/unicode_base/lib/unicode_base_udf.h b/yql/essentials/udfs/common/unicode_base/lib/unicode_base_udf.h index 52bfbb2785f..4a852a5a6f6 100644 --- a/yql/essentials/udfs/common/unicode_base/lib/unicode_base_udf.h +++ b/yql/essentials/udfs/common/unicode_base/lib/unicode_base_udf.h @@ -218,7 +218,7 @@ namespace { static void SplitToListImpl( const IValueBuilder* valueBuilder, const TUnboxedValue& input, - const TUtf16String::const_iterator start, + const TUtf32String::const_iterator start, const TIt& it, TTmpVector& result) { const std::string_view& original = input.AsStringRef(); @@ -281,8 +281,8 @@ namespace { SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result); } } else { - const auto& input = UTF8ToWide(args[0].AsStringRef()); - const auto& delimeter = UTF8ToWide(args[1].AsStringRef()); + const auto& input = UTF8ToUTF32<true>(args[0].AsStringRef()); + const auto& delimeter = UTF8ToUTF32<true>(args[1].AsStringRef()); if (limit) { auto it = StringSplitter(input).SplitBySet(delimeter.c_str()).Limit(limit + 1); SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result); @@ -313,9 +313,9 @@ namespace { Y_UNUSED(valueBuilder); const TStringBuf left(args[0].AsStringRef()); const TStringBuf right(args[1].AsStringRef()); - const TUtf16String& leftWide = UTF8ToWide(left); - const TUtf16String& rightWide = UTF8ToWide(right); - const ui64 result = NLevenshtein::Distance(leftWide, rightWide); + const auto& leftUtf32 = UTF8ToUTF32<true>(left); + const auto& rightUtf32 = UTF8ToUTF32<true>(right); + const ui64 result = NLevenshtein::Distance(leftUtf32, rightUtf32); return TUnboxedValuePod(result); } diff --git a/yql/essentials/udfs/common/unicode_base/test/canondata/result.json b/yql/essentials/udfs/common/unicode_base/test/canondata/result.json index c9012f2333d..8d19afc4281 100644 --- a/yql/essentials/udfs/common/unicode_base/test/canondata/result.json +++ b/yql/essentials/udfs/common/unicode_base/test/canondata/result.json @@ -9,6 +9,11 @@ "uri": "file://test.test_IsCategory_/results.txt" } ], + "test.test[LevensteinDistanceCodePoints]": [ + { + "uri": "file://test.test_LevensteinDistanceCodePoints_/results.txt" + } + ], "test.test[List]": [ { "uri": "file://test.test_List_/results.txt" @@ -24,6 +29,11 @@ "uri": "file://test.test_Replace_/results.txt" } ], + "test.test[SplitToListNoCrash]": [ + { + "uri": "file://test.test_SplitToListNoCrash_/results.txt" + } + ], "test.test[Strip]": [ { "uri": "file://test.test_Strip_/results.txt" diff --git a/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_LevensteinDistanceCodePoints_/results.txt b/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_LevensteinDistanceCodePoints_/results.txt new file mode 100644 index 00000000000..0f35bd34bdf --- /dev/null +++ b/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_LevensteinDistanceCodePoints_/results.txt @@ -0,0 +1,28 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "Uint64" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "1" + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_SplitToListNoCrash_/results.txt b/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_SplitToListNoCrash_/results.txt new file mode 100644 index 00000000000..8516052a94d --- /dev/null +++ b/yql/essentials/udfs/common/unicode_base/test/canondata/test.test_SplitToListNoCrash_/results.txt @@ -0,0 +1,33 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "ListType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "\xF0\x90\x8E\x80" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/unicode_base/test/cases/LevensteinDistanceCodePoints.sql b/yql/essentials/udfs/common/unicode_base/test/cases/LevensteinDistanceCodePoints.sql new file mode 100644 index 00000000000..ec072b726aa --- /dev/null +++ b/yql/essentials/udfs/common/unicode_base/test/cases/LevensteinDistanceCodePoints.sql @@ -0,0 +1,2 @@ +SELECT + Unicode::LevensteinDistance("\xF0\x90\x8E\x80"u, "\xF0\x9B\x80\x80"u) diff --git a/yql/essentials/udfs/common/unicode_base/test/cases/SplitToListNoCrash.sql b/yql/essentials/udfs/common/unicode_base/test/cases/SplitToListNoCrash.sql new file mode 100644 index 00000000000..643292ea0b8 --- /dev/null +++ b/yql/essentials/udfs/common/unicode_base/test/cases/SplitToListNoCrash.sql @@ -0,0 +1,2 @@ +SELECT + Unicode::SplitToList("\xF0\x90\x8E\x80"u, "\xF0\x90\x8E\x81"u, false AS DelimeterString) diff --git a/yt/cpp/mapreduce/http_client/raw_client.cpp b/yt/cpp/mapreduce/http_client/raw_client.cpp index 3586751191e..378fbdfde6f 100644 --- a/yt/cpp/mapreduce/http_client/raw_client.cpp +++ b/yt/cpp/mapreduce/http_client/raw_client.cpp @@ -244,7 +244,9 @@ void THttpRawClient::Concatenate( THttpHeader header("POST", "concatenate"); header.AddMutationId(); header.MergeParameters(NRawClient::SerializeParamsForConcatenate(transactionId, Context_.Config->Prefix, sourcePaths, destinationPath, options)); - RequestWithoutRetry(Context_, mutationId, header)->GetResponse(); + TRequestConfig config; + config.IsHeavy = true; + RequestWithoutRetry(Context_, mutationId, header, /*body*/ {}, config)->GetResponse(); } TTransactionId THttpRawClient::StartTransaction( diff --git a/yt/yql/providers/yt/provider/yql_yt_logical_optimize.cpp b/yt/yql/providers/yt/provider/yql_yt_logical_optimize.cpp index 55c3956a7cd..caf99f54d51 100644 --- a/yt/yql/providers/yt/provider/yql_yt_logical_optimize.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_logical_optimize.cpp @@ -230,6 +230,7 @@ protected: mapOut.RowSpec->CopySortness(ctx, TYqlRowSpecInfo(map.Output().Item(0).RowSpec())); } mapOut.SetUnique(path.Ref().GetConstraint<TDistinctConstraintNode>(), map.Mapper().Pos(), ctx); + mapOut.RowSpec->SetConstraints(path.Ref().GetConstraintSet()); TExprBase newColumns = Build<TCoVoid>(ctx, path.Pos()).Done(); if (keepColumns) { @@ -271,6 +272,7 @@ protected: mergeOut.RowSpec->CopyTypeOrders(*nativeType); } mergeOut.SetUnique(path.Ref().GetConstraint<TDistinctConstraintNode>(), merge.Pos(), ctx); + mergeOut.RowSpec->SetConstraints(path.Ref().GetConstraintSet()); TSet<TStringBuf> columnSet(effectiveColumns.begin(), effectiveColumns.end()); if (mergeOut.RowSpec->HasAuxColumns()) { diff --git a/yt/yt/client/object_client/helpers.cpp b/yt/yt/client/object_client/helpers.cpp index 6fe11c337e6..6dbea8270cc 100644 --- a/yt/yt/client/object_client/helpers.cpp +++ b/yt/yt/client/object_client/helpers.cpp @@ -46,6 +46,7 @@ bool IsVersionedType(EObjectType type) type == EObjectType::ChunkMap || type == EObjectType::LostChunkMap || type == EObjectType::LostVitalChunkMap || + type == EObjectType::LostVitalChunksSampleMap || type == EObjectType::PrecariousChunkMap || type == EObjectType::PrecariousVitalChunkMap || type == EObjectType::OverreplicatedChunkMap || diff --git a/yt/yt/client/object_client/public.h b/yt/yt/client/object_client/public.h index 4d006c78733..4830121acc7 100644 --- a/yt/yt/client/object_client/public.h +++ b/yt/yt/client/object_client/public.h @@ -146,6 +146,7 @@ DEFINE_ENUM(EObjectType, ((ChunkMap) (402)) ((LostChunkMap) (403)) ((LostVitalChunkMap) (413)) + ((LostVitalChunksSampleMap) (464)) ((PrecariousChunkMap) (410)) ((PrecariousVitalChunkMap) (411)) ((OverreplicatedChunkMap) (404)) diff --git a/yt/yt/client/queue_client/config.h b/yt/yt/client/queue_client/config.h index de862c1e6bd..a8090b64545 100644 --- a/yt/yt/client/queue_client/config.h +++ b/yt/yt/client/queue_client/config.h @@ -64,7 +64,7 @@ bool operator==(const TQueueAutoTrimConfig& lhs, const TQueueAutoTrimConfig& rhs //////////////////////////////////////////////////////////////////////////////// class TQueueStaticExportConfig - : public NYTree::TYsonStructLite + : public NYTree::TYsonStruct { public: //! Export will be performed at times that are multiple of this period. @@ -95,11 +95,13 @@ public: //! commit_ordering=%false queues, since commit timestamp monotonicty within a tablet is not guaranteed for them. bool UseUpperBoundForTableNames; - REGISTER_YSON_STRUCT_LITE(TQueueStaticExportConfig); + REGISTER_YSON_STRUCT(TQueueStaticExportConfig); static void Register(TRegistrar registrar); }; +DEFINE_REFCOUNTED_TYPE(TQueueStaticExportConfig) + bool operator==(const TQueueStaticExportConfig& lhs, const TQueueStaticExportConfig& rhs); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/queue_client/consumer_client.cpp b/yt/yt/client/queue_client/consumer_client.cpp index e17e2df3945..d3d8a7cb223 100644 --- a/yt/yt/client/queue_client/consumer_client.cpp +++ b/yt/yt/client/queue_client/consumer_client.cpp @@ -671,6 +671,11 @@ ISubConsumerClientPtr CreateSubConsumerClient( TRichYPath queuePath) { auto queueCluster = queuePath.GetCluster(); + if (!queueCluster && queueClusterClient) { + if (auto queueClusterFromClient = queueClusterClient->GetClusterName()) { + queueCluster = *queueClusterFromClient; + } + } if (!queueCluster) { if (auto clientCluster = consumerClusterClient->GetClusterName()) { queueCluster = *clientCluster; diff --git a/yt/yt/client/queue_client/public.h b/yt/yt/client/queue_client/public.h index f9a0e2e1324..7a7a3f2eb91 100644 --- a/yt/yt/client/queue_client/public.h +++ b/yt/yt/client/queue_client/public.h @@ -30,6 +30,7 @@ DECLARE_REFCOUNTED_STRUCT(IProducerSession) DECLARE_REFCOUNTED_STRUCT(IPartitionReader) DECLARE_REFCOUNTED_CLASS(TPartitionReaderConfig) +DECLARE_REFCOUNTED_CLASS(TQueueStaticExportConfig) DECLARE_REFCOUNTED_CLASS(TQueueStaticExportDestinationConfig) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h index 3e3e77dc40b..01bd620ed26 100644 --- a/yt/yt/client/scheduler/public.h +++ b/yt/yt/client/scheduler/public.h @@ -166,7 +166,7 @@ DEFINE_ENUM(EAbortReason, DEFINE_ENUM_UNKNOWN_VALUE(EAbortReason, Unknown); -DEFINE_ENUM(EInterruptReason, +DEFINE_ENUM(EInterruptionReason, ((None) (0)) ((Preemption) (1)) ((UserRequest) (2)) @@ -175,7 +175,7 @@ DEFINE_ENUM(EInterruptReason, ((JobsDisabledOnNode) (5)) ); -DEFINE_ENUM_UNKNOWN_VALUE(EInterruptReason, Unknown); +DEFINE_ENUM_UNKNOWN_VALUE(EInterruptionReason, Unknown); DEFINE_ENUM(EAutoMergeMode, (Disabled) diff --git a/yt/yt/client/table_client/columnar_statistics.cpp b/yt/yt/client/table_client/columnar_statistics.cpp index 1639338d46b..8eca683f5ce 100644 --- a/yt/yt/client/table_client/columnar_statistics.cpp +++ b/yt/yt/client/table_client/columnar_statistics.cpp @@ -186,6 +186,8 @@ TLargeColumnarStatistics& TLargeColumnarStatistics::operator+=(const TLargeColum TColumnarStatistics& TColumnarStatistics::operator+=(const TColumnarStatistics& other) { + ReadDataSizeEstimate.reset(); + if (GetColumnCount() == 0) { Resize(other.GetColumnCount(), other.HasValueStatistics(), other.HasLargeStatistics()); } @@ -261,7 +263,8 @@ TLightweightColumnarStatistics TColumnarStatistics::MakeLightweightStatistics() return TLightweightColumnarStatistics{ .ColumnDataWeightsSum = std::accumulate(ColumnDataWeights.begin(), ColumnDataWeights.end(), (i64)0), .TimestampTotalWeight = TimestampTotalWeight, - .LegacyChunkDataWeight = LegacyChunkDataWeight + .LegacyChunkDataWeight = LegacyChunkDataWeight, + .ReadDataSizeEstimate = ReadDataSizeEstimate, }; } @@ -306,6 +309,8 @@ int TColumnarStatistics::GetColumnCount() const void TColumnarStatistics::Resize(int columnCount, bool keepValueStatistics, bool keepLargeStatistics) { + ReadDataSizeEstimate.reset(); + if (columnCount < GetColumnCount()) { // Downsizes are not allowed. If reducing column count, must clear the stats completely. YT_VERIFY(columnCount == 0); @@ -332,6 +337,8 @@ void TColumnarStatistics::Resize(int columnCount, bool keepValueStatistics, bool void TColumnarStatistics::Update(TRange<TUnversionedRow> rows) { + ReadDataSizeEstimate.reset(); + UpdateColumnarStatistics(*this, rows); if (ChunkRowCount) { @@ -341,6 +348,8 @@ void TColumnarStatistics::Update(TRange<TUnversionedRow> rows) void TColumnarStatistics::Update(TRange<TVersionedRow> rows) { + ReadDataSizeEstimate.reset(); + std::vector<TRange<TUnversionedValue>> keyColumnRows; keyColumnRows.reserve(rows.Size()); for (const auto& row : rows) { diff --git a/yt/yt/client/table_client/columnar_statistics.h b/yt/yt/client/table_client/columnar_statistics.h index 9ace07df3db..27af62003b6 100644 --- a/yt/yt/client/table_client/columnar_statistics.h +++ b/yt/yt/client/table_client/columnar_statistics.h @@ -16,6 +16,8 @@ struct TLightweightColumnarStatistics std::optional<i64> TimestampTotalWeight; //! Total data weight of legacy chunks whose meta misses columnar statistics. i64 LegacyChunkDataWeight = 0; + //! Data size expected to be read from disk. + std::optional<i64> ReadDataSizeEstimate; }; struct TNamedColumnarStatistics @@ -89,6 +91,12 @@ struct TColumnarStatistics //! Large per-column statistics, including columnar HLL. TLargeColumnarStatistics LargeStatistics; + //! This field provides an estimate, in bytes, of the data size expected to be read from disk. + //! It is calculated on nodes for specified columns set and cannot be recalculated on-the-fly. + //! To prevent errors, this field is set to null whenever the set of columns is modified or + //! when new rows are added to statistics. + std::optional<i64> ReadDataSizeEstimate; + TColumnarStatistics& operator+=(const TColumnarStatistics& other); bool operator==(const TColumnarStatistics& other) const = default; diff --git a/yt/yt/client/table_client/comparator.h b/yt/yt/client/table_client/comparator.h index 5221cec3e99..54fcb236dc1 100644 --- a/yt/yt/client/table_client/comparator.h +++ b/yt/yt/client/table_client/comparator.h @@ -105,7 +105,7 @@ using TPrefixComparer = TUUComparerSignature; int GetCompareSign(int value); -//! Obeys the usual rule: the result's sign incidates the comparison outcome. +//! Obeys the usual rule: the result's sign indicates the comparison outcome. //! Also |abs(result) - 1| is equal to index of first non-equal component. template <typename TComparer> int CompareKeys(TUnversionedValueRange lhs, TUnversionedValueRange rhs, const TComparer& prefixComparer); diff --git a/yt/yt/client/tablet_client/table_mount_cache.h b/yt/yt/client/tablet_client/table_mount_cache.h index 554977c8b09..d8c1d649483 100644 --- a/yt/yt/client/tablet_client/table_mount_cache.h +++ b/yt/yt/client/tablet_client/table_mount_cache.h @@ -51,7 +51,7 @@ DEFINE_REFCOUNTED_TYPE(TTabletInfo) struct TTableReplicaInfo final { TTableReplicaId ReplicaId; - TString ClusterName; + std::string ClusterName; NYPath::TYPath ReplicaPath; ETableReplicaMode Mode; }; diff --git a/yt/yt/core/concurrency/fair_share_thread_pool.cpp b/yt/yt/core/concurrency/fair_share_thread_pool.cpp index fb8dec13cf3..6169f944776 100644 --- a/yt/yt/core/concurrency/fair_share_thread_pool.cpp +++ b/yt/yt/core/concurrency/fair_share_thread_pool.cpp @@ -83,7 +83,6 @@ public: TWeakPtr<TFairShareQueue> Parent; TRingQueue<TEnqueuedAction> Queue; THeapItem* HeapIterator = nullptr; - i64 WaitTime = 0; TCpuDuration ExcessTime = 0; int CurrentExecutions = 0; @@ -264,10 +263,10 @@ public: auto tscp = NProfiling::TTscp::Get(); - TBucketPtr bucket; + i64 waitTime = 0; { auto guard = Guard(SpinLock_); - bucket = GetStarvingBucket(action, tscp); + auto bucket = GetStarvingBucket(action, tscp); if (!bucket) { return false; @@ -279,12 +278,12 @@ public: threadState.AccountedAt = tscp.Instant; action->StartedAt = tscp.Instant; - bucket->WaitTime = action->StartedAt - action->EnqueuedAt; + waitTime = action->StartedAt - action->EnqueuedAt; } YT_ASSERT(action && !action->Finished); - WaitTimeCounter_.Record(CpuDurationToDuration(bucket->WaitTime)); + WaitTimeCounter_.Record(CpuDurationToDuration(waitTime)); return true; } diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index d09b5f846ce..5325a74c3fa 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -1016,15 +1016,16 @@ public: void EnsureLocalHostName() { - if (Config_->LocalHostNameOverride) { + const auto config = Config_.Acquire(); + if (config->LocalHostNameOverride) { return; } - UpdateLocalHostName(Config_); + UpdateLocalHostName(config); YT_LOG_INFO("Localhost name determined via system call (LocalHostName: %v, ResolveHostNameIntoFqdn: %v)", GetLocalHostName(), - Config_->ResolveHostNameIntoFqdn); + config->ResolveHostNameIntoFqdn); } bool IsLocalAddress(const TNetworkAddress& address) @@ -1045,22 +1046,22 @@ public: void Configure(TAddressResolverConfigPtr config) { - Config_ = std::move(config); + Config_.Store(config); - SetDnsResolver(CreateAresDnsResolver(Config_)); - TAsyncExpiringCache::Reconfigure(Config_); + SetDnsResolver(CreateAresDnsResolver(config)); + TAsyncExpiringCache::Reconfigure(config); - if (Config_->LocalHostNameOverride) { - SetLocalHostName(*Config_->LocalHostNameOverride); + if (config->LocalHostNameOverride) { + SetLocalHostName(*config->LocalHostNameOverride); YT_LOG_INFO("Localhost name configured via config override (LocalHostName: %v)", - Config_->LocalHostNameOverride); + config->LocalHostNameOverride); } - UpdateLoopbackAddress(Config_); + UpdateLoopbackAddress(config); } private: - TAddressResolverConfigPtr Config_; + TAtomicIntrusivePtr<TAddressResolverConfig> Config_; std::atomic<bool> HasCachedLocalAddresses_ = false; std::vector<TNetworkAddress> CachedLocalAddresses_; @@ -1072,9 +1073,10 @@ private: TFuture<TNetworkAddress> DoGet(const std::string& hostName, bool /*isPeriodicUpdate*/) noexcept override { + const auto config = Config_.Acquire(); TDnsResolveOptions options{ - .EnableIPv4 = Config_->EnableIPv4, - .EnableIPv6 = Config_->EnableIPv6, + .EnableIPv4 = config->EnableIPv4, + .EnableIPv6 = config->EnableIPv6, }; return GetDnsResolver()->Resolve(hostName, options) .Apply(BIND([=] (const TErrorOr<TNetworkAddress>& result) { diff --git a/yt/yt/core/net/config.h b/yt/yt/core/net/config.h index 15cc198196f..97e4f3d7985 100644 --- a/yt/yt/core/net/config.h +++ b/yt/yt/core/net/config.h @@ -41,6 +41,7 @@ class TAddressResolverConfig public: bool EnableIPv4; bool EnableIPv6; + //! If true, when determining local host name, it will additionally be resolved //! into FQDN by calling |getaddrinfo|. Setting this option to false may be //! useful in MTN environment, in which hostnames are barely resolvable. @@ -48,9 +49,10 @@ public: //! exposed under localhost name to anyone; in particular, any kind of discovery //! should be done using some other kind of addresses. bool ResolveHostNameIntoFqdn; + //! If set, localhost name will be forcefully set to the given value rather - //! than retrieved via |NYT::NNet::UpdateLocalHostName|. - std::optional<TString> LocalHostNameOverride; + //! than retrieved from the system. + std::optional<std::string> LocalHostNameOverride; REGISTER_YSON_STRUCT(TAddressResolverConfig); diff --git a/yt/yt/library/process/config.cpp b/yt/yt/library/process/config.cpp deleted file mode 100644 index 9099aca7f07..00000000000 --- a/yt/yt/library/process/config.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "config.h" - -namespace NYT::NPipes { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void TIODispatcherConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("thread_pool_polling_period", &TThis::ThreadPoolPollingPeriod) - .Default(TDuration::MilliSeconds(10)); -} - -TIODispatcherConfigPtr TIODispatcherConfig::ApplyDynamic( - const TIODispatcherDynamicConfigPtr& dynamicConfig) const -{ - auto mergedConfig = CloneYsonStruct(MakeStrong(this)); - UpdateYsonStructField(mergedConfig->ThreadPoolPollingPeriod, dynamicConfig->ThreadPoolPollingPeriod); - mergedConfig->Postprocess(); - return mergedConfig; -} - -//////////////////////////////////////////////////////////////////////////////// - -void TIODispatcherDynamicConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("thread_pool_polling_period", &TThis::ThreadPoolPollingPeriod) - .Optional(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/config.h b/yt/yt/library/process/config.h deleted file mode 100644 index 84be0ef7beb..00000000000 --- a/yt/yt/library/process/config.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -class TIODispatcherConfig - : public NYTree::TYsonStruct -{ -public: - TDuration ThreadPoolPollingPeriod; - - TIODispatcherConfigPtr ApplyDynamic(const TIODispatcherDynamicConfigPtr& dynamicConfig) const; - - REGISTER_YSON_STRUCT(TIODispatcherConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TIODispatcherConfig) - -//////////////////////////////////////////////////////////////////////////////// - -class TIODispatcherDynamicConfig - : public NYTree::TYsonStruct -{ -public: - std::optional<TDuration> ThreadPoolPollingPeriod; - - REGISTER_YSON_STRUCT(TIODispatcherDynamicConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TIODispatcherDynamicConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/configure_io_dispatcher.cpp b/yt/yt/library/process/configure_io_dispatcher.cpp deleted file mode 100644 index d2f834b0f24..00000000000 --- a/yt/yt/library/process/configure_io_dispatcher.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "io_dispatcher.h" -#include "config.h" - -#include <yt/yt/core/misc/configurable_singleton_def.h> - -namespace NYT::NPipes { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void SetupSingletonConfigParameter(TYsonStructParameter<TIODispatcherConfigPtr>& parameter) -{ - parameter.DefaultNew(); -} - -void SetupSingletonConfigParameter(TYsonStructParameter<TIODispatcherDynamicConfigPtr>& parameter) -{ - parameter.DefaultNew(); -} - -void ConfigureSingleton(const TIODispatcherConfigPtr& config) -{ - TIODispatcher::Get()->Configure(config); -} - -void ReconfigureSingleton( - const TIODispatcherConfigPtr& config, - const TIODispatcherDynamicConfigPtr& dynamicConfig) -{ - TIODispatcher::Get()->Configure(config->ApplyDynamic(dynamicConfig)); -} - -YT_DEFINE_RECONFIGURABLE_SINGLETON( - "io_dispatcher", - TIODispatcherConfig, - TIODispatcherDynamicConfig); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/io_dispatcher.cpp b/yt/yt/library/process/io_dispatcher.cpp deleted file mode 100644 index 96e1f88087e..00000000000 --- a/yt/yt/library/process/io_dispatcher.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "io_dispatcher.h" - -#include "config.h" - -#include <yt/yt/core/concurrency/thread_pool_poller.h> -#include <yt/yt/core/concurrency/poller.h> - -namespace NYT::NPipes { - -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -TIODispatcher::TIODispatcher() - : Poller_(BIND([] { return CreateThreadPoolPoller(1, "Pipes"); })) -{ } - -TIODispatcher::~TIODispatcher() = default; - -TIODispatcher* TIODispatcher::Get() -{ - return Singleton<TIODispatcher>(); -} - -void TIODispatcher::Configure(const TIODispatcherConfigPtr& config) -{ - Poller_->SetPollingPeriod(config->ThreadPoolPollingPeriod); -} - -IInvokerPtr TIODispatcher::GetInvoker() -{ - return Poller_.Value()->GetInvoker(); -} - -IPollerPtr TIODispatcher::GetPoller() -{ - return Poller_.Value(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/io_dispatcher.h b/yt/yt/library/process/io_dispatcher.h deleted file mode 100644 index 3c47bddf784..00000000000 --- a/yt/yt/library/process/io_dispatcher.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/concurrency/public.h> - -#include <yt/yt/core/misc/lazy_ptr.h> - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -class TIODispatcher -{ -public: - static TIODispatcher* Get(); - ~TIODispatcher(); - - void Configure(const TIODispatcherConfigPtr& config); - - IInvokerPtr GetInvoker(); - NConcurrency::IPollerPtr GetPoller(); - -private: - TIODispatcher(); - - Y_DECLARE_SINGLETON_FRIEND() - - TLazyIntrusivePtr<NConcurrency::IThreadPoolPoller> Poller_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pipe.cpp b/yt/yt/library/process/pipe.cpp deleted file mode 100644 index e4051f6604f..00000000000 --- a/yt/yt/library/process/pipe.cpp +++ /dev/null @@ -1,266 +0,0 @@ -#include "pipe.h" -#include "private.h" -#include "io_dispatcher.h" - -#include <yt/yt/core/net/connection.h> - -#include <yt/yt/core/misc/proc.h> -#include <yt/yt/core/misc/fs.h> - -#include <sys/types.h> -#include <sys/stat.h> - -namespace NYT::NPipes { - -using namespace NNet; - -//////////////////////////////////////////////////////////////////////////////// - -static constexpr auto& Logger = PipesLogger; - -//////////////////////////////////////////////////////////////////////////////// - -TNamedPipe::TNamedPipe(const TString& path, std::optional<int> capacity, bool owning) - : Path_(path) - , Capacity_(capacity) - , Owning_(owning) -{ } - -TNamedPipe::~TNamedPipe() -{ - if (!Owning_) { - return; - } - - if (unlink(Path_.c_str()) == -1) { - YT_LOG_INFO(TError::FromSystem(), "Failed to unlink pipe %v", Path_); - } -} - -TNamedPipePtr TNamedPipe::Create(const TString& path, int permissions, std::optional<int> capacity) -{ - auto pipe = New<TNamedPipe>(path, capacity, /*owning*/ true); - pipe->Open(permissions); - YT_LOG_DEBUG("Named pipe created (Path: %v, Permissions: %v)", path, permissions); - return pipe; -} - -TNamedPipePtr TNamedPipe::FromPath(const TString& path) -{ - return New<TNamedPipe>(path, /*capacity*/ std::nullopt, /*owning*/ false); -} - -void TNamedPipe::Open(int permissions) -{ - if (mkfifo(Path_.c_str(), permissions) == -1) { - THROW_ERROR_EXCEPTION("Failed to create named pipe %v", Path_) - << TError::FromSystem(); - } -} - -IConnectionReaderPtr TNamedPipe::CreateAsyncReader() -{ - YT_VERIFY(!Path_.empty()); - return CreateInputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this)); -} - -IConnectionWriterPtr TNamedPipe::CreateAsyncWriter(bool useDeliveryFence) -{ - YT_VERIFY(!Path_.empty()); - return CreateOutputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this), Capacity_, useDeliveryFence); -} - -TString TNamedPipe::GetPath() const -{ - return Path_; -} - -//////////////////////////////////////////////////////////////////////////////// - -TNamedPipeConfigPtr TNamedPipeConfig::Create(TString path, int fd, bool write) -{ - auto result = New<TNamedPipeConfig>(); - result->Path = std::move(path); - result->FD = fd; - result->Write = write; - - return result; -} - -void TNamedPipeConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("path", &TThis::Path) - .Default(); - - registrar.Parameter("fd", &TThis::FD) - .Default(0); - - registrar.Parameter("write", &TThis::Write) - .Default(false); -} - -DEFINE_REFCOUNTED_TYPE(TNamedPipeConfig) - -//////////////////////////////////////////////////////////////////////////////// - -TPipe::TPipe() -{ } - -TPipe::TPipe(TPipe&& pipe) -{ - Init(std::move(pipe)); -} - -TPipe::TPipe(int fd[2]) - : ReadFD_(fd[0]) - , WriteFD_(fd[1]) -{ } - -void TPipe::Init(TPipe&& other) -{ - ReadFD_ = other.ReadFD_; - WriteFD_ = other.WriteFD_; - other.ReadFD_ = InvalidFD; - other.WriteFD_ = InvalidFD; -} - -TPipe::~TPipe() -{ - if (ReadFD_ != InvalidFD) { - YT_VERIFY(TryClose(ReadFD_, false)); - } - - if (WriteFD_ != InvalidFD) { - YT_VERIFY(TryClose(WriteFD_, false)); - } -} - -void TPipe::operator=(TPipe&& other) -{ - if (this == &other) { - return; - } - - Init(std::move(other)); -} - -IConnectionWriterPtr TPipe::CreateAsyncWriter() -{ - YT_VERIFY(WriteFD_ != InvalidFD); - SafeMakeNonblocking(WriteFD_); - return CreateConnectionFromFD(ReleaseWriteFD(), {}, {}, TIODispatcher::Get()->GetPoller()); -} - -IConnectionReaderPtr TPipe::CreateAsyncReader() -{ - YT_VERIFY(ReadFD_ != InvalidFD); - SafeMakeNonblocking(ReadFD_); - return CreateConnectionFromFD(ReleaseReadFD(), {}, {}, TIODispatcher::Get()->GetPoller()); -} - -int TPipe::ReleaseReadFD() -{ - YT_VERIFY(ReadFD_ != InvalidFD); - auto fd = ReadFD_; - ReadFD_ = InvalidFD; - return fd; -} - -int TPipe::ReleaseWriteFD() -{ - YT_VERIFY(WriteFD_ != InvalidFD); - auto fd = WriteFD_; - WriteFD_ = InvalidFD; - return fd; -} - -int TPipe::GetReadFD() const -{ - YT_VERIFY(ReadFD_ != InvalidFD); - return ReadFD_; -} - -int TPipe::GetWriteFD() const -{ - YT_VERIFY(WriteFD_ != InvalidFD); - return WriteFD_; -} - -void TPipe::CloseReadFD() -{ - if (ReadFD_ == InvalidFD) { - return; - } - auto fd = ReadFD_; - ReadFD_ = InvalidFD; - SafeClose(fd, false); -} - -void TPipe::CloseWriteFD() -{ - if (WriteFD_ == InvalidFD) { - return; - } - auto fd = WriteFD_; - WriteFD_ = InvalidFD; - SafeClose(fd, false); -} - -//////////////////////////////////////////////////////////////////////////////// - -void FormatValue(TStringBuilderBase* builder, const TPipe& pipe, TStringBuf spec) -{ - // TODO(arkady-e1ppa): We format pipe twice - // (pipe itself and its serialization) - // This is probably redundant. - // Check if it is later and remove - // the second step. - FormatValue( - builder, - Format( - "{ReadFD: %v, WriteFD: %v}", - pipe.GetReadFD(), - pipe.GetWriteFD()), - spec); -} - -//////////////////////////////////////////////////////////////////////////////// - -TPipeFactory::TPipeFactory(int minFD) - : MinFD_(minFD) -{ } - -TPipeFactory::~TPipeFactory() -{ - for (int fd : ReservedFDs_) { - YT_VERIFY(TryClose(fd, false)); - } -} - -TPipe TPipeFactory::Create() -{ - while (true) { - int fd[2]; - SafePipe(fd); - if (fd[0] >= MinFD_ && fd[1] >= MinFD_) { - TPipe pipe(fd); - return pipe; - } else { - ReservedFDs_.push_back(fd[0]); - ReservedFDs_.push_back(fd[1]); - } - } -} - -void TPipeFactory::Clear() -{ - for (int& fd : ReservedFDs_) { - YT_VERIFY(TryClose(fd, false)); - fd = TPipe::InvalidFD; - } - ReservedFDs_.clear(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pipe.h b/yt/yt/library/process/pipe.h deleted file mode 100644 index b4d1b3916d8..00000000000 --- a/yt/yt/library/process/pipe.h +++ /dev/null @@ -1,115 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/net/public.h> - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -class TNamedPipe - : public TRefCounted -{ -public: - ~TNamedPipe(); - static TNamedPipePtr Create(const TString& path, int permissions = 0660, std::optional<int> capacity = {}); - static TNamedPipePtr FromPath(const TString& path); - - NNet::IConnectionReaderPtr CreateAsyncReader(); - NNet::IConnectionWriterPtr CreateAsyncWriter(bool useDeliveryFence = false); - - TString GetPath() const; - -private: - const TString Path_; - const std::optional<int> Capacity_; - - //! Whether pipe was created by this class - //! and should be removed in destructor. - const bool Owning_; - - explicit TNamedPipe(const TString& path, std::optional<int> capacity, bool owning); - void Open(int permissions); - DECLARE_NEW_FRIEND() -}; - -DEFINE_REFCOUNTED_TYPE(TNamedPipe) - -//////////////////////////////////////////////////////////////////////////////// - -class TNamedPipeConfig - : public NYTree::TYsonStruct -{ -public: - TString Path; - int FD = 0; - bool Write = false; - - static TNamedPipeConfigPtr Create(TString path, int fd, bool write); - - REGISTER_YSON_STRUCT(TNamedPipeConfig); - - static void Register(TRegistrar registrar); -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TPipe - : public TNonCopyable -{ -public: - static const int InvalidFD = -1; - - TPipe(); - TPipe(TPipe&& pipe); - ~TPipe(); - - void operator=(TPipe&& other); - - void CloseReadFD(); - void CloseWriteFD(); - - NNet::IConnectionReaderPtr CreateAsyncReader(); - NNet::IConnectionWriterPtr CreateAsyncWriter(); - - int ReleaseReadFD(); - int ReleaseWriteFD(); - - int GetReadFD() const; - int GetWriteFD() const; - -private: - int ReadFD_ = InvalidFD; - int WriteFD_ = InvalidFD; - - TPipe(int fd[2]); - void Init(TPipe&& other); - - friend class TPipeFactory; -}; - -void FormatValue(TStringBuilderBase* builder, const TPipe& pipe, TStringBuf spec); - -//////////////////////////////////////////////////////////////////////////////// - -class TPipeFactory -{ -public: - explicit TPipeFactory(int minFD = 0); - ~TPipeFactory(); - - TPipe Create(); - - void Clear(); - -private: - const int MinFD_; - std::vector<int> ReservedFDs_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/private.h b/yt/yt/library/process/private.h deleted file mode 100644 index c185792e37e..00000000000 --- a/yt/yt/library/process/private.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include <yt/yt/core/logging/log.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -YT_DEFINE_GLOBAL(const NLogging::TLogger, PipesLogger, "Pipes"); -YT_DEFINE_GLOBAL(const NLogging::TLogger, PtyLogger, "Pty"); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/process.cpp b/yt/yt/library/process/process.cpp deleted file mode 100644 index 3895c688b7e..00000000000 --- a/yt/yt/library/process/process.cpp +++ /dev/null @@ -1,1041 +0,0 @@ -#include "process.h" -#include "pipe.h" - -#include <yt/yt/core/misc/proc.h> - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/misc/error.h> -#include <yt/yt/core/misc/fs.h> -#include <yt/yt/core/misc/finally.h> - -#include <yt/yt/core/concurrency/periodic_executor.h> -#include <yt/yt/core/concurrency/delayed_executor.h> - -#include <yt/yt/core/actions/invoker_util.h> - -#include <library/cpp/yt/system/handle_eintr.h> -#include <library/cpp/yt/system/exit.h> - -#include <util/folder/dirut.h> - -#include <util/generic/guid.h> - -#include <util/string/ascii.h> -#include <util/string/util.h> - -#include <util/system/env.h> -#include <util/system/execpath.h> -#include <util/system/maxlen.h> -#include <util/system/shellcommand.h> - -#ifdef _unix_ - #include <unistd.h> - #include <errno.h> - #include <sys/wait.h> - #include <sys/resource.h> - #include <spawn.h> -#endif - -#ifdef _darwin_ - #include <crt_externs.h> - #define environ (*_NSGetEnviron()) -#endif - -#if defined(__APPLE__) - #define YT_USE_POSIX_SPAWN_API -#endif - -#if defined(YT_USE_POSIX_SPAWN_API) - #include <spawn.h> -#endif - -namespace NYT { - -using namespace NPipes; -using namespace NNet; -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Process"); - -static constexpr pid_t InvalidProcessId = -1; - -static constexpr int ResolveRetryCount = 5; -static constexpr auto ResolveRetryTimeout = TDuration::Seconds(1); - -//////////////////////////////////////////////////////////////////////////////// - -#if defined(YT_USE_POSIX_SPAWN_API) - -class TPosixSpawnFileActions -{ -public: - TPosixSpawnFileActions() - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawn_file_actions_init(&Actions_) != 0, - TError("Failed to initialize spawn file actions object") - << TError::FromSystem()); - } - - ~TPosixSpawnFileActions() - { - auto res = ::posix_spawn_file_actions_destroy(&Actions_); - YT_LOG_FATAL_UNLESS( - res != 0, - "Failed to destroy spawn file actions object"); - } - - void AddDup2FileAction(int oldFD, int newFD) - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawn_file_actions_adddup2(&Actions_, oldFD, newFD) != 0, - TError("Failed to add dup2 file action (OldFD: %v, NewFD: %v)", oldFD, newFD) - << TError::FromSystem()); - } - - void AddChdirFileAction(TString path) - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawn_file_actions_addchdir_np(&Actions_, path.c_str()) != 0, - TError("Failed to add chdir file actions (Path: %v)", path) - << TError::FromSystem()); - } - - posix_spawn_file_actions_t* Get() - { - return &Actions_; - } - -private: - posix_spawn_file_actions_t Actions_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TPosixSpawnAttrs -{ -public: - TPosixSpawnAttrs() - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_init(&Actions_) != 0, - TError("Failed to initialize spawnattrs object") - << TError::FromSystem()); - } - - ~TPosixSpawnAttrs() - { - auto res = ::posix_spawnattr_destroy(&Actions_); - - YT_LOG_FATAL_UNLESS( - res != 0, - "Failed to destroy spawnattrs object"); - } - - void AddResetAllSignals() - { - sigset_t allBlocked; - sigfillset(&allBlocked); - - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_setsigdefault(&Actions_, &allBlocked) != 0, - TError("Failed to set default signal handlers for spawnattrs") - << TError::FromSystem()); - - UpdateFlags(POSIX_SPAWN_SETSIGDEF); - } - - void AddSetSigMask(sigset_t* sigset) - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_setsigmask(&Actions_, sigset) != 0, - TError("Failed to set signal mask for spawnattrs") - << TError::FromSystem()); - - UpdateFlags(POSIX_SPAWN_SETSIGMASK); - } - - void AddCreateProcessGroup() - { - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_setpgroup(&Actions_, 0) != 0, - TError("Failed to set pgroup value to 0 of spawnattrs") - << TError::FromSystem()); - - UpdateFlags(POSIX_SPAWN_SETPGROUP); - } - - posix_spawnattr_t* Get() - { - return &Actions_; - } - -private: - posix_spawnattr_t Actions_; - - void UpdateFlags(int mask) - { - auto currentFlags = GetFlags(); - - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_setflags(&Actions_, currentFlags | mask) != 0, - TError("Failed to set flags to spawnattrs (OldFlags: %v, Mask: %v)", currentFlags, mask) - << TError::FromSystem()); - } - - short GetFlags() const - { - short flags; - THROW_ERROR_EXCEPTION_IF( - ::posix_spawnattr_getflags(&Actions_, &flags) != 0, - TError("Failed to flags from spawnattrs") - << TError::FromSystem()); - return flags; - } -}; - -#endif - -//////////////////////////////////////////////////////////////////////////////// - -TErrorOr<TString> ResolveBinaryPath(const TString& binary) -{ - auto Logger = NYT::Logger() - .WithTag("Binary: %v", binary); - - YT_LOG_DEBUG("Resolving binary path"); - - std::vector<TError> accumulatedErrors; - - auto test = [&] (const char* path) { - YT_LOG_DEBUG("Probing path (Path: %v)", path); - if (access(path, R_OK | X_OK) == 0) { - return true; - } else { - auto error = TError("Cannot run %Qlv", path) << TError::FromSystem(); - accumulatedErrors.push_back(std::move(error)); - return false; - } - }; - - auto failure = [&] { - auto error = TError( - EProcessErrorCode::CannotResolveBinary, - "Cannot resolve binary %Qlv", - binary); - error.MutableInnerErrors()->swap(accumulatedErrors); - YT_LOG_DEBUG(error, "Error resolving binary path"); - return error; - }; - - auto success = [&] (const TString& path) { - YT_LOG_DEBUG("Binary resolved (Path: %v)", path); - return path; - }; - - if (test(binary.c_str())) { - return success(binary); - } - - // If this is an absolute path, stop here. - if (binary.empty() || binary[0] == '/') { - return failure(); - } - - // XXX(sandello): Sometimes we drop PATH from environment when spawning isolated processes. - // In this case, try to locate somewhere nearby. - { - auto execPathDirName = GetDirName(GetExecPath()); - YT_LOG_DEBUG("Looking in our exec path directory (ExecPathDir: %v)", execPathDirName); - auto probe = TString::Join(execPathDirName, "/", binary); - if (test(probe.c_str())) { - return success(probe); - } - } - - std::array<char, MAX_PATH> buffer; - - auto envPathStr = GetEnv("PATH"); - TStringBuf envPath(envPathStr); - TStringBuf envPathItem; - - YT_LOG_DEBUG("Looking for binary in PATH (Path: %v)", envPathStr); - - while (envPath.NextTok(':', envPathItem)) { - if (buffer.size() < 2 + envPathItem.size() + binary.size()) { - continue; - } - - size_t index = 0; - std::copy(envPathItem.begin(), envPathItem.end(), buffer.begin() + index); - index += envPathItem.size(); - buffer[index] = '/'; - index += 1; - std::copy(binary.begin(), binary.end(), buffer.begin() + index); - index += binary.size(); - buffer[index] = 0; - - if (test(buffer.data())) { - return success(TString(buffer.data(), index)); - } - } - - return failure(); -} - -std::vector<TString> GetEnviron() -{ - std::vector<TString> env; - size_t size = 0; - for (char** envIt = environ; *envIt; ++envIt) { - ++size; - } - env.reserve(size); - for (char** envIt = environ; *envIt; ++envIt) { - env.emplace_back(*envIt); - } - return env; -} - -bool TryKillProcessByPid(int pid, int signal) -{ -#ifdef _unix_ - YT_VERIFY(pid != -1); - int result = ::kill(pid, signal); - // Ignore ESRCH because process may have died just before TryKillProcessByPid. - if (result < 0 && errno != ESRCH) { - return false; - } - return true; -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -//////////////////////////////////////////////////////////////////////////////// - -namespace { - -#ifdef _unix_ - -bool TryWaitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) -{ - if (infop != nullptr) { - // See comment below. - infop->si_pid = 0; - } - - siginfo_t info; - ::memset(&info, 0, sizeof(info)); - auto res = HandleEintr(::waitid, idtype, id, infop != nullptr ? infop : &info, options); - - if (res == 0) { - // According to man wait. - // If WNOHANG was specified in options and there were - // no children in a waitable state, then waitid() returns 0 immediately. - // To distinguish this case from that where a child - // was in a waitable state, zero out the si_pid field - // before the call and check for a nonzero value in this field after - // the call returns. - if (infop && infop->si_pid == 0) { - return false; - } - return true; - } - - return false; -} - -void Wait4OrDie(pid_t id, int* status, int options, rusage* rusage) -{ - auto res = HandleEintr(::wait4, id, status, options, rusage); - if (res == -1) { - YT_LOG_FATAL(TError::FromSystem(), "Wait4 failed"); - } -} - -bool TrySetSignalMask(const sigset_t* sigmask, sigset_t* oldSigmask) -{ - int error = pthread_sigmask(SIG_SETMASK, sigmask, oldSigmask); - if (error != 0) { - return false; - } - return true; -} - -#if !defined(YT_USE_POSIX_SPAWN_API) - -void Cleanup(int pid) -{ - YT_VERIFY(pid > 0); - - YT_VERIFY(TryKillProcessByPid(pid, 9)); - YT_VERIFY(TryWaitid(P_PID, pid, nullptr, WEXITED)); -} - -bool TryResetSignals() -{ - for (int sig = 1; sig < NSIG; ++sig) { - // Ignore invalid signal errors. - ::signal(sig, SIG_DFL); - } - return true; -} - -#endif - -#endif - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -TProcessBase::TProcessBase(const TString& path) - : Path_(path) - , ProcessId_(InvalidProcessId) -{ } - -void TProcessBase::AddArgument(TStringBuf arg) -{ - YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); - - Args_.push_back(Capture(arg)); -} - -void TProcessBase::AddEnvVar(TStringBuf var) -{ - YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); - - Env_.push_back(Capture(var)); -} - -void TProcessBase::AddArguments(std::initializer_list<TStringBuf> args) -{ - for (auto arg : args) { - AddArgument(arg); - } -} - -void TProcessBase::AddArguments(const std::vector<TString>& args) -{ - for (const auto& arg : args) { - AddArgument(arg); - } -} - -void TProcessBase::SetWorkingDirectory(const TString& path) -{ - WorkingDirectory_ = path; -} - -void TProcessBase::CreateProcessGroup() -{ - CreateProcessGroup_ = true; -} - -//////////////////////////////////////////////////////////////////////////////// - -class TSimpleProcess::TProcessSpawnState -{ -public: - TProcessSpawnState() = default; - -#if !defined(YT_USE_POSIX_SPAWN_API) - void SetErrorPipe(TPipe* errorPipe) - { - Pipe_ = errorPipe; - } - - TError ValidateSpawnResult() - { - #ifdef _unix_ - int data[2]; - ssize_t res; - YT_VERIFY(Pipe_); - res = HandleEintr(::read, Pipe_->GetReadFD(), &data, sizeof(data)); - Pipe_->CloseReadFD(); - - if (res == 0) { - return {}; - } - - YT_VERIFY(res == sizeof(data)); - - int actionIndex = data[0]; - int errorCode = data[1]; - - YT_VERIFY(0 <= actionIndex && actionIndex < std::ssize(SpawnActions_)); - const auto& action = SpawnActions_[actionIndex]; - - return TError("%v", action.ErrorMessage) - << TError::FromSystem(errorCode); - #else - THROW_ERROR_EXPECTION("Unsupported platform"); - #endif - } -#endif - - void AddDup2FileAction(int oldFD, int newFD) - { - #if defined(YT_USE_POSIX_SPAWN_API) - SpawnFileActions_.AddDup2FileAction(oldFD, newFD); - #else - TSpawnAction action{ - std::bind(TryDup2, oldFD, newFD), - Format("Error duplicating %v file descriptor to %v in child process", oldFD, newFD) - }; - SpawnActions_.push_back(action); - #endif - } - - void AddSignalSafetySpawnActions(sigset_t* oldSignals) - { - #if defined(YT_USE_POSIX_SPAWN_API) - SpawnAttrs_.AddResetAllSignals(); - SpawnAttrs_.AddSetSigMask(oldSignals); - #else - SpawnActions_.push_back(TSpawnAction{ - TryResetSignals, - "Error resetting signals to default disposition in child process: signal failed" - }); - - SpawnActions_.push_back(TSpawnAction{ - std::bind(TrySetSignalMask, oldSignals, nullptr), - "Error unblocking signals in child process: pthread_sigmask failed" - }); - #endif - } - - void AddChdirSpawnAction(const TString& workingDirectory) - { - #if defined(YT_USE_POSIX_SPAWN_API) - SpawnFileActions_.AddChdirFileAction(workingDirectory); - #else - SpawnActions_.push_back(TSpawnAction{ - [&] { - NFs::SetCurrentWorkingDirectory(workingDirectory); - return true; - }, - "Error changing working directory" - }); - #endif - } - - void AddCreateProcessGroupAction() - { - #if defined(YT_USE_POSIX_SPAWN_API) - SpawnAttrs_.AddCreateProcessGroup(); - #else - SpawnActions_.push_back(TSpawnAction{ - [&] { - setpgrp(); - return true; - }, - "Error creating process group" - }); - #endif - } - - void AddExecvSpawnAction( - const char* resolvedPath, - char* const* argv, - char* const* env) - { - Path_ = resolvedPath; - Argv_ = argv; - Env_ = env; - #if defined(YT_USE_POSIX_SPAWN_API) - // This step is automatic for posix_spawn. - #else - SpawnActions_.push_back(TSpawnAction{ - [=] { - // Execve may fail, if called binary is being updated, e.g. during yandex-yt package update - // with errno ETXTBSY - executable file is open for writing. For example see YT-6352. - // Initiator could retry after getting error EProcessErrorCode::CannotStartProcess. - return TryExecve(resolvedPath, argv, env); - }, - "Error starting child process: execve failed" - }); - #endif - } - - pid_t SpawnChild() - { - #if defined(YT_USE_POSIX_SPAWN_API) - return DoSpawnChildPosixSpawn(); - #else - return DoSpawnChildVFork(); - #endif - } - -private: - const char* Path_; - char* const* Argv_; - char* const* Env_; - -#if defined(YT_USE_POSIX_SPAWN_API) - TPosixSpawnFileActions SpawnFileActions_; - TPosixSpawnAttrs SpawnAttrs_; -#else - struct TSpawnAction - { - std::function<bool()> Callback; - TString ErrorMessage; - }; - - std::vector<TSpawnAction> SpawnActions_; - - TPipe* Pipe_; -#endif - -#if defined(YT_USE_POSIX_SPAWN_API) - pid_t DoSpawnChildPosixSpawn() - { - pid_t pid; - - auto result = ::posix_spawn( - &pid, - Path_, - SpawnFileActions_.Get(), - SpawnAttrs_.Get(), - Argv_, - Env_); - - THROW_ERROR_EXCEPTION_IF( - result != 0, - TError("Failed to spawn process with posix_spawn") - << TError::FromSystem()); - - return pid; - } - -#else - pid_t DoSpawnChildVFork() - { - // NB: Fork() copy-on-write cause undefined behaviour when run concurrently with - // Disk IO on O_DIRECT file descriptor. vfork don't suffer from the same issue. - // NB: vfork() blocks parent until child executes new program or exits. - int pid = vfork(); - - if (pid < 0) { - THROW_ERROR_EXCEPTION("Error starting child process: vfork failed") - << TErrorAttribute("path", Path_) - << TError::FromSystem(); - } - - if (pid == 0) { - try { - // Successful call to Execve leaves - // the current context and starts execution - // of a specified binary. Child never returns - // from this call. - Child(); - } catch (...) { - YT_ABORT(); - } - } - - // Parent returns. - return pid; - } - - [[noreturn]] void Child() - { - for (int actionIndex = 0; actionIndex < std::ssize(SpawnActions_); ++actionIndex) { - auto& action = SpawnActions_[actionIndex]; - if (!action.Callback()) { - // Report error through the pipe. - int data[] = { - actionIndex, - errno - }; - - // According to pipe(7) write of small buffer is atomic. - YT_VERIFY(Pipe_); - ssize_t size = HandleEintr(::write, Pipe_->GetWriteFD(), &data, sizeof(data)); - YT_VERIFY(size == sizeof(data)); - AbortProcessSilently(EProcessExitCode::GenericError); - } - } - YT_ABORT(); - } - -#endif -}; - -//////////////////////////////////////////////////////////////////////////////// - -TSimpleProcess::TSimpleProcess(const TString& path, bool copyEnv, TDuration pollPeriod) - // TString is guaranteed to be zero-terminated. - // https://wiki.yandex-team.ru/Development/Poisk/arcadia/util/TStringAndTStringBuf#sobstvennosimvoly - : TProcessBase(path) - , PollPeriod_(pollPeriod) - , PipeFactory_(3) - , SpawnState_(std::make_unique<TProcessSpawnState>()) -{ - AddArgument(path); - - if (copyEnv) { - for (char** envIt = environ; *envIt; ++envIt) { - Env_.push_back(Capture(*envIt)); - } - } -} - -TSimpleProcess::~TSimpleProcess() -{ } - -void TSimpleProcess::AddDup2FileAction(int oldFD, int newFD) -{ - SpawnState_->AddDup2FileAction(oldFD, newFD); - MaxSpawnActionFD_ = std::max(MaxSpawnActionFD_, newFD); -} - -IConnectionReaderPtr TSimpleProcess::GetStdOutReader() -{ - auto& pipe = StdPipes_[STDOUT_FILENO]; - pipe = PipeFactory_.Create(); - AddDup2FileAction(pipe.GetWriteFD(), STDOUT_FILENO); - return pipe.CreateAsyncReader(); -} - -IConnectionReaderPtr TSimpleProcess::GetStdErrReader() -{ - auto& pipe = StdPipes_[STDERR_FILENO]; - pipe = PipeFactory_.Create(); - AddDup2FileAction(pipe.GetWriteFD(), STDERR_FILENO); - return pipe.CreateAsyncReader(); -} - -IConnectionWriterPtr TSimpleProcess::GetStdInWriter() -{ - auto& pipe = StdPipes_[STDIN_FILENO]; - pipe = PipeFactory_.Create(); - AddDup2FileAction(pipe.GetReadFD(), STDIN_FILENO); - return pipe.CreateAsyncWriter(); -} - -TFuture<void> TProcessBase::Spawn() -{ - auto finally = Finally([&] { - CleanUpParent(); - }); - - try { - // Resolve binary path. - std::vector<TError> innerErrors; - for (int retryIndex = ResolveRetryCount; retryIndex >= 0; --retryIndex) { - auto errorOrPath = ResolveBinaryPath(Path_); - if (errorOrPath.IsOK()) { - ResolvedPath_ = errorOrPath.Value(); - break; - } - - innerErrors.push_back(errorOrPath); - - if (retryIndex == 0) { - THROW_ERROR_EXCEPTION("Failed to resolve binary path %v", Path_) - << innerErrors; - } - - TDelayedExecutor::WaitForDuration(ResolveRetryTimeout); - } - - DoSpawn(); - } catch (const std::exception& ex) { - FinishedPromise_.TrySet(TError(EProcessErrorCode::CannotStartProcess, - "Cannot spawn child process %v", - ResolvedPath_) - << ex); - } - return FinishedPromise_; -} - -void TSimpleProcess::DoSpawn() -{ -#ifdef _unix_ - YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); - - // Make sure no spawn action closes Pipe_.WriteFD - PrepareErrorPipe(); - - Env_.push_back(nullptr); - Args_.push_back(nullptr); - - // Block all signals around vfork; see http://ewontfix.com/7/ - - // As the child may run in the same address space as the parent until - // the actual execve() system call, any (custom) signal handlers that - // the parent has might alter parent's memory if invoked in the child, - // with undefined results. So we block all signals in the parent before - // vfork(), which will cause them to be blocked in the child as well (we - // rely on the fact that Linux, just like all sane implementations, only - // clones the calling thread). Then, in the child, we reset all signals - // to their default dispositions (while still blocked), and unblock them - // (so the exec()ed process inherits the parent's signal mask) - - sigset_t allBlocked; - sigfillset(&allBlocked); - sigset_t oldSignals; - - if (!TrySetSignalMask(&allBlocked, &oldSignals)) { - THROW_ERROR_EXCEPTION("Failed to block all signals") - << TError::FromSystem(); - } - - PrepareSpawnActions(&oldSignals); - - SpawnChild(); - - // This should not fail ever. - YT_VERIFY(TrySetSignalMask(&oldSignals, nullptr)); - - CloseErrorPipe(); - - ValidateSpawnResult(); - - AsyncWaitExecutor_ = New<TPeriodicExecutor>( - GetSyncInvoker(), - BIND(&TSimpleProcess::AsyncPeriodicTryWait, MakeStrong(this)), - PollPeriod_); - - AsyncWaitExecutor_->Start(); - - YT_LOG_INFO("Process spawned (Pid: %v)", ProcessId_); -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -void TSimpleProcess::CleanUpParent() -{ - StdPipes_[STDIN_FILENO].CloseReadFD(); - StdPipes_[STDOUT_FILENO].CloseWriteFD(); - StdPipes_[STDERR_FILENO].CloseWriteFD(); - PipeFactory_.Clear(); -} - -void TSimpleProcess::PrepareErrorPipe() -{ -#if defined(YT_USE_POSIX_SPAWN_API) - YT_LOG_DEBUG("Spawning new process (Path: %v, Arguments: %v, Environment: %v)", - ResolvedPath_, - Args_, - Env_); -#else - TPipeFactory pipeFactory(MaxSpawnActionFD_ + 1); - Pipe_ = pipeFactory.Create(); - SpawnState_->SetErrorPipe(&Pipe_); - - pipeFactory.Clear(); - - YT_LOG_DEBUG("Spawning new process (Path: %v, ErrorPipe: %v, Arguments: %v, Environment: %v)", - ResolvedPath_, - Pipe_, - Args_, - Env_); -#endif -} - -void TSimpleProcess::CloseErrorPipe() -{ -#if !defined(YT_USE_POSIX_SPAWN_API) - Pipe_.CloseWriteFD(); -#endif -} - -void TSimpleProcess::ValidateSpawnResult() -{ -#if defined (YT_USE_POSIX_SPAWN_API) - - YT_LOG_DEBUG("Child process spawned successfully (Pid: %v)", ProcessId_); - return; -#else - auto result = SpawnState_->ValidateSpawnResult(); - - if (result.IsOK()) { - // Child successfully spawned or was killed by a signal. - // But there is no way to distinguish between these two cases: - // * child killed by signal before exec - // * child killed by signal after exec - // So we treat kill-before-exec the same way as kill-after-exec. - YT_LOG_DEBUG("Child process spawned successfully (Pid: %v)", ProcessId_); - return; - } - - Finished_ = true; - - Cleanup(ProcessId_); - ProcessId_ = InvalidProcessId; - - THROW_ERROR_EXCEPTION(std::move(result)); -#endif -} - -void TSimpleProcess::PrepareSpawnActions(sigset_t* oldSignals) -{ - SpawnState_->AddSignalSafetySpawnActions(oldSignals); - - if (!WorkingDirectory_.empty()) { - SpawnState_->AddChdirSpawnAction(WorkingDirectory_); - } - - if (CreateProcessGroup_) { - SpawnState_->AddCreateProcessGroupAction(); - } - - SpawnState_->AddExecvSpawnAction( - ResolvedPath_.c_str(), - const_cast<char* const*>(Args_.data()), - const_cast<char* const*>(Env_.data())); -} - -void TSimpleProcess::SpawnChild() -{ -#ifdef _unix_ - ProcessId_ = SpawnState_->SpawnChild(); - Started_ = true; -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -void TSimpleProcess::AsyncPeriodicTryWait() -{ -#ifdef _unix_ - siginfo_t processInfo; - memset(&processInfo, 0, sizeof(siginfo_t)); - - // Note WNOWAIT flag. - // This call just waits for a process to be finished but does not clear zombie flag. - - if (!TryWaitid(P_PID, ProcessId_, &processInfo, WEXITED | WNOWAIT | WNOHANG) || - processInfo.si_pid != ProcessId_) - { - return; - } - - YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop()); - AsyncWaitExecutor_ = nullptr; - - // This call just should return immediately - // because we have already waited for this process with WNOHANG - rusage rusage; - Wait4OrDie(ProcessId_, nullptr, WNOHANG, &rusage); - - Finished_ = true; - auto error = ProcessInfoToError(processInfo); - YT_LOG_DEBUG(error, "Process finished (Pid: %v, MajorFaults: %v)", - ProcessId_, - rusage.ru_majflt); - - FinishedPromise_.Set(error); -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -void TSimpleProcess::Kill(int signal) -{ -#ifdef _unix_ - if (!Started_) { - THROW_ERROR_EXCEPTION("Process is not started yet"); - } - - if (Finished_) { - return; - } - - YT_LOG_DEBUG("Killing child process (Pid: %v)", ProcessId_); - - bool result = false; - if (!CreateProcessGroup_) { - result = TryKillProcessByPid(ProcessId_, signal); - } else { - result = TryKillProcessByPid(-1 * ProcessId_, signal); - } - - if (!result) { - THROW_ERROR_EXCEPTION("Failed to kill child process %v", ProcessId_) - << TError::FromSystem(); - } - return; -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -TString TProcessBase::GetPath() const -{ - return Path_; -} - -int TProcessBase::GetProcessId() const -{ - return ProcessId_; -} - -bool TProcessBase::IsStarted() const -{ - return Started_; -} - -bool TProcessBase::IsFinished() const -{ - return Finished_; -} - -TString TProcessBase::GetCommandLine() const -{ - TStringBuilder builder; - builder.AppendString(Path_); - - bool first = true; - for (const auto& arg_ : Args_) { - TStringBuf arg(arg_); - if (first) { - first = false; - } else { - if (arg) { - builder.AppendChar(' '); - bool needQuote = false; - for (size_t i = 0; i < arg.length(); ++i) { - if (!IsAsciiAlnum(arg[i]) && - arg[i] != '-' && arg[i] != '_' && arg[i] != '=' && arg[i] != '/') - { - needQuote = true; - break; - } - } - if (needQuote) { - builder.AppendChar('"'); - TStringBuf left, right; - while (arg.TrySplit('"', left, right)) { - builder.AppendString(left); - builder.AppendString("\\\""); - arg = right; - } - builder.AppendString(arg); - builder.AppendChar('"'); - } else { - builder.AppendString(arg); - } - } - } - } - - return builder.Flush(); -} - -const char* TProcessBase::Capture(TStringBuf arg) -{ - StringHolders_.push_back(TString(arg)); - return StringHolders_.back().c_str(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/process/process.h b/yt/yt/library/process/process.h deleted file mode 100644 index b88defec459..00000000000 --- a/yt/yt/library/process/process.h +++ /dev/null @@ -1,139 +0,0 @@ -#pragma once - -#include "pipe.h" - -#include <yt/yt/core/misc/error.h> - -#include <yt/yt/core/actions/future.h> - -#include <yt/yt/core/concurrency/public.h> - -#include <atomic> -#include <vector> -#include <array> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TErrorOr<TString> ResolveBinaryPath(const TString& binary); -std::vector<TString> GetEnviron(); -bool TryKillProcessByPid(int pid, int signal); - -//////////////////////////////////////////////////////////////////////////////// - -class TProcessBase - : public TRefCounted -{ -public: - explicit TProcessBase(const TString& path); - - void AddArgument(TStringBuf arg); - void AddEnvVar(TStringBuf var); - - void AddArguments(std::initializer_list<TStringBuf> args); - void AddArguments(const std::vector<TString>& args); - - void SetWorkingDirectory(const TString& path); - void CreateProcessGroup(); - - virtual NNet::IConnectionWriterPtr GetStdInWriter() = 0; - virtual NNet::IConnectionReaderPtr GetStdOutReader() = 0; - virtual NNet::IConnectionReaderPtr GetStdErrReader() = 0; - - //! Returns process completion future, which ends with EErrorCode::OK or EProcessErrorCode. - TFuture<void> Spawn(); - - virtual void Kill(int signal) = 0; - - TString GetPath() const; - int GetProcessId() const; - bool IsStarted() const; - bool IsFinished() const; - - TString GetCommandLine() const; - -protected: - const TString Path_; - - int ProcessId_; - std::atomic<bool> Started_ = {false}; - std::atomic<bool> Finished_ = {false}; - int MaxSpawnActionFD_ = - 1; - NPipes::TPipe Pipe_; - // Container for owning string data. Use std::deque because it never moves contained objects. - std::deque<std::string> StringHolders_; - std::vector<const char*> Args_; - std::vector<const char*> Env_; - TString ResolvedPath_; - TString WorkingDirectory_; - bool CreateProcessGroup_ = false; - TPromise<void> FinishedPromise_ = NewPromise<void>(); - - virtual void DoSpawn() = 0; - const char* Capture(TStringBuf arg); - - virtual void CleanUpParent() = 0; - -private: - void SpawnChild(); - void ValidateSpawnResult(); - void Child(); - void AsyncPeriodicTryWait(); -}; - -DEFINE_REFCOUNTED_TYPE(TProcessBase) - -//////////////////////////////////////////////////////////////////////////////// - -// Read this -// http://ewontfix.com/7/ -// before making any changes. -class TSimpleProcess - : public TProcessBase -{ -public: - explicit TSimpleProcess( - const TString& path, - bool copyEnv = true, - TDuration pollPeriod = TDuration::MilliSeconds(100)); - // We move dtor in .cpp file to avoid - // instantiation of ~std::unique_ptr of a forward - // declared class. - ~TSimpleProcess(); - - void Kill(int signal) override; - NNet::IConnectionWriterPtr GetStdInWriter() override; - NNet::IConnectionReaderPtr GetStdOutReader() override; - NNet::IConnectionReaderPtr GetStdErrReader() override; - -private: - const TDuration PollPeriod_; - - NPipes::TPipeFactory PipeFactory_; - std::array<NPipes::TPipe, 3> StdPipes_; - - NConcurrency::TPeriodicExecutorPtr AsyncWaitExecutor_; - - class TProcessSpawnState; - std::unique_ptr<TProcessSpawnState> SpawnState_; - - void AddDup2FileAction(int oldFD, int newFD); - - void PrepareErrorPipe(); - void CloseErrorPipe(); - - void PrepareSpawnActions(sigset_t* oldSignals); - - void DoSpawn() override; - void SpawnChild(); - void AsyncPeriodicTryWait(); - - void ValidateSpawnResult(); - - void CleanUpParent() override; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/process/pty.cpp b/yt/yt/library/process/pty.cpp deleted file mode 100644 index fc972d38ea5..00000000000 --- a/yt/yt/library/process/pty.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "pty.h" - -#include "io_dispatcher.h" - -#include <yt/yt/core/misc/common.h> -#include <yt/yt/core/misc/proc.h> - -#include <yt/yt/core/net/connection.h> - -namespace NYT::NPipes { - -using namespace NNet; - -//////////////////////////////////////////////////////////////////////////////// - -TPty::TPty(int height, int width) -{ - SafeOpenPty(&MasterFD_, &SlaveFD_, height, width); -} - -TPty::~TPty() -{ - if (MasterFD_ != InvalidFD) { - YT_VERIFY(TryClose(MasterFD_, false)); - } - - if (SlaveFD_ != InvalidFD) { - YT_VERIFY(TryClose(SlaveFD_, false)); - } -} - -IConnectionWriterPtr TPty::CreateMasterAsyncWriter() -{ - YT_VERIFY(MasterFD_ != InvalidFD); - int fd = SafeDup(MasterFD_); - SafeSetCloexec(fd); - SafeMakeNonblocking(fd); - return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller()); -} - -IConnectionReaderPtr TPty::CreateMasterAsyncReader() -{ - YT_VERIFY(MasterFD_ != InvalidFD); - int fd = SafeDup(MasterFD_); - SafeSetCloexec(fd); - SafeMakeNonblocking(fd); - return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller()); -} - -int TPty::GetMasterFD() const -{ - YT_VERIFY(MasterFD_ != InvalidFD); - return MasterFD_; -} - -int TPty::GetSlaveFD() const -{ - YT_VERIFY(SlaveFD_ != InvalidFD); - return SlaveFD_; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pty.h b/yt/yt/library/process/pty.h deleted file mode 100644 index b585782d12b..00000000000 --- a/yt/yt/library/process/pty.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/net/public.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -class TPty - : public TNonCopyable -{ -public: - static const int InvalidFD = -1; - - TPty(int height, int width); - ~TPty(); - - NNet::IConnectionReaderPtr CreateMasterAsyncReader(); - NNet::IConnectionWriterPtr CreateMasterAsyncWriter(); - - int GetMasterFD() const; - int GetSlaveFD() const; - -private: - int MasterFD_ = InvalidFD; - int SlaveFD_ = InvalidFD; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/public.h b/yt/yt/library/process/public.h deleted file mode 100644 index 76cfff13404..00000000000 --- a/yt/yt/library/process/public.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/configurable_singleton_decl.h> - -#include <library/cpp/yt/memory/ref_counted.h> - -namespace NYT::NPipes { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_CLASS(TNamedPipe) -DECLARE_REFCOUNTED_CLASS(TNamedPipeConfig) - -DECLARE_REFCOUNTED_CLASS(TIODispatcherConfig) -DECLARE_REFCOUNTED_CLASS(TIODispatcherDynamicConfig) - - -YT_DECLARE_RECONFIGURABLE_SINGLETON(TIODispatcherConfig, TIODispatcherDynamicConfig); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NPipes diff --git a/yt/yt/library/process/subprocess.cpp b/yt/yt/library/process/subprocess.cpp deleted file mode 100644 index c1116a8c904..00000000000 --- a/yt/yt/library/process/subprocess.cpp +++ /dev/null @@ -1,177 +0,0 @@ -#include "subprocess.h" - -#include <yt/yt/core/misc/blob.h> -#include <yt/yt/core/misc/proc.h> -#include <yt/yt/core/misc/finally.h> - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/net/connection.h> - -#include <util/system/execpath.h> - -#include <array> - -namespace NYT { - -using namespace NConcurrency; -using namespace NPipes; - -//////////////////////////////////////////////////////////////////////////////// - -const static size_t PipeBlockSize = 64 * 1024; - -static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Subprocess"); - -//////////////////////////////////////////////////////////////////////////////// - -TSubprocess::TSubprocess(TString path, bool copyEnv) - : Path_(std::move(path)) - , Process_(New<TSimpleProcess>(Path_, copyEnv)) -{ } - -TSubprocess TSubprocess::CreateCurrentProcessSpawner() -{ - return TSubprocess(GetExecPath()); -} - -void TSubprocess::AddArgument(TStringBuf arg) -{ - Process_->AddArgument(arg); -} - -void TSubprocess::AddArguments(std::initializer_list<TStringBuf> args) -{ - Process_->AddArguments(args); -} - -TSubprocessResult TSubprocess::Execute(const TSharedRef& input, TDuration timeout) -{ -#ifdef _unix_ - auto killCookie = TDelayedExecutor::Submit( - BIND([=, path = Path_, process = GetProcess()] { - YT_LOG_WARNING("Killing process due to timeout (Path: %v, ProcessId: %v, Timeout: %v)", - path, - process->GetProcessId(), - timeout); - - try { - process->Kill(SIGKILL); - } catch (const std::exception& ex) { - YT_LOG_ERROR(ex, "Failed to kill process (Path: %v, ProcessId: %v)", - path, - process->GetProcessId()); - } - }), - timeout); - - auto cookieGuard = Finally([&] { - TDelayedExecutor::Cancel(killCookie); - }); - - auto inputStream = Process_->GetStdInWriter(); - auto outputStream = Process_->GetStdOutReader(); - auto errorStream = Process_->GetStdErrReader(); - auto finished = Process_->Spawn(); - - auto readIntoBlob = [] (IAsyncInputStreamPtr stream) { - TBlob output; - auto buffer = TSharedMutableRef::Allocate(PipeBlockSize, {.InitializeStorage = false}); - while (true) { - auto size = WaitFor(stream->Read(buffer)) - .ValueOrThrow(); - - if (size == 0) { - break; - } - - // ToDo(psushin): eliminate copying. - output.Append(buffer.Begin(), size); - } - return TSharedRef::FromBlob(std::move(output)); - }; - - auto writeStdin = BIND([=] { - if (input.Size() > 0) { - WaitFor(inputStream->Write(input)) - .ThrowOnError(); - } - - WaitFor(inputStream->Close()) - .ThrowOnError(); - - //! Return dummy ref, so later we cat put Future into vector - //! along with stdout and stderr. - return TSharedRef::MakeEmpty(); - }); - - std::vector<TFuture<TSharedRef>> futures = { - BIND(readIntoBlob, outputStream).AsyncVia(GetCurrentInvoker()).Run(), - BIND(readIntoBlob, errorStream).AsyncVia(GetCurrentInvoker()).Run(), - writeStdin.AsyncVia(GetCurrentInvoker()).Run(), - }; - - try { - auto outputsOrError = WaitFor(AllSucceeded(futures)); - THROW_ERROR_EXCEPTION_IF_FAILED( - outputsOrError, - "IO error occurred during subprocess call"); - - const auto& outputs = outputsOrError.Value(); - YT_VERIFY(outputs.size() == 3); - - // This can block indefinitely. - auto exitCode = WaitFor(finished); - return TSubprocessResult{outputs[0], outputs[1], exitCode}; - } catch (...) { - try { - Process_->Kill(SIGKILL); - } catch (...) { } - Y_UNUSED(WaitFor(finished)); - throw; - } -#else - THROW_ERROR_EXCEPTION("Unsupported platform"); -#endif -} - -void TSubprocess::Kill(int signal) -{ - Process_->Kill(signal); -} - -TString TSubprocess::GetCommandLine() const -{ - return Process_->GetCommandLine(); -} - -TProcessBasePtr TSubprocess::GetProcess() const -{ - return Process_; -} - -//////////////////////////////////////////////////////////////////////////////// - -void RunSubprocess(const std::vector<TString>& cmd) -{ - if (cmd.empty()) { - THROW_ERROR_EXCEPTION("Command can't be empty"); - } - - auto process = TSubprocess(cmd[0]); - for (int index = 1; index < std::ssize(cmd); ++index) { - process.AddArgument(cmd[index]); - } - - auto result = process.Execute(); - if (!result.Status.IsOK()) { - THROW_ERROR_EXCEPTION("Failed to run %v", cmd[0]) - << result.Status - << TErrorAttribute("command_line", process.GetCommandLine()) - << TErrorAttribute("error", TString(result.Error.Begin(), result.Error.End())); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/process/subprocess.h b/yt/yt/library/process/subprocess.h deleted file mode 100644 index 68dab8896c5..00000000000 --- a/yt/yt/library/process/subprocess.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include "public.h" -#include "process.h" - -#include <library/cpp/yt/memory/ref.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -struct TSubprocessResult -{ - TSharedRef Output; - TSharedRef Error; - TError Status; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TSubprocess -{ -public: - explicit TSubprocess(TString path, bool copyEnv = true); - - static TSubprocess CreateCurrentProcessSpawner(); - - void AddArgument(TStringBuf arg); - void AddArguments(std::initializer_list<TStringBuf> args); - - TSubprocessResult Execute( - const TSharedRef& input = TSharedRef::MakeEmpty(), - TDuration timeout = TDuration::Max()); - - void Kill(int signal); - - TString GetCommandLine() const; - - TProcessBasePtr GetProcess() const; - -private: - const TString Path_; - - const TProcessBasePtr Process_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -void RunSubprocess(const std::vector<TString>& cmd); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/process/ya.make b/yt/yt/library/process/ya.make deleted file mode 100644 index 6b3ea41ca23..00000000000 --- a/yt/yt/library/process/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - config.cpp - GLOBAL configure_io_dispatcher.cpp - io_dispatcher.cpp - pipe.cpp - process.cpp - pty.cpp - subprocess.cpp -) - -PEERDIR( - yt/yt/core - contrib/libs/re2 -) - -END() - -RECURSE_FOR_TESTS( - unittests -) diff --git a/yt/yt/library/program/build_attributes.cpp b/yt/yt/library/program/build_attributes.cpp deleted file mode 100644 index e2024c255b7..00000000000 --- a/yt/yt/library/program/build_attributes.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "build_attributes.h" - -#include <yt/yt/build/build.h> - -#include <yt/yt/core/ytree/fluent.h> -#include <yt/yt/core/ytree/ypath_client.h> - -#include <yt/yt/core/misc/error_code.h> - -namespace NYT { - -using namespace NYTree; -using namespace NYson; - -//////////////////////////////////////////////////////////////////////////////// - -static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Build"); - -//////////////////////////////////////////////////////////////////////////////// - -void TBuildInfo::Register(TRegistrar registrar) -{ - registrar.Parameter("name", &TThis::Name) - .Default(); - - registrar.Parameter("version", &TThis::Version) - .Default(GetVersion()); - - registrar.Parameter("build_host", &TThis::BuildHost) - .Default(GetBuildHost()); - - registrar.Parameter("build_time", &TThis::BuildTime) - .Default(ParseBuildTime()); - - registrar.Parameter("start_time", &TThis::StartTime) - .Default(TInstant::Now()); -} - -std::optional<TInstant> TBuildInfo::ParseBuildTime() -{ - TString rawBuildTime(GetBuildTime()); - - // Build time may be empty if code is building - // without -DBUILD_DATE (for example, in opensource build). - if (rawBuildTime.empty()) { - return std::nullopt; - } - - try { - return TInstant::ParseIso8601(rawBuildTime); - } catch (const std::exception& ex) { - YT_LOG_ERROR(ex, "Error parsing build time"); - return std::nullopt; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -TBuildInfoPtr BuildBuildAttributes(const char* serviceName) -{ - auto info = New<TBuildInfo>(); - if (serviceName) { - info->Name = serviceName; - } - return info; -} - -void SetBuildAttributes(IYPathServicePtr orchidRoot, const char* serviceName) -{ - SyncYPathSet( - orchidRoot, - "/service", - BuildYsonStringFluently() - .BeginAttributes() - .Item("opaque").Value(true) - .EndAttributes() - .Value(BuildBuildAttributes(serviceName))); - SyncYPathSet( - orchidRoot, - "/error_codes", - BuildYsonStringFluently() - .BeginAttributes() - .Item("opaque").Value(true) - .EndAttributes() - .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodes(), [] (TFluentMap fluent, const auto& pair) { - fluent - .Item(ToString(pair.first)).BeginMap() - .Item("cpp_literal").Value(ToString(pair.second)) - .EndMap(); - })); - SyncYPathSet( - orchidRoot, - "/error_code_ranges", - BuildYsonStringFluently() - .BeginAttributes() - .Item("opaque").Value(true) - .EndAttributes() - .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodeRanges(), [] (TFluentMap fluent, const TErrorCodeRegistry::TErrorCodeRangeInfo& range) { - fluent - .Item(ToString(range)).BeginMap() - .Item("cpp_enum").Value(range.Namespace) - .EndMap(); - })); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT - diff --git a/yt/yt/library/program/build_attributes.h b/yt/yt/library/program/build_attributes.h deleted file mode 100644 index e02f86b351b..00000000000 --- a/yt/yt/library/program/build_attributes.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/public.h> -#include <yt/yt/core/ytree/yson_struct.h> - -#include <yt/yt/core/yson/public.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TBuildInfo - : public NYTree::TYsonStruct -{ -public: - std::optional<TString> Name; - TString Version; - TString BuildHost; - std::optional<TInstant> BuildTime; - TInstant StartTime; - - REGISTER_YSON_STRUCT(TBuildInfo); - - static void Register(TRegistrar registrar); - -private: - static std::optional<TInstant> ParseBuildTime(); -}; - -DEFINE_REFCOUNTED_TYPE(TBuildInfo) - -//////////////////////////////////////////////////////////////////////////////// - -//! Build build (pun intended) attributes as a TBuildInfo a-la /orchid/service. If service name is not provided, -//! it is omitted from the result. -TBuildInfoPtr BuildBuildAttributes(const char* serviceName = nullptr); - -void SetBuildAttributes(NYTree::IYPathServicePtr orchidRoot, const char* serviceName); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp deleted file mode 100644 index 762feea0f25..00000000000 --- a/yt/yt/library/program/config.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "config.h" - -#include <yt/yt/core/concurrency/fiber_scheduler_thread.h> - -namespace NYT { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void THeapProfilerConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("sampling_rate", &TThis::SamplingRate) - .Default(); - registrar.Parameter("snapshot_update_period", &TThis::SnapshotUpdatePeriod) - .Default(TDuration::Seconds(5)); -} - -//////////////////////////////////////////////////////////////////////////////// - -void WarnForUnrecognizedOptionsImpl( - const NLogging::TLogger& logger, - const IMapNodePtr& unrecognized) -{ - const auto& Logger = logger; - if (unrecognized && unrecognized->GetChildCount() > 0) { - YT_LOG_WARNING("Bootstrap config contains unrecognized options (Unrecognized: %v)", - ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text)); - } -} - -void WarnForUnrecognizedOptions( - const NLogging::TLogger& logger, - const NYTree::TYsonStructPtr& config) -{ - WarnForUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized()); -} - -void AbortOnUnrecognizedOptionsImpl( - const NLogging::TLogger& logger, - const IMapNodePtr& unrecognized) -{ - const auto& Logger = logger; - if (unrecognized && unrecognized->GetChildCount() > 0) { - YT_LOG_ERROR("Bootstrap config contains unrecognized options, terminating (Unrecognized: %v)", - ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text)); - YT_ABORT(); - } -} - -void AbortOnUnrecognizedOptions( - const NLogging::TLogger& logger, - const NYTree::TYsonStructPtr& config) -{ - AbortOnUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized()); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT - diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h deleted file mode 100644 index 38f6eb65ee6..00000000000 --- a/yt/yt/library/program/config.h +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/yson_struct.h> - -#include <yt/yt/core/misc/configurable_singleton_def.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class THeapProfilerConfig - : public NYTree::TYsonStruct -{ -public: - // Sampling rate for tcmalloc in bytes. - // See https://github.com/google/tcmalloc/blob/master/docs/sampling.md - std::optional<i64> SamplingRate; - - // Period of update snapshot in heap profiler. - std::optional<TDuration> SnapshotUpdatePeriod; - - REGISTER_YSON_STRUCT(THeapProfilerConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(THeapProfilerConfig) - -//////////////////////////////////////////////////////////////////////////////// - -// NB: These functions should not be called from bootstrap -// config validator since logger is not set up yet. -void WarnForUnrecognizedOptions( - const NLogging::TLogger& logger, - const NYTree::TYsonStructPtr& config); - -void AbortOnUnrecognizedOptions( - const NLogging::TLogger& logger, - const NYTree::TYsonStructPtr& config); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp deleted file mode 100644 index d53aa477d0d..00000000000 --- a/yt/yt/library/program/helpers.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "helpers.h" -#include "config.h" - -#include <yt/yt/core/misc/ref_counted_tracker_profiler.h> - -#include <yt/yt/core/logging/log_manager.h> - -#include <yt/yt/core/net/address.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -void ConfigureSingletons(const TSingletonsConfigPtr& config) -{ - TSingletonManager::Configure(config); - - // TODO(babenko): move to server program base - NLogging::TLogManager::Get()->EnableReopenOnSighup(); - - // By default, server components must have a reasonable FQDN. - // Failure to do so may result in issues like YT-4561. - // TODO(babenko): move to server program base - NNet::TAddressResolver::Get()->EnsureLocalHostName(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/helpers.h b/yt/yt/library/program/helpers.h deleted file mode 100644 index 3f0ea32b334..00000000000 --- a/yt/yt/library/program/helpers.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "public.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -void ConfigureSingletons(const TSingletonsConfigPtr& config); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program-inl.h b/yt/yt/library/program/program-inl.h deleted file mode 100644 index baf71791d0a..00000000000 --- a/yt/yt/library/program/program-inl.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef PROGRAM_INL_H_ -#error "Direct inclusion of this file is not allowed, include program.h" -// For the sake of sane code completion. -#include "program.h" -#endif - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -T FromStringArgMapper(TStringBuf arg) -{ - T result; - if (!T::FromString(arg, &result)) { - throw TProgramException(Format("Error parsing %Qv", arg)); - } - return result; -} - -template <class T> -T ParseEnumArgMapper(TStringBuf arg) -{ - return ParseEnum<T>(arg); -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class E> - requires std::is_enum_v<E> -void TProgram::Abort(E exitCode) noexcept -{ - Abort(ToUnderlying(exitCode)); -} - -template <class E> - requires std::is_enum_v<E> -[[noreturn]] -void TProgram::Exit(E exitCode) noexcept -{ - Exit(ToUnderlying(exitCode)); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program.cpp b/yt/yt/library/program/program.cpp deleted file mode 100644 index 6b2e012afe1..00000000000 --- a/yt/yt/library/program/program.cpp +++ /dev/null @@ -1,318 +0,0 @@ -#include "program.h" - -#include "build_attributes.h" - -#include <yt/yt/build/build.h> - -#include <yt/yt/core/misc/crash_handler.h> -#include <yt/yt/core/misc/signal_registry.h> -#include <yt/yt/core/misc/fs.h> -#include <yt/yt/core/misc/shutdown.h> - -#include <yt/yt/core/yson/writer.h> -#include <yt/yt/core/yson/null_consumer.h> - -#include <yt/yt/core/logging/log_manager.h> - -#include <yt/yt/library/ytprof/heap_profiler.h> - -#include <yt/yt/library/profiling/tcmalloc/profiler.h> - -#include <library/cpp/yt/system/exit.h> - -#include <library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h> - -#include <tcmalloc/malloc_extension.h> - -#include <absl/debugging/stacktrace.h> - -#include <util/system/thread.h> -#include <util/system/sigset.h> - -#include <stdlib.h> - -#ifdef _unix_ -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#endif - -#ifdef _linux_ -#include <grp.h> -#include <sys/prctl.h> -#endif - -namespace NYT { - -using namespace NYson; - -//////////////////////////////////////////////////////////////////////////////// - -class TProgram::TOptsParseResult - : public NLastGetopt::TOptsParseResult -{ -public: - TOptsParseResult(TProgram* owner, int argc, const char** argv) - : Owner_(owner) - { - Init(&Owner_->Opts_, argc, argv); - } - - void HandleError() const override - { - Owner_->OnError(CurrentExceptionMessage()); - Cerr << Endl << "Try running '" << Owner_->Argv0_ << " --help' for more information." << Endl; - Owner_->Exit(ToUnderlying(EProcessExitCode::ArgumentsError)); - } - -private: - TProgram* const Owner_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -TProgram::TProgram() -{ - Opts_.AddHelpOption(); - Opts_.AddLongOption("yt-version", "Prints YT version") - .NoArgument() - .StoreValue(&PrintYTVersion_, true); - Opts_.AddLongOption("version", "Print version") - .NoArgument() - .StoreValue(&PrintVersion_, true); - Opts_.AddLongOption("yson", "Prints build information in YSON") - .NoArgument() - .StoreValue(&UseYson_, true); - Opts_.AddLongOption("build", "Prints build information") - .NoArgument() - .StoreValue(&PrintBuild_, true); - Opts_.SetFreeArgsNum(0); -} - -TProgram::~TProgram() = default; - -void TProgram::SetCrashOnError() -{ - CrashOnError_ = true; -} - -void TProgram::HandleVersionAndBuild() -{ - if (PrintVersion_) { - PrintVersionAndExit(); - } - if (PrintYTVersion_) { - PrintYTVersionAndExit(); - } - if (PrintBuild_) { - PrintBuildAndExit(); - } -} - -int TProgram::Run(int argc, const char** argv) -{ - ::srand(time(nullptr)); - - Argv0_ = TString(argv[0]); - OptsParseResult_ = std::make_unique<TOptsParseResult>(this, argc, argv); - - auto run = [&] { - HandleVersionAndBuild(); - DoRun(); - }; - - if (!CrashOnError_) { - try { - run(); - Exit(ToUnderlying(EProcessExitCode::OK)); - } catch (...) { - OnError(CurrentExceptionMessage()); - Exit(ToUnderlying(EProcessExitCode::GenericError)); - } - } else { - run(); - Exit(ToUnderlying(EProcessExitCode::OK)); - } - - // Cannot reach this due to #Exit calls above. - YT_ABORT(); -} - -void TProgram::Abort(int code) noexcept -{ - NLogging::TLogManager::Get()->Shutdown(); - AbortProcessSilently(code); -} - -void TProgram::Exit(int code) noexcept -{ -#if defined(_linux_) && defined(CLANG_COVERAGE) - __llvm_profile_write_file(); -#endif - - // This explicit call may become obsolete some day; - // cf. the comment section for NYT::Shutdown. - Shutdown({ - .AbortOnHang = ShouldAbortOnHungShutdown(), - .HungExitCode = code, - }); - - ::exit(code); -} - -bool TProgram::ShouldAbortOnHungShutdown() noexcept -{ - return true; -} - -void TProgram::OnError(const TString& message) noexcept -{ - try { - Cerr << message << Endl; - } catch (...) { - // Just ignore it; STDERR might be closed already, - // and write() would result in EPIPE. - } -} - -void TProgram::PrintYTVersionAndExit() -{ - if (UseYson_) { - THROW_ERROR_EXCEPTION("--yson is not supported when printing version"); - } - Cout << GetVersion() << Endl; - Exit(0); -} - -void TProgram::PrintBuildAndExit() -{ - if (UseYson_) { - TYsonWriter writer(&Cout, EYsonFormat::Pretty); - Serialize(BuildBuildAttributes(), &writer); - Cout << Endl; - } else { - Cout << "Build Time: " << GetBuildTime() << Endl; - Cout << "Build Host: " << GetBuildHost() << Endl; - } - Exit(0); -} - -void TProgram::PrintVersionAndExit() -{ - PrintYTVersionAndExit(); -} - -const NLastGetopt::TOptsParseResult& TProgram::GetOptsParseResult() const -{ - return *OptsParseResult_; -} - -//////////////////////////////////////////////////////////////////////////////// - -TProgramException::TProgramException(TString what) - : What_(std::move(what)) -{ } - -const char* TProgramException::what() const noexcept -{ - return What_.c_str(); -} - -//////////////////////////////////////////////////////////////////////////////// - -TString CheckPathExistsArgMapper(const TString& arg) -{ - if (!NFS::Exists(arg)) { - throw TProgramException(Format("File %v does not exist", arg)); - } - return arg; -} - -NYson::TYsonString CheckYsonArgMapper(const TString& arg) -{ - ParseYsonStringBuffer(arg, EYsonType::Node, GetNullYsonConsumer()); - return NYson::TYsonString(arg); -} - -void ConfigureUids() -{ -#ifdef _unix_ - uid_t ruid, euid; -#ifdef _linux_ - uid_t suid; - YT_VERIFY(getresuid(&ruid, &euid, &suid) == 0); -#else - ruid = getuid(); - euid = geteuid(); -#endif - if (euid == 0) { - // if real uid is already root do not set root as supplementary ids. - if (ruid != 0) { - YT_VERIFY(setgroups(0, nullptr) == 0); - } - // if effective uid == 0 (e. g. set-uid-root), alter saved = effective, effective = real. -#ifdef _linux_ - YT_VERIFY(setresuid(ruid, ruid, euid) == 0); - // Make server suid_dumpable = 1. - YT_VERIFY(prctl(PR_SET_DUMPABLE, 1) == 0); -#else - YT_VERIFY(setuid(euid) == 0); - YT_VERIFY(seteuid(ruid) == 0); - YT_VERIFY(setruid(ruid) == 0); -#endif - } - umask(0000); -#endif -} - -void ConfigureIgnoreSigpipe() -{ -#ifdef _unix_ - signal(SIGPIPE, SIG_IGN); -#endif -} - -void ConfigureCrashHandler() -{ - TSignalRegistry::Get()->PushCallback(AllCrashSignals, CrashSignalHandler); - TSignalRegistry::Get()->PushDefaultSignalHandler(AllCrashSignals); -} - -namespace { - -void ExitZero(int /*unused*/) -{ -#if defined(_linux_) && defined(CLANG_COVERAGE) - __llvm_profile_write_file(); -#endif - // TODO(babenko): replace with pure "exit" some day. - // Currently this causes some RPC requests to master to be replied with "Promise abandoned" error, - // which is not retriable. - AbortProcessSilently(EProcessExitCode::OK); -} - -} // namespace - -void ConfigureExitZeroOnSigterm() -{ -#ifdef _unix_ - signal(SIGTERM, ExitZero); -#endif -} - -void ConfigureAllocator(const TAllocatorOptions& options) -{ -#ifdef _linux_ - NProfiling::EnableTCMallocProfiler(); - - NYTProf::EnableMemoryProfilingTags(options.SnapshotUpdatePeriod); - - NBacktrace::SetAbslStackUnwinder(); -#else - Y_UNUSED(options); -#endif -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program.h b/yt/yt/library/program/program.h deleted file mode 100644 index e60d374574e..00000000000 --- a/yt/yt/library/program/program.h +++ /dev/null @@ -1,152 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/public.h> - -#include <library/cpp/getopt/last_getopt.h> - -#include <yt/yt/core/yson/string.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TProgram -{ -public: - TProgram(); - ~TProgram(); - - TProgram(const TProgram&) = delete; - TProgram(TProgram&&) = delete; - - // This call actually never returns; - // |int| return type is just for the symmetry with |main|. - [[noreturn]] - int Run(int argc, const char** argv); - - //! Handles --version/--yt-version/--build [--yson] if they are present. - void HandleVersionAndBuild(); - - //! Nongracefully aborts the program. - /*! - * Tries to flush logging messages. - * Aborts via |_exit| call. - */ - [[noreturn]] - static void Abort(int code) noexcept; - - //! A typed version of #Abort. - template <class E> - requires std::is_enum_v<E> - [[noreturn]] - static void Abort(E exitCode) noexcept; - -protected: - NLastGetopt::TOpts Opts_; - TString Argv0_; - bool PrintYTVersion_ = false; - bool PrintVersion_ = false; - bool PrintBuild_ = false; - bool UseYson_ = false; - - virtual void DoRun() = 0; - - virtual void OnError(const TString& message) noexcept; - - virtual bool ShouldAbortOnHungShutdown() noexcept; - - void SetCrashOnError(); - - //! Handler for --yt-version command argument. - [[noreturn]] - void PrintYTVersionAndExit(); - - //! Handler for --build command argument. - [[noreturn]] - void PrintBuildAndExit(); - - //! Handler for --version command argument. - //! By default, --version and --yt-version work the same way, - //! but some YT components (e.g. CHYT) can override it to provide its own version. - [[noreturn]] - virtual void PrintVersionAndExit(); - - [[noreturn]] - void Exit(int code) noexcept; - - //! A typed version of #Exit. - template <class E> - requires std::is_enum_v<E> - [[noreturn]] - void Exit(E exitCode) noexcept; - - const NLastGetopt::TOptsParseResult& GetOptsParseResult() const; - -private: - // Custom handler for option parsing errors. - class TOptsParseResult; - - std::unique_ptr<TOptsParseResult> OptsParseResult_; - bool CrashOnError_ = false; -}; - -//////////////////////////////////////////////////////////////////////////////// - -//! The simplest exception possible. -//! Here we refrain from using TErrorException, as it relies on proper configuration of singleton subsystems, -//! which might not be the case during startup. -class TProgramException - : public std::exception -{ -public: - explicit TProgramException(TString what); - - const char* what() const noexcept override; - -private: - const TString What_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -//! Helper for TOpt::StoreMappedResult to validate file paths for existence. -TString CheckPathExistsArgMapper(const TString& arg); - -//! Helper for TOpt::StoreMappedResult to parse types with #FromString. -template <class T> -T FromStringArgMapper(TStringBuf arg); - -//! Helper for TOpt::StoreMappedResult to parse enums. -template <class T> -T ParseEnumArgMapper(TStringBuf arg); - -//! Helper for TOpt::StoreMappedResult to parse YSON strings. -NYson::TYsonString CheckYsonArgMapper(const TString& arg); - -//! Drop privileges and save them if running with suid-bit. -void ConfigureUids(); - -void ConfigureIgnoreSigpipe(); - -//! Intercepts standard crash signals (see signal_registry.h for full list) with a nice handler. -void ConfigureCrashHandler(); - -//! Intercepts SIGTERM and terminates the process immediately with zero exit code. -void ConfigureExitZeroOnSigterm(); - -//////////////////////////////////////////////////////////////////////////////// - -struct TAllocatorOptions -{ - std::optional<TDuration> SnapshotUpdatePeriod; -}; - -void ConfigureAllocator(const TAllocatorOptions& options = {}); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT - -#define PROGRAM_INL_H_ -#include "program-inl.h" -#undef PROGRAM_INL_H_ diff --git a/yt/yt/library/program/program_config_mixin.cpp b/yt/yt/library/program/program_config_mixin.cpp deleted file mode 100644 index 9ced4de64fc..00000000000 --- a/yt/yt/library/program/program_config_mixin.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "program_config_mixin.h" diff --git a/yt/yt/library/program/program_config_mixin.h b/yt/yt/library/program/program_config_mixin.h deleted file mode 100644 index 88e9ab65a2d..00000000000 --- a/yt/yt/library/program/program_config_mixin.h +++ /dev/null @@ -1,248 +0,0 @@ -#pragma once - -#include "program_mixin.h" - -#include <yt/yt/core/yson/writer.h> - -#include <yt/yt/core/ytree/convert.h> -#include <yt/yt/core/ytree/yson_struct.h> - -#include <library/cpp/yt/string/enum.h> - -#include <library/cpp/yt/system/exit.h> - -#include <util/stream/file.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -template <class TConfig, class TDynamicConfig = void> -class TProgramConfigMixin - : public virtual TProgramMixinBase -{ -protected: - explicit TProgramConfigMixin( - NLastGetopt::TOpts& opts, - bool required = true, - const TString& argumentName = "config") - : ArgumentName_(argumentName) - { - auto opt = opts - .AddLongOption(TString(argumentName), Format("path to %v file (in YSON format)", argumentName)) - .Handler0([&] { ConfigFlag_ = true; }) - .StoreMappedResult(&ConfigPath_, &CheckPathExistsArgMapper) - .RequiredArgument("FILE"); - if (required) { - opt.Required(); - } else { - opt.Optional(); - } - - opts - .AddLongOption( - Format("%v-schema", argumentName), - Format("Prints %v schema", argumentName)) - .OptionalValue(YsonSchemaFormat_, "FORMAT") - .Handler0([&] { ConfigSchemaFlag_ = true; }) - .StoreResult(&ConfigSchema_); - opts - .AddLongOption( - Format("%v-template", argumentName), - Format("Prints %v template", argumentName)) - .OptionalArgument() - .SetFlag(&ConfigTemplateFlag_); - opts - .AddLongOption( - Format("%v-actual", argumentName), - Format("Prints actual %v", argumentName)) - .OptionalArgument() - .SetFlag(&ConfigActualFlag_); - opts - .AddLongOption( - Format("%v-unrecognized", argumentName), - Format("Prints unrecognized %v", argumentName)) - .OptionalArgument() - .SetFlag(&ConfigUnrecognizedFlag_); - - opts - .AddLongOption( - Format("%v-unrecognized-strategy", argumentName), - Format("Configures strategy for unrecognized attributes in %v, variants: %v", - argumentName, - JoinToString( - TEnumTraits<NYTree::EUnrecognizedStrategy>::GetDomainValues(), - [] (TStringBuilderBase* builder, NYTree::EUnrecognizedStrategy strategy) { - builder->AppendFormat(FormatEnum(strategy)); - }, - TStringBuf(", ")))) - .DefaultValue(FormatEnum(UnrecognizedStrategy_)) - .template Handler1T<TStringBuf>([&] (TStringBuf value) { - UnrecognizedStrategy_ = ParseEnum<NYTree::EUnrecognizedStrategy>(value); - }); - - if constexpr (!std::is_same_v<TDynamicConfig, void>) { - opts - .AddLongOption( - Format("dynamic-%v-schema", argumentName), - Format("Prints %v schema", argumentName)) - .OptionalValue(YsonSchemaFormat_, "FORMAT") - .Handler0([&] { DynamicConfigSchemaFlag_ = true; }) - .StoreResult(&DynamicConfigSchema_); - opts - .AddLongOption( - Format("dynamic-%v-template", argumentName), - Format("Prints dynamic %v template", argumentName)) - .OptionalArgument() - .SetFlag(&DynamicConfigTemplateFlag_); - } - - RegisterMixinCallback([&] { Handle(); }); - } - - TIntrusivePtr<TConfig> GetConfig(bool returnNullIfNotSupplied = false) - { - if (returnNullIfNotSupplied && !ConfigFlag_) { - return nullptr; - } - - if (!Config_) { - LoadConfig(); - } - return Config_; - } - - NYTree::INodePtr GetConfigNode(bool returnNullIfNotSupplied = false) - { - if (returnNullIfNotSupplied && !ConfigFlag_) { - return nullptr; - } - - if (!ConfigNode_) { - LoadConfigNode(); - } - return ConfigNode_; - } - -private: - const TString ArgumentName_; - - bool ConfigFlag_; - TString ConfigPath_; - bool ConfigSchemaFlag_ = false; - TString ConfigSchema_; - bool ConfigTemplateFlag_; - bool ConfigActualFlag_; - bool ConfigUnrecognizedFlag_; - bool DynamicConfigSchemaFlag_ = false; - TString DynamicConfigSchema_; - bool DynamicConfigTemplateFlag_ = false; - NYTree::EUnrecognizedStrategy UnrecognizedStrategy_ = NYTree::EUnrecognizedStrategy::KeepRecursive; - - static constexpr auto YsonSchemaFormat_ = "yson-schema"; - - TIntrusivePtr<TConfig> Config_; - NYTree::INodePtr ConfigNode_; - - void LoadConfigNode() - { - using namespace NYTree; - - if (!ConfigFlag_){ - THROW_ERROR_EXCEPTION("Missing %qv option", ArgumentName_); - } - - try { - TIFStream stream(ConfigPath_); - ConfigNode_ = ConvertToNode(&stream); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error parsing %v file %v", - ArgumentName_, - ConfigPath_) - << ex; - } - } - - void LoadConfig() - { - if (!ConfigNode_) { - LoadConfigNode(); - } - - try { - Config_ = New<TConfig>(); - Config_->SetUnrecognizedStrategy(UnrecognizedStrategy_); - Config_->Load(ConfigNode_); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading %v file %v", - ArgumentName_, - ConfigPath_) - << ex; - } - } - - void Handle() - { - auto print = [] (const auto& config) { - using namespace NYson; - TYsonWriter writer(&Cout, EYsonFormat::Pretty); - config->Save(&writer); - Cout << Endl; - }; - - auto printNode = [] (const auto& node) { - using namespace NYson; - TYsonWriter writer(&Cout, EYsonFormat::Pretty); - NYTree::Serialize(node, &writer); - Cout << Endl; - }; - - auto printSchema = [] (const auto& config, TString format) { - if (format == YsonSchemaFormat_) { - using namespace NYson; - TYsonWriter writer(&Cout, EYsonFormat::Pretty); - config->WriteSchema(&writer); - Cout << Endl; - } else { - THROW_ERROR_EXCEPTION("Unknown schema format %Qv", format); - } - }; - - if (ConfigSchemaFlag_) { - printSchema(New<TConfig>(), ConfigSchema_); - Exit(EProcessExitCode::OK); - } - - if (ConfigTemplateFlag_) { - print(New<TConfig>()); - Exit(EProcessExitCode::OK); - } - - if (ConfigActualFlag_) { - print(GetConfig()); - Exit(EProcessExitCode::OK); - } - - if (ConfigUnrecognizedFlag_) { - auto unrecognized = GetConfig()->GetRecursiveUnrecognized(); - printNode(*unrecognized); - Exit(EProcessExitCode::OK); - } - - if constexpr (!std::is_same_v<TDynamicConfig, void>) { - if (DynamicConfigSchemaFlag_) { - printSchema(New<TDynamicConfig>(), DynamicConfigSchema_); - Exit(EProcessExitCode::OK); - } - - if (DynamicConfigTemplateFlag_) { - print(New<TDynamicConfig>()); - Exit(EProcessExitCode::OK); - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_mixin.cpp b/yt/yt/library/program/program_mixin.cpp deleted file mode 100644 index b9c2c5bd1a5..00000000000 --- a/yt/yt/library/program/program_mixin.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "program_mixin.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -void TProgramMixinBase::RegisterMixinCallback(std::function<void()> callback) -{ - MixinCallbacks_.push_back(std::move(callback)); -} - -void TProgramMixinBase::RunMixinCallbacks() -{ - for (const auto& callback : MixinCallbacks_) { - callback(); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_mixin.h b/yt/yt/library/program/program_mixin.h deleted file mode 100644 index 3104d989465..00000000000 --- a/yt/yt/library/program/program_mixin.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "program.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TProgramMixinBase - : public virtual TProgram -{ -protected: - void RegisterMixinCallback(std::function<void()> callback); - void RunMixinCallbacks(); - -private: - std::vector<std::function<void()>> MixinCallbacks_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_pdeathsig_mixin.cpp b/yt/yt/library/program/program_pdeathsig_mixin.cpp deleted file mode 100644 index f78623121c3..00000000000 --- a/yt/yt/library/program/program_pdeathsig_mixin.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "program_pdeathsig_mixin.h" - -#ifdef _linux_ -#include <sys/prctl.h> -#endif - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TProgramPdeathsigMixin::TProgramPdeathsigMixin(NLastGetopt::TOpts& opts) -{ - opts.AddLongOption("pdeathsig", "parent death signal") - .StoreResult(&ParentDeathSignal_) - .RequiredArgument("PDEATHSIG"); - - RegisterMixinCallback([&] { Handle(); }); -} - -void TProgramPdeathsigMixin::Handle() -{ - if (ParentDeathSignal_ > 0) { -#ifdef _linux_ - // Parent death signal is set by testing framework to avoid dangling processes when test runner crashes. - // Unfortunately, setting pdeathsig in preexec_fn in subprocess call in test runner is not working - // when the program has suid bit (pdeath_sig is reset after exec call in this case) - // More details can be found in - // http://linux.die.net/man/2/prctl - // http://www.isec.pl/vulnerabilities/isec-0024-death-signal.txt - YT_VERIFY(prctl(PR_SET_PDEATHSIG, ParentDeathSignal_) == 0); -#endif - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_pdeathsig_mixin.h b/yt/yt/library/program/program_pdeathsig_mixin.h deleted file mode 100644 index 5c26bdf3a08..00000000000 --- a/yt/yt/library/program/program_pdeathsig_mixin.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "program_mixin.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TProgramPdeathsigMixin - : public virtual TProgramMixinBase -{ -protected: - explicit TProgramPdeathsigMixin(NLastGetopt::TOpts& opts); - -private: - int ParentDeathSignal_ = -1; - - void Handle(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_setsid_mixin.cpp b/yt/yt/library/program/program_setsid_mixin.cpp deleted file mode 100644 index 19a0504ab67..00000000000 --- a/yt/yt/library/program/program_setsid_mixin.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "program_setsid_mixin.h" - -#ifdef _linux_ -#include <unistd.h> -#endif - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TProgramSetsidMixin::TProgramSetsidMixin(NLastGetopt::TOpts& opts) -{ - opts.AddLongOption("setsid", "create a new session") - .StoreTrue(&Setsid_) - .Optional(); - - RegisterMixinCallback([&] { Handle(); }); -} - -void TProgramSetsidMixin::Handle() -{ - if (Setsid_) { -#ifdef _linux_ - setsid(); -#endif - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/program_setsid_mixin.h b/yt/yt/library/program/program_setsid_mixin.h deleted file mode 100644 index 38465e97d7d..00000000000 --- a/yt/yt/library/program/program_setsid_mixin.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "program_mixin.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TProgramSetsidMixin - : public virtual TProgramMixinBase -{ -protected: - explicit TProgramSetsidMixin(NLastGetopt::TOpts& opts); - -private: - bool Setsid_ = false; - - void Handle(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/public.h b/yt/yt/library/program/public.h deleted file mode 100644 index e45512239b0..00000000000 --- a/yt/yt/library/program/public.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/public.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_CLASS(TBuildInfo) -DECLARE_REFCOUNTED_CLASS(TRpcConfig) -DECLARE_REFCOUNTED_CLASS(THeapSizeLimitConfig) -DECLARE_REFCOUNTED_CLASS(THeapProfilerConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/library/program/ya.make b/yt/yt/library/program/ya.make deleted file mode 100644 index 2a2fec5c0e0..00000000000 --- a/yt/yt/library/program/ya.make +++ /dev/null @@ -1,27 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - build_attributes.cpp - config.cpp - helpers.cpp - program.cpp - program_mixin.cpp - program_config_mixin.cpp - program_pdeathsig_mixin.cpp - program_setsid_mixin.cpp -) - -PEERDIR( - yt/yt/core - yt/yt/library/profiling/tcmalloc - yt/yt/library/ytprof - yt/yt/library/tcmalloc # for tcmalloc singleton - library/cpp/yt/string - library/cpp/yt/system - library/cpp/yt/backtrace/absl_unwinder - library/cpp/getopt/small -) - -END() diff --git a/yt/yt/library/tcmalloc/config.cpp b/yt/yt/library/tcmalloc/config.cpp deleted file mode 100644 index 3219ffcf22d..00000000000 --- a/yt/yt/library/tcmalloc/config.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include "config.h" - -namespace NYT::NTCMalloc { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void THeapSizeLimitConfig::ApplyDynamicInplace(const TDynamicHeapSizeLimitConfigPtr& dynamicConfig) -{ - UpdateYsonStructField(ContainerMemoryRatio, dynamicConfig->ContainerMemoryRatio); - UpdateYsonStructField(ContainerMemoryMargin, dynamicConfig->ContainerMemoryMargin); - UpdateYsonStructField(Hard, dynamicConfig->Hard); - UpdateYsonStructField(DumpMemoryProfileOnViolation, dynamicConfig->DumpMemoryProfileOnViolation); - UpdateYsonStructField(MemoryProfileDumpTimeout, dynamicConfig->MemoryProfileDumpTimeout); - UpdateYsonStructField(MemoryProfileDumpPath, dynamicConfig->MemoryProfileDumpPath); - UpdateYsonStructField(MemoryProfileDumpFilenameSuffix, dynamicConfig->MemoryProfileDumpFilenameSuffix); -} - -THeapSizeLimitConfigPtr THeapSizeLimitConfig::ApplyDynamic(const TDynamicHeapSizeLimitConfigPtr& dynamicConfig) const -{ - auto mergedConfig = CloneYsonStruct(MakeStrong(this)); - mergedConfig->ApplyDynamicInplace(dynamicConfig); - mergedConfig->Postprocess(); - return mergedConfig; -} - -void THeapSizeLimitConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("container_memory_ratio", &TThis::ContainerMemoryRatio) - .Optional(); - registrar.Parameter("container_memory_margin", &TThis::ContainerMemoryMargin) - .Optional(); - registrar.Parameter("hard", &TThis::Hard) - .Default(false); - registrar.Parameter("dump_memory_profile_on_violation", &TThis::DumpMemoryProfileOnViolation) - .Default(false); - registrar.Parameter("memory_profile_dump_timeout", &TThis::MemoryProfileDumpTimeout) - .Default(TDuration::Minutes(10)); - registrar.Parameter("memory_profile_dump_path", &TThis::MemoryProfileDumpPath) - .Default(); - registrar.Parameter("memory_profile_dump_filename_suffix", &TThis::MemoryProfileDumpFilenameSuffix) - .Default(); - - registrar.Postprocessor([] (THeapSizeLimitConfig* config) { - if (config->DumpMemoryProfileOnViolation && !config->MemoryProfileDumpPath) { - THROW_ERROR_EXCEPTION("\"memory_profile_dump_path\" must be set when \"dump_memory_profile_on_violation\" is true"); - } - }); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TDynamicHeapSizeLimitConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("container_memory_ratio", &TThis::ContainerMemoryRatio) - .Default(); - registrar.Parameter("container_memory_margin", &TThis::ContainerMemoryMargin) - .Default(); - registrar.Parameter("hard", &TThis::Hard) - .Default(); - registrar.Parameter("dump_memory_profile_on_violation", &TThis::DumpMemoryProfileOnViolation) - .Default(); - registrar.Parameter("memory_profile_dump_timeout", &TThis::MemoryProfileDumpTimeout) - .Default(); - registrar.Parameter("memory_profile_dump_path", &TThis::MemoryProfileDumpPath) - .Default(); - registrar.Parameter("memory_profile_dump_filename_suffix", &TThis::MemoryProfileDumpFilenameSuffix) - .Default(); -} - -//////////////////////////////////////////////////////////////////////////////// - -TTCMallocConfigPtr TTCMallocConfig::ApplyDynamic(const TDynamicTCMallocConfigPtr& dynamicConfig) const -{ - auto mergedConfig = CloneYsonStruct(MakeStrong(this)); - UpdateYsonStructField(mergedConfig->AggressiveReleaseThreshold, dynamicConfig->AggressiveReleaseThreshold); - UpdateYsonStructField(mergedConfig->AggressiveReleaseThresholdRatio, dynamicConfig->AggressiveReleaseThresholdRatio); - UpdateYsonStructField(mergedConfig->AggressiveReleaseSize, dynamicConfig->AggressiveReleaseSize); - UpdateYsonStructField(mergedConfig->AggressiveReleasePeriod, dynamicConfig->AggressiveReleasePeriod); - UpdateYsonStructField(mergedConfig->GuardedSamplingRate, dynamicConfig->GuardedSamplingRate); - UpdateYsonStructField(mergedConfig->ProfileSamplingRate, dynamicConfig->ProfileSamplingRate); - UpdateYsonStructField(mergedConfig->MaxPerCpuCacheSize, dynamicConfig->MaxPerCpuCacheSize); - UpdateYsonStructField(mergedConfig->MaxTotalThreadCacheBytes, dynamicConfig->MaxTotalThreadCacheBytes); - UpdateYsonStructField(mergedConfig->BackgroundReleaseRate, dynamicConfig->BackgroundReleaseRate); - mergedConfig->HeapSizeLimit->ApplyDynamicInplace(dynamicConfig->HeapSizeLimit); - mergedConfig->Postprocess(); - return mergedConfig; -} - -void TTCMallocConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("aggressive_release_threshold", &TThis::AggressiveReleaseThreshold) - .Default(20_GB); - registrar.Parameter("aggressive_release_threshold_ratio", &TThis::AggressiveReleaseThresholdRatio) - .Optional(); - - registrar.Parameter("aggressive_release_size", &TThis::AggressiveReleaseSize) - .Default(128_MB); - registrar.Parameter("aggressive_release_period", &TThis::AggressiveReleasePeriod) - .Default(TDuration::MilliSeconds(100)); - registrar.Parameter("guarded_sampling_rate", &TThis::GuardedSamplingRate) - .Default(128_MB); - registrar.Parameter("profile_sampling_rate", &TThis::ProfileSamplingRate) - .Default(2_MB); - registrar.Parameter("max_per_cpu_cache_size", &TThis::MaxPerCpuCacheSize) - .Default(3_MB); - registrar.Parameter("max_total_thread_cache_bytes", &TThis::MaxTotalThreadCacheBytes) - .Default(24_MB); - registrar.Parameter("background_release_rate", &TThis::BackgroundReleaseRate) - .Default(32_MB); - - registrar.Parameter("heap_size_limit", &TThis::HeapSizeLimit) - .DefaultNew(); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TDynamicTCMallocConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("aggressive_release_threshold", &TThis::AggressiveReleaseThreshold) - .Default(); - registrar.Parameter("aggressive_release_threshold_ratio", &TThis::AggressiveReleaseThresholdRatio) - .Optional(); - - registrar.Parameter("aggressive_release_size", &TThis::AggressiveReleaseSize) - .Default(); - registrar.Parameter("aggressive_release_period", &TThis::AggressiveReleasePeriod) - .Default(); - registrar.Parameter("guarded_sampling_rate", &TThis::GuardedSamplingRate) - .Default(); - registrar.Parameter("profile_sampling_rate", &TThis::ProfileSamplingRate) - .Default(); - registrar.Parameter("max_per_cpu_cache_size", &TThis::MaxPerCpuCacheSize) - .Default(); - registrar.Parameter("max_total_thread_cache_bytes", &TThis::MaxTotalThreadCacheBytes) - .Default(); - registrar.Parameter("background_release_rate", &TThis::BackgroundReleaseRate) - .Default(); - - registrar.Parameter("heap_size_limit", &TThis::HeapSizeLimit) - .DefaultNew(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc - diff --git a/yt/yt/library/tcmalloc/config.h b/yt/yt/library/tcmalloc/config.h deleted file mode 100644 index 86d52781466..00000000000 --- a/yt/yt/library/tcmalloc/config.h +++ /dev/null @@ -1,135 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NTCMalloc { - -//////////////////////////////////////////////////////////////////////////////// - -struct THeapSizeLimitConfig - : public NYTree::TYsonStruct -{ - //! Limit program memory in terms of container memory. - //! If program heap size exceeds the limit tcmalloc is instructed to release memory to the kernel. - std::optional<double> ContainerMemoryRatio; - - //! Similar to #ContainerMemoryRatio, but is set in terms of absolute difference from - //! the container memory limit. - //! For example, if ContainerMemoryLimit=200Gb and ContainerMemoryMargin=1Gb - //! then tcmalloc limit will be 199Gb. - std::optional<i64> ContainerMemoryMargin; - - //! If true tcmalloc crashes when system allocates more memory than #ContainerMemoryRatio/#ContainerMemoryMargin. - bool Hard; - - bool DumpMemoryProfileOnViolation; - - TDuration MemoryProfileDumpTimeout; - - //! Filenames are as follows: - //! $(MemoryProfileDumpPath)/$(Name)_$(MemoryProfileDumpFilenameSuffix)_$(Timestamp).$(Ext) or - //! $(MemoryProfileDumpPath)/$(Name)_$(Timestamp).$(Ext) (if MemoryProfileDumpFilenameSuffix is missing) - std::optional<TString> MemoryProfileDumpPath; - std::optional<TString> MemoryProfileDumpFilenameSuffix; - - void ApplyDynamicInplace(const TDynamicHeapSizeLimitConfigPtr& dynamicConfig); - THeapSizeLimitConfigPtr ApplyDynamic(const TDynamicHeapSizeLimitConfigPtr& dynamicConfig) const; - - REGISTER_YSON_STRUCT(THeapSizeLimitConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(THeapSizeLimitConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TDynamicHeapSizeLimitConfig - : public NYTree::TYsonStruct -{ - std::optional<double> ContainerMemoryRatio; - std::optional<i64> ContainerMemoryMargin; - - std::optional<bool> Hard; - - std::optional<bool> DumpMemoryProfileOnViolation; - - std::optional<TDuration> MemoryProfileDumpTimeout; - - std::optional<TString> MemoryProfileDumpPath; - std::optional<TString> MemoryProfileDumpFilenameSuffix; - - REGISTER_YSON_STRUCT(TDynamicHeapSizeLimitConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TDynamicHeapSizeLimitConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TTCMallocConfig - : public NYTree::TYsonStruct -{ - //! Threshold in bytes. - i64 AggressiveReleaseThreshold; - - //! Threshold in fractions of total memory of the container. - std::optional<double> AggressiveReleaseThresholdRatio; - - i64 AggressiveReleaseSize; - TDuration AggressiveReleasePeriod; - - //! Approximately 1/#GuardedSamplingRate of all allocations of - //! size <= 256 KiB will be under GWP-ASAN. - std::optional<i64> GuardedSamplingRate; - - i64 ProfileSamplingRate; - i64 MaxPerCpuCacheSize; - i64 MaxTotalThreadCacheBytes; - i64 BackgroundReleaseRate; - - THeapSizeLimitConfigPtr HeapSizeLimit; - - TTCMallocConfigPtr ApplyDynamic(const TDynamicTCMallocConfigPtr& dynamicConfig) const; - - REGISTER_YSON_STRUCT(TTCMallocConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TTCMallocConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TDynamicTCMallocConfig - : public NYTree::TYsonStruct -{ - std::optional<i64> AggressiveReleaseThreshold; - - std::optional<double> AggressiveReleaseThresholdRatio; - - std::optional<i64> AggressiveReleaseSize; - std::optional<TDuration> AggressiveReleasePeriod; - - std::optional<i64> GuardedSamplingRate; - - std::optional<i64> ProfileSamplingRate; - std::optional<i64> MaxPerCpuCacheSize; - std::optional<i64> MaxTotalThreadCacheBytes; - std::optional<i64> BackgroundReleaseRate; - - TDynamicHeapSizeLimitConfigPtr HeapSizeLimit; - - REGISTER_YSON_STRUCT(TDynamicTCMallocConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TDynamicTCMallocConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc diff --git a/yt/yt/library/tcmalloc/configure_tcmalloc_manager.cpp b/yt/yt/library/tcmalloc/configure_tcmalloc_manager.cpp deleted file mode 100644 index 49829b93577..00000000000 --- a/yt/yt/library/tcmalloc/configure_tcmalloc_manager.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "tcmalloc_manager.h" -#include "config.h" - -#include <yt/yt/core/misc/configurable_singleton_def.h> - -namespace NYT::NTCMalloc { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void SetupSingletonConfigParameter(TYsonStructParameter<TTCMallocConfigPtr>& parameter) -{ - parameter.DefaultNew(); -} - -void SetupSingletonConfigParameter(TYsonStructParameter<TDynamicTCMallocConfigPtr>& parameter) -{ - parameter.DefaultNew(); -} - -void ConfigureSingleton(const TTCMallocConfigPtr& config) -{ - TTCMallocManager::Configure(config); -} - -void ReconfigureSingleton( - const TTCMallocConfigPtr& config, - const TDynamicTCMallocConfigPtr& dynamicConfig) -{ - TTCMallocManager::Configure(config->ApplyDynamic(dynamicConfig)); -} - -YT_DEFINE_RECONFIGURABLE_SINGLETON( - "tcmalloc", - TTCMallocConfig, - TDynamicTCMallocConfig); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc diff --git a/yt/yt/library/tcmalloc/public.h b/yt/yt/library/tcmalloc/public.h deleted file mode 100644 index c9600da99a9..00000000000 --- a/yt/yt/library/tcmalloc/public.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/configurable_singleton_decl.h> - -#include <library/cpp/yt/memory/ref_counted.h> - -namespace NYT::NTCMalloc { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(TTCMallocConfig) -DECLARE_REFCOUNTED_STRUCT(THeapSizeLimitConfig) -DECLARE_REFCOUNTED_STRUCT(TDynamicTCMallocConfig) -DECLARE_REFCOUNTED_STRUCT(TDynamicHeapSizeLimitConfig) - -YT_DECLARE_RECONFIGURABLE_SINGLETON(TTCMallocConfig, TTCMallocConfig); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc diff --git a/yt/yt/library/tcmalloc/tcmalloc_manager.cpp b/yt/yt/library/tcmalloc/tcmalloc_manager.cpp deleted file mode 100644 index fbd938f3a9d..00000000000 --- a/yt/yt/library/tcmalloc/tcmalloc_manager.cpp +++ /dev/null @@ -1,478 +0,0 @@ -#include "tcmalloc_manager.h" - -#include "config.h" - -#include <yt/yt/library/profiling/resource_tracker/resource_tracker.h> - -#include <yt/yt/library/ytprof/external_pprof.h> -#include <yt/yt/library/ytprof/heap_profiler.h> -#include <yt/yt/library/ytprof/profile.h> - -#include <yt/yt/core/misc/fs.h> -#include <yt/yt/core/misc/crash_handler.h> - -#include <library/cpp/yt/memory/leaky_singleton.h> -#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> - -#include <library/cpp/yt/system/exit.h> - -#include <util/system/thread.h> -#include <util/system/shellcommand.h> - -#include <tcmalloc/malloc_extension.h> - -#include <thread> -#include <mutex> - -namespace NYT::NTCMalloc { - -//////////////////////////////////////////////////////////////////////////////// - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TCMalloc"); - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(TOomProfileManifest) - -struct TOomProfileManifest - : public NYTree::TYsonStruct -{ - TString CurrentProfilePath; - TString PeakProfilePath; - - REGISTER_YSON_STRUCT(TOomProfileManifest); - - static void Register(TRegistrar registrar) - { - // TODO(babenko): fix names, see ytserver_mail_profile_link.py - registrar.Parameter("heap_profile_path", &TThis::CurrentProfilePath) - .Default(); - registrar.Parameter("peak_profile_path", &TThis::PeakProfilePath) - .Default(); - } -}; - -DEFINE_REFCOUNTED_TYPE(TOomProfileManifest) - -//////////////////////////////////////////////////////////////////////////////// - -TString MakeIncompletePath(const TString& path) -{ - return NYT::Format("%v_incomplete", path); -} - -void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::ProfileType profileType) -{ - auto profile = NYTProf::CaptureHeapProfile(profileType); - SymbolizeByExternalPProf(&profile, NYTProf::TSymbolizationOptions{ - .RunTool = [] (const std::vector<TString>& args) { - TShellCommand command{args[0], TList<TString>{args.begin() + 1, args.end()}}; - command.Run(); - }, - }); - - auto incompletePath = MakeIncompletePath(memoryProfilePath); - - TFileOutput output(incompletePath); - NYTProf::WriteCompressedProfile(&output, profile); - output.Finish(); - NFS::Rename(incompletePath, memoryProfilePath); -} - -void MemoryProfileTimeoutHandler(int /*signal*/) -{ - AbortProcessDramatically( - EProcessExitCode::GenericError, - "Process hung while dumping heap profile"); -} - -void SetupMemoryProfileTimeout(int timeout) -{ - ::signal(SIGALRM, &MemoryProfileTimeoutHandler); - ::alarm(timeout); -} - -void DumpManifest(const TOomProfileManifestPtr& manifest, const TString& fileName) -{ - TFileOutput output(fileName); - NYson::TYsonWriter writer(&output, NYson::EYsonFormat::Pretty); - Serialize(manifest, &writer); - writer.Flush(); - output.Finish(); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TMemoryLimitHandler - : public TRefCounted -{ -public: - explicit TMemoryLimitHandler(THeapSizeLimitConfigPtr config) - : Config_(std::move(config)) - { - Thread_ = std::thread([this] { - Handle(); - }); - } - - ~TMemoryLimitHandler() - { - { - std::unique_lock<std::mutex> lock(Mutex_); - Fired_ = true; - CV_.notify_all(); - } - - Thread_.join(); - } - - void Fire() - { - std::unique_lock<std::mutex> lock(Mutex_); - Fired_ = true; - NeedToHandle_ = true; - CV_.notify_all(); - } - -private: - const THeapSizeLimitConfigPtr Config_; - - bool Fired_ = false; - bool NeedToHandle_ = false; - std::mutex Mutex_; - std::condition_variable CV_; - std::thread Thread_; - - void Handle() - { - std::unique_lock<std::mutex> lock(Mutex_); - CV_.wait(lock, [&] { - return Fired_; - }); - - if (!NeedToHandle_) { - return; - } - - auto timestamp = TInstant::Now().FormatLocalTime("%Y%m%dT%H%M%S"); - auto manifest = New<TOomProfileManifest>(); - auto manifestPath = GetManifestPath(timestamp); - manifest->CurrentProfilePath = GetCurrentDumpPath(timestamp); - manifest->PeakProfilePath = GetPeakDumpPath(timestamp); - - Cerr << "*** Forking process to write heap profiles" << Endl - << " current: " << manifest->CurrentProfilePath << Endl - << " peak: " << manifest->PeakProfilePath << Endl - << " manifest: " << manifestPath << Endl; - - SetupMemoryProfileTimeout(Config_->MemoryProfileDumpTimeout.Seconds()); - - auto childPid = fork(); - if (childPid == 0) { - NFS::MakeDirRecursive(*Config_->MemoryProfileDumpPath); - SetupMemoryProfileTimeout(Config_->MemoryProfileDumpTimeout.Seconds()); - CollectAndDumpMemoryProfile(manifest->CurrentProfilePath, tcmalloc::ProfileType::kHeap); - CollectAndDumpMemoryProfile(manifest->PeakProfilePath, tcmalloc::ProfileType::kPeakHeap); - DumpManifest(manifest, manifestPath); - - Cerr << "*** Heap profiles are written" << Endl; - AbortProcessSilently(EProcessExitCode::OK); - } - - if (childPid < 0) { - Cerr << "*** Fork failed: " << LastSystemErrorText() << Endl; - AbortProcessSilently(EProcessExitCode::GenericError); - } - - ExecWaitForChild(childPid); - AbortProcessSilently(EProcessExitCode::OK); - } - - auto MakeSuffixFormatter(const TString& timestamp) const - { - return NYT::MakeFormatterWrapper([this, ×tamp] (TStringBuilderBase* builder) { - if (Config_->MemoryProfileDumpFilenameSuffix) { - builder->AppendFormat("%v_", *Config_->MemoryProfileDumpFilenameSuffix); - } - FormatValue(builder, timestamp, "v"); - }); - } - - TString GetCurrentDumpPath(const TString& timestamp) const - { - return Format( - "%v/current_%v.pb.gz", - Config_->MemoryProfileDumpPath, - MakeSuffixFormatter(timestamp)); - } - - TString GetPeakDumpPath(const TString& timestamp) const - { - return Format( - "%v/peak_%v.pb.gz", - Config_->MemoryProfileDumpPath, - MakeSuffixFormatter(timestamp)); - } - - TString GetManifestPath(const TString& timestamp) const - { - return Format( - "%v/oom_profile_paths_%v.yson", - Config_->MemoryProfileDumpPath, - MakeSuffixFormatter(timestamp)); - } - - void ExecWaitForChild(int pid) - { - Cerr << "*** Start waiting for the child" << Endl; - - auto command = Format("while [ -e /proc/%v ]; do sleep 1; done;", pid); - execl("/bin/bash", "/bin/bash", "-c", command.c_str(), (void*)nullptr); - - Cerr << "*** Failed to switch main process to dummy child waiter: " - << LastSystemErrorText() << Endl; - } -}; - -DEFINE_REFCOUNTED_TYPE(TMemoryLimitHandler); - -//////////////////////////////////////////////////////////////////////////////// - -template <class TMallocExtension> -concept CSupportsLimitHandler = requires (TMallocExtension extension) -{ - { extension.GetSoftMemoryLimitHandler() }; -}; - -template <typename TMallocExtension, typename THandler> -void SetSoftMemoryLimitHandler(THandler) -{ - WriteToStderr("TCMalloc does not support memory limit handler\n"); -} - -template <CSupportsLimitHandler TMallocExtension, typename THandler> -void SetSoftMemoryLimitHandler(THandler handler) -{ - TMallocExtension::SetSoftMemoryLimitHandler(handler); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TMemoryLimitHandlerInstaller -{ -public: - static TMemoryLimitHandlerInstaller* Get() - { - return LeakySingleton<TMemoryLimitHandlerInstaller>(); - } - - void Reconfigure(const THeapSizeLimitConfigPtr& config) - { - if (config->DumpMemoryProfileOnViolation) { - Enable(config); - } else { - Disable(); - } - } - -private: - DECLARE_LEAKY_SINGLETON_FRIEND(); - - TAtomicIntrusivePtr<TMemoryLimitHandler> LimitHandler_; - - TMemoryLimitHandlerInstaller() - { - SetSoftMemoryLimitHandler<tcmalloc::MallocExtension>(&HandleTCMallocLimit); - } - - static void HandleTCMallocLimit() - { - if (auto handler = Get()->LimitHandler_.Acquire()) { - handler->Fire(); - } - } - - void Enable(const THeapSizeLimitConfigPtr& config) - { - if (LimitHandler_.Acquire()) { - return; - } - - TAtomicIntrusivePtr<TMemoryLimitHandler>::TRawPtr expected = nullptr; - LimitHandler_.CompareAndSwap(expected, New<TMemoryLimitHandler>(config)); - } - - void Disable() - { - LimitHandler_.Reset(); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TLimitsAdjuster -{ -public: - static TLimitsAdjuster* Get() - { - return LeakySingleton<TLimitsAdjuster>(); - } - - void Reconfigure(const TTCMallocConfigPtr& config) - { - i64 totalMemory = GetAnonymousMemoryLimit(); - AdjustPageHeapLimit(totalMemory, config); - AdjustAggressiveReleaseThreshold(totalMemory, config); - } - - i64 GetAggressiveReleaseThreshold() - { - return AggressiveReleaseThreshold_; - } - -private: - DECLARE_LEAKY_SINGLETON_FRIEND(); - - TLimitsAdjuster() = default; - - using TAllocatorMemoryLimit = tcmalloc::MallocExtension::MemoryLimit; - - TAllocatorMemoryLimit AppliedLimit_; - i64 AggressiveReleaseThreshold_ = 0; - - - void AdjustPageHeapLimit(i64 totalMemory, const TTCMallocConfigPtr& config) - { - auto proposed = ProposeHeapMemoryLimit(totalMemory, config); - - if (proposed.limit == AppliedLimit_.limit && proposed.hard == AppliedLimit_.hard) { - // Already applied. - return; - } - - YT_LOG_INFO("Changing TCMalloc memory limit (Limit: %v, Hard: %v)", - proposed.limit, - proposed.hard); - - tcmalloc::MallocExtension::SetMemoryLimit(proposed); - AppliedLimit_ = proposed; - } - - void AdjustAggressiveReleaseThreshold(i64 totalMemory, const TTCMallocConfigPtr& config) - { - if (totalMemory && config->AggressiveReleaseThresholdRatio) { - AggressiveReleaseThreshold_ = *config->AggressiveReleaseThresholdRatio * totalMemory; - } else { - AggressiveReleaseThreshold_ = config->AggressiveReleaseThreshold; - } - } - - i64 GetAnonymousMemoryLimit() const - { - return NProfiling::TResourceTracker::GetAnonymousMemoryLimit(); - } - - TAllocatorMemoryLimit ProposeHeapMemoryLimit(i64 totalMemory, const TTCMallocConfigPtr& config) const - { - const auto& heapSizeConfig = config->HeapSizeLimit; - - if (totalMemory == 0 || !heapSizeConfig->ContainerMemoryRatio && !heapSizeConfig->ContainerMemoryMargin) { - return {}; - } - - TAllocatorMemoryLimit proposed; - proposed.hard = heapSizeConfig->Hard; - - if (heapSizeConfig->ContainerMemoryMargin) { - proposed.limit = totalMemory - *heapSizeConfig->ContainerMemoryMargin; - } else { - proposed.limit = *heapSizeConfig->ContainerMemoryRatio * totalMemory; - } - - return proposed; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -class TTCMallocManagerImpl -{ -public: - static TTCMallocManagerImpl* Get() - { - return LeakySingleton<TTCMallocManagerImpl>(); - } - - void Configure(const TTCMallocConfigPtr& config) - { - tcmalloc::MallocExtension::SetProfileSamplingRate(config->ProfileSamplingRate); - tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(config->MaxPerCpuCacheSize); - tcmalloc::MallocExtension::SetMaxTotalThreadCacheBytes(config->MaxTotalThreadCacheBytes); - tcmalloc::MallocExtension::SetBackgroundReleaseRate( - tcmalloc::MallocExtension::BytesPerSecond{static_cast<size_t>(config->BackgroundReleaseRate)}); - - tcmalloc::MallocExtension::EnableForkSupport(); - - if (config->GuardedSamplingRate) { - tcmalloc::MallocExtension::SetGuardedSamplingRate(*config->GuardedSamplingRate); - tcmalloc::MallocExtension::ActivateGuardedSampling(); - } - - Config_.Store(config); - - if (tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) { - std::call_once(InitAggressiveReleaseThread_, [&] { - std::thread([&] { - ::TThread::SetCurrentThreadName("TCMallocBack"); - - while (true) { - auto config = Config_.Acquire(); - TLimitsAdjuster::Get()->Reconfigure(config); - TMemoryLimitHandlerInstaller::Get()->Reconfigure(config->HeapSizeLimit); - - auto freeBytes = tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.page_heap_free"); - YT_VERIFY(freeBytes); - - if (static_cast<i64>(*freeBytes) > TLimitsAdjuster::Get()->GetAggressiveReleaseThreshold()) { - - YT_LOG_DEBUG("Aggressively releasing memory (FreeBytes: %v, Threshold: %v)", - static_cast<i64>(*freeBytes), - TLimitsAdjuster::Get()->GetAggressiveReleaseThreshold()); - - tcmalloc::MallocExtension::ReleaseMemoryToSystem(config->AggressiveReleaseSize); - } - - Sleep(config->AggressiveReleasePeriod); - } - }).detach(); - }); - } - } - -private: - DECLARE_LEAKY_SINGLETON_FRIEND(); - - TTCMallocManagerImpl() = default; - - TAtomicIntrusivePtr<TTCMallocConfig> Config_; - std::once_flag InitAggressiveReleaseThread_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -void TTCMallocManager::Configure(const TTCMallocConfigPtr& config) -{ - TTCMallocManagerImpl::Get()->Configure(config); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc diff --git a/yt/yt/library/tcmalloc/tcmalloc_manager.h b/yt/yt/library/tcmalloc/tcmalloc_manager.h deleted file mode 100644 index 633246cf48f..00000000000 --- a/yt/yt/library/tcmalloc/tcmalloc_manager.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "public.h" - -namespace NYT::NTCMalloc { - -//////////////////////////////////////////////////////////////////////////////// - -class TTCMallocManager -{ -public: - static void Configure(const TTCMallocConfigPtr& config); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTCMalloc diff --git a/yt/yt/library/tcmalloc/ya.make b/yt/yt/library/tcmalloc/ya.make deleted file mode 100644 index 35e68d60f8f..00000000000 --- a/yt/yt/library/tcmalloc/ya.make +++ /dev/null @@ -1,18 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - config.cpp - tcmalloc_manager.cpp - GLOBAL configure_tcmalloc_manager.cpp -) - -PEERDIR( - yt/yt/core - yt/yt/library/ytprof - yt/yt/library/profiling/resource_tracker - contrib/libs/tcmalloc/malloc_extension -) - -END() diff --git a/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.cpp b/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.cpp deleted file mode 100644 index a12bced15ef..00000000000 --- a/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include "allocation_tag_profiler.h" - -#ifdef __x86_64__ -#include <yt/yt/core/concurrency/periodic_executor.h> - -#include <yt/yt/core/ypath/token.h> - -#include <yt/yt/library/ytprof/heap_profiler.h> - -#include <tcmalloc/malloc_extension.h> -#endif - -namespace NYT::NYTProf { - -#ifdef __x86_64__ - -using namespace NProfiling; -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -class TAllocationTagProfiler - : public IAllocationTagProfiler -{ -public: - TAllocationTagProfiler( - std::vector<TAllocationTagKey> tagKeys, - IInvokerPtr invoker, - std::optional<TDuration> updatePeriod, - std::optional<i64> samplingRate, - NProfiling::TProfiler profiler) - : Profiler_(std::move(profiler)) - , TagKeys_(std::move(tagKeys)) - , UpdateExecutor_(New<TPeriodicExecutor>( - std::move(invoker), - BIND(&TAllocationTagProfiler::UpdateGauges, MakeWeak(this)), - updatePeriod)) - { - if (samplingRate) { - tcmalloc::MallocExtension::SetProfileSamplingRate(*samplingRate); - } - - UpdateExecutor_->Start(); - } - -private: - const NProfiling::TProfiler Profiler_; - const std::vector<TAllocationTagKey> TagKeys_; - - const NConcurrency::TPeriodicExecutorPtr UpdateExecutor_; - - THashMap<TAllocationTagKey, THashMap<TAllocationTagValue, NProfiling::TGauge>> Guages_; - - void UpdateGauges() - { - auto memorySnapshot = GetGlobalMemoryUsageSnapshot(); - - for (const auto& tagKey : TagKeys_) { - auto& guages = Guages_.emplace(tagKey, THashMap<TAllocationTagValue, TGauge>{}).first->second; - const auto& slice = memorySnapshot->GetUsageSlice(tagKey); - - for (auto& [tagValue, gauge] : guages) { - if (auto it = slice.find(tagValue)) { - gauge.Update(it->second); - } else { - gauge.Update(0.0); - } - } - - for (const auto& [tagValue, usage] : slice) { - auto it = guages.find(tagValue); - if (it == guages.end()) { - it = guages.emplace(tagValue, Profiler_ - .WithTag(tagKey, tagValue) - .Gauge(Format("/%v", NYPath::ToYPathLiteral(tagKey)))) - .first; - it->second.Update(usage); - } - } - } - } -}; - -#else - -class TAllocationTagProfiler - : public IAllocationTagProfiler -{ -public: - TAllocationTagProfiler(auto&&... /*args*/) - { } -}; - -#endif - -//////////////////////////////////////////////////////////////////////////////// - -IAllocationTagProfilerPtr CreateAllocationTagProfiler( - std::vector<TAllocationTagKey> tagKeys, - IInvokerPtr invoker, - std::optional<TDuration> updatePeriod, - std::optional<i64> samplingRate, - NYT::NProfiling::TProfiler profiler) -{ - return New<TAllocationTagProfiler>( - std::move(tagKeys), - std::move(invoker), - std::move(updatePeriod), - std::move(samplingRate), - std::move(profiler)); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.h b/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.h deleted file mode 100644 index 18c4c904d42..00000000000 --- a/yt/yt/library/ytprof/allocation_tag_profiler/allocation_tag_profiler.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/library/profiling/sensor.h> - -#include <yt/yt/core/actions/public.h> - -#include <yt/yt/core/concurrency/public.h> - -#include <yt/yt/core/tracing/public.h> - -#include <yt/yt/core/profiling/public.h> - -namespace NYT::NYTProf { - -//////////////////////////////////////////////////////////////////////////////// - -struct IAllocationTagProfiler - : public TRefCounted -{ }; - -DEFINE_REFCOUNTED_TYPE(IAllocationTagProfiler); - -//////////////////////////////////////////////////////////////////////////////// - -IAllocationTagProfilerPtr CreateAllocationTagProfiler( - std::vector<TAllocationTagKey> tagKeys, - IInvokerPtr invoker, - std::optional<TDuration> updatePeriod, - std::optional<i64> samplingRate, - NProfiling::TProfiler profiler = NProfiling::TProfiler{"/memory/heap_usage"}); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/allocation_tag_profiler/public.h b/yt/yt/library/ytprof/allocation_tag_profiler/public.h deleted file mode 100644 index 9bfbefabd33..00000000000 --- a/yt/yt/library/ytprof/allocation_tag_profiler/public.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include <library/cpp/yt/memory/ref_counted.h> - -namespace NYT::NYTProf { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(IAllocationTagProfiler); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/allocation_tag_profiler/ya.make b/yt/yt/library/ytprof/allocation_tag_profiler/ya.make deleted file mode 100644 index 583d7744610..00000000000 --- a/yt/yt/library/ytprof/allocation_tag_profiler/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - allocation_tag_profiler.cpp -) - -PEERDIR( - yt/yt/library/profiling - yt/yt/library/ytprof - yt/yt/core -) - -IF(ARCH_X86_64) - PEERDIR( - yt/yt/library/ytprof - ) -ENDIF() - -END() - - diff --git a/yt/yt/library/ytprof/bundle/ya.make b/yt/yt/library/ytprof/bundle/ya.make deleted file mode 100644 index 7f885834948..00000000000 --- a/yt/yt/library/ytprof/bundle/ya.make +++ /dev/null @@ -1,28 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -# Built with ya make -DNO_DEBUGINFO=yes -r --musl contrib/libs/llvm12/tools/llvm-symbolizer -FROM_SANDBOX( - FILE 2531143113 - OUT_NOAUTO llvm-symbolizer -) - -RESOURCE( - yt/yt/library/ytprof/bundle/llvm-symbolizer - /ytprof/llvm-symbolizer -) - -# Built with env CGO_ENABLED=0 ya tool go install github.com/google/pprof@latest -FROM_SANDBOX( - FILE 2531135322 - OUT_NOAUTO pprof -) - -RESOURCE( - yt/yt/library/ytprof/bundle/pprof - /ytprof/pprof -) - -END() - diff --git a/yt/yt/library/ytprof/example/main.cpp b/yt/yt/library/ytprof/example/main.cpp deleted file mode 100644 index d50bf2181cd..00000000000 --- a/yt/yt/library/ytprof/example/main.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include <yt/yt/core/concurrency/poller.h> -#include <yt/yt/core/concurrency/thread_pool_poller.h> -#include <yt/yt/core/concurrency/action_queue.h> -#include <yt/yt/core/concurrency/thread_pool.h> -#include <yt/yt/core/http/server.h> - -#include <yt/yt/library/ytprof/http/handler.h> - -#include <library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h> - -#include <tcmalloc/malloc_extension.h> - -using namespace NYT; -using namespace NYT::NHttp; -using namespace NYT::NConcurrency; -using namespace NYT::NYTProf; - -int main(int argc, char* argv[]) -{ - NBacktrace::SetAbslStackUnwinder(); - tcmalloc::MallocExtension::SetProfileSamplingRate(2_MB); - - try { - if (argc != 2 && argc != 3) { - throw yexception() << "usage: " << argv[0] << " PORT"; - } - - auto port = FromString<int>(argv[1]); - auto poller = CreateThreadPoolPoller(1, "Example"); - auto server = CreateServer(port, poller); - - Register(server, ""); - server->Start(); - - THashMap<TString, std::vector<int>> data; - for (int i = 0; i < 1024 * 16; i++) { - data[ToString(i)].resize(1024); - } - - auto burnCpu = [] { - ui64 value = 0; - while (true) { - THash<TString> hasher; - for (int i = 0; i < 10000000; i++) { - value += hasher(ToString(i)); - } - - std::vector<TString> data; - for (int i = 0; i < 10000; i++) { - data.push_back(TString(1024, 'x')); - } - - if (value == 1) { - Sleep(TDuration::Seconds(1)); - } - } - }; - - auto pool = CreateThreadPool(64, "Pool"); - for (int i = 0; i < 64; i++) { - pool->GetInvoker()->Invoke(BIND(burnCpu)); - } - - burnCpu(); - } catch (const std::exception& ex) { - Cerr << ex.what() << Endl; - _exit(1); - } - - return 0; -} diff --git a/yt/yt/library/ytprof/example/ya.make b/yt/yt/library/ytprof/example/ya.make deleted file mode 100644 index 1a762c401b4..00000000000 --- a/yt/yt/library/ytprof/example/ya.make +++ /dev/null @@ -1,20 +0,0 @@ -PROGRAM(ytprof-example) - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -IF (OS_LINUX) - ALLOCATOR(TCMALLOC_256K) -ENDIF() - -SRCS(main.cpp) - -IF (OS_LINUX) - LDFLAGS("-Wl,--build-id=sha1") -ENDIF() - -PEERDIR( - yt/yt/library/ytprof/http - library/cpp/yt/backtrace/absl_unwinder -) - -END() diff --git a/yt/yt/library/ytprof/http/handler.cpp b/yt/yt/library/ytprof/http/handler.cpp deleted file mode 100644 index be653276a78..00000000000 --- a/yt/yt/library/ytprof/http/handler.cpp +++ /dev/null @@ -1,320 +0,0 @@ -#include "handler.h" - -#include <yt/yt/core/concurrency/async_stream.h> - -#include <yt/yt/core/http/http.h> -#include <yt/yt/core/http/server.h> - -#include <yt/yt/library/ytprof/cpu_profiler.h> -#include <yt/yt/library/ytprof/spinlock_profiler.h> -#include <yt/yt/library/ytprof/heap_profiler.h> -#include <yt/yt/library/ytprof/profile.h> -#include <yt/yt/library/ytprof/symbolize.h> -#include <yt/yt/library/ytprof/external_pprof.h> - -#include <yt/yt/library/process/subprocess.h> - -#include <yt/yt/core/misc/finally.h> - -#include <library/cpp/cgiparam/cgiparam.h> - -#include <library/cpp/yt/threading/traceless_guard.h> - -#include <util/system/mutex.h> - -namespace NYT::NYTProf { - -using namespace NHttp; -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -class TBaseHandler - : public IHttpHandler -{ -public: - explicit TBaseHandler(const TBuildInfo& buildInfo) - : BuildInfo_(buildInfo) - { } - - virtual NProto::Profile BuildProfile(const TCgiParameters& params) = 0; - - void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override - { - try { - auto guard = NThreading::TracelessTryGuard(Lock_); - - if (!guard) { - rsp->SetStatus(EStatusCode::TooManyRequests); - WaitFor(rsp->WriteBody(TSharedRef::FromString("Profile fetch already running"))) - .ThrowOnError(); - return; - } - - TCgiParameters params(req->GetUrl().RawQuery); - auto profile = BuildProfile(params); - Symbolize(&profile, true); - AddBuildInfo(&profile, BuildInfo_); - - if (auto it = params.Find("symbolize"); it == params.end() || it->second != "0") { - SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ - .RunTool = RunSubprocess, - }); - } - - TStringStream profileBlob; - WriteCompressedProfile(&profileBlob, profile); - - rsp->SetStatus(EStatusCode::OK); - WaitFor(rsp->WriteBody(TSharedRef::FromString(profileBlob.Str()))) - .ThrowOnError(); - } catch (const std::exception& ex) { - if (rsp->AreHeadersFlushed()) { - throw; - } - - rsp->SetStatus(EStatusCode::InternalServerError); - WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what()))) - .ThrowOnError(); - - throw; - } - } - -protected: - const TBuildInfo BuildInfo_; - -private: - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_); -}; - -class TCpuProfilerHandler - : public TBaseHandler -{ -public: - using TBaseHandler::TBaseHandler; - - NProto::Profile BuildProfile(const TCgiParameters& params) override - { - auto duration = TDuration::Seconds(15); - if (auto it = params.Find("d"); it != params.end()) { - duration = TDuration::Parse(it->second); - } - - TCpuProfilerOptions options; - if (auto it = params.Find("freq"); it != params.end()) { - options.SamplingFrequency = FromString<int>(it->second); - } - - if (auto it = params.Find("record_action_run_time"); it != params.end()) { - options.RecordActionRunTime = true; - } - - if (auto it = params.Find("action_min_exec_time"); it != params.end()) { - options.SampleFilters.push_back(GetActionMinExecTimeFilter(TDuration::Parse(it->second))); - } - - TCpuProfiler profiler{options}; - profiler.Start(); - TDelayedExecutor::WaitForDuration(duration); - profiler.Stop(); - - return profiler.ReadProfile(); - } -}; - -class TSpinlockProfilerHandler - : public TBaseHandler -{ -public: - TSpinlockProfilerHandler(const TBuildInfo& buildInfo, bool yt) - : TBaseHandler(buildInfo) - , YT_(yt) - { } - - NProto::Profile BuildProfile(const TCgiParameters& params) override - { - auto duration = TDuration::Seconds(15); - if (auto it = params.Find("d"); it != params.end()) { - duration = TDuration::Parse(it->second); - } - - TSpinlockProfilerOptions options; - if (auto it = params.Find("frac"); it != params.end()) { - options.ProfileFraction = FromString<int>(it->second); - } - - if (YT_) { - TBlockingProfiler profiler{options}; - profiler.Start(); - TDelayedExecutor::WaitForDuration(duration); - profiler.Stop(); - - return profiler.ReadProfile(); - } else { - TSpinlockProfiler profiler{options}; - profiler.Start(); - TDelayedExecutor::WaitForDuration(duration); - profiler.Stop(); - - return profiler.ReadProfile(); - } - } - -private: - const bool YT_; -}; - -class TTCMallocSnapshotProfilerHandler - : public TBaseHandler -{ -public: - TTCMallocSnapshotProfilerHandler(const TBuildInfo& buildInfo, tcmalloc::ProfileType profileType) - : TBaseHandler(buildInfo) - , ProfileType_(profileType) - { } - - NProto::Profile BuildProfile(const TCgiParameters& /*params*/) override - { - return CaptureHeapProfile(ProfileType_); - } - -private: - const tcmalloc::ProfileType ProfileType_; -}; - -class TTCMallocAllocationProfilerHandler - : public TBaseHandler -{ -public: - using TBaseHandler::TBaseHandler; - - NProto::Profile BuildProfile(const TCgiParameters& params) override - { - auto duration = TDuration::Seconds(15); - if (auto it = params.Find("d"); it != params.end()) { - duration = TDuration::Parse(it->second); - } - - auto token = tcmalloc::MallocExtension::StartAllocationProfiling(); - TDelayedExecutor::WaitForDuration(duration); - return TCMallocProfileToProtoProfile(std::move(token).Stop()); - } -}; - -class TTCMallocStatHandler - : public IHttpHandler -{ -public: - void HandleRequest(const IRequestPtr& /*req*/, const IResponseWriterPtr& rsp) override - { - auto stat = tcmalloc::MallocExtension::GetStats(); - rsp->SetStatus(EStatusCode::OK); - WaitFor(rsp->WriteBody(TSharedRef::FromString(TString{stat}))) - .ThrowOnError(); - } -}; - -class TBinaryHandler - : public IHttpHandler -{ -public: - void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override - { - try { - auto buildId = GetBuildId(); - TCgiParameters params(req->GetUrl().RawQuery); - - if (auto it = params.Find("check_build_id"); it != params.end()) { - if (it->second != buildId) { - THROW_ERROR_EXCEPTION("Wrong build id: %v != %v", it->second, buildId); - } - } - - rsp->SetStatus(EStatusCode::OK); - - TFileInput file{"/proc/self/exe"}; - auto adapter = CreateBufferedSyncAdapter(rsp); - file.ReadAll(*adapter); - adapter->Finish(); - - WaitFor(rsp->Close()) - .ThrowOnError(); - } catch (const std::exception& ex) { - if (rsp->AreHeadersFlushed()) { - throw; - } - - rsp->SetStatus(EStatusCode::InternalServerError); - WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what()))) - .ThrowOnError(); - - throw; - } - } -}; - -class TVersionHandler - : public IHttpHandler -{ -public: - void HandleRequest(const IRequestPtr& /*req*/, const IResponseWriterPtr& rsp) override - { - rsp->SetStatus(EStatusCode::OK); - WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion()))) - .ThrowOnError(); - } -}; - -class TBuildIdHandler - : public IHttpHandler -{ -public: - void HandleRequest(const IRequestPtr& /*req*/, const IResponseWriterPtr& rsp) override - { - rsp->SetStatus(EStatusCode::OK); - WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion()))) - .ThrowOnError(); - } -}; - -void Register( - const NHttp::IServerPtr& server, - const TString& prefix, - const TBuildInfo& buildInfo) -{ - Register(server->GetPathMatcher(), prefix, buildInfo); -} - -void Register( - const IRequestPathMatcherPtr& handlers, - const TString& prefix, - const TBuildInfo& buildInfo) -{ - handlers->Add(prefix + "/cpu/profile", New<TCpuProfilerHandler>(buildInfo)); - - handlers->Add(prefix + "/spinlock/lock", New<TSpinlockProfilerHandler>(buildInfo, false)); - handlers->Add(prefix + "/spinlock/block", New<TSpinlockProfilerHandler>(buildInfo, true)); - - handlers->Add(prefix + "/tcmalloc/current", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kHeap)); - handlers->Add(prefix + "/tcmalloc/peak", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kPeakHeap)); - handlers->Add(prefix + "/tcmalloc/fragmentation", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kFragmentation)); - handlers->Add(prefix + "/tcmalloc/allocation", New<TTCMallocAllocationProfilerHandler>(buildInfo)); - handlers->Add(prefix + "/tcmalloc/stat", New<TTCMallocStatHandler>()); - - handlers->Add(prefix + "/binary", New<TBinaryHandler>()); - handlers->Add(prefix + "/build_id", New<TBuildIdHandler>()); - handlers->Add(prefix + "/version", New<TVersionHandler>()); - - // COMPAT(babenko): consider dropping these - handlers->Add(prefix + "/profile", New<TCpuProfilerHandler>(buildInfo)); - handlers->Add(prefix + "/buildid", New<TBuildIdHandler>()); - handlers->Add(prefix + "/heap", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kHeap)); - handlers->Add(prefix + "/peak", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kPeakHeap)); - handlers->Add(prefix + "/fragmentation", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kFragmentation)); - handlers->Add(prefix + "/allocations", New<TTCMallocAllocationProfilerHandler>(buildInfo)); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/http/handler.h b/yt/yt/library/ytprof/http/handler.h deleted file mode 100644 index fa96412d95f..00000000000 --- a/yt/yt/library/ytprof/http/handler.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include <yt/yt/core/http/public.h> - -#include <yt/yt/library/ytprof/build_info.h> - -namespace NYT::NYTProf { - -//////////////////////////////////////////////////////////////////////////////// - -//! Register profiling handlers. -void Register( - const NHttp::IServerPtr& server, - const TString& prefix, - const TBuildInfo& buildInfo = TBuildInfo::GetDefault()); - -void Register( - const NHttp::IRequestPathMatcherPtr& handlers, - const TString& prefix, - const TBuildInfo& buildInfo = TBuildInfo::GetDefault()); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/http/ya.make b/yt/yt/library/ytprof/http/ya.make deleted file mode 100644 index 1a1f3ff20c7..00000000000 --- a/yt/yt/library/ytprof/http/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - handler.cpp -) - -PEERDIR( - library/cpp/cgiparam - yt/yt/core/http - yt/yt/library/ytprof - yt/yt/library/process -) - -END() diff --git a/yt/yt/library/ytprof/integration/test_http.py b/yt/yt/library/ytprof/integration/test_http.py deleted file mode 100644 index 5d3b5ae17bd..00000000000 --- a/yt/yt/library/ytprof/integration/test_http.py +++ /dev/null @@ -1,114 +0,0 @@ -from collections import Counter -import pytest -import requests -import time - -import asyncio -import httpx - -import yatest.common -import yatest.common.network - - -TIMEOUT = 5000 - - [email protected](scope="session") -def running_example(): - with yatest.common.network.PortManager() as pm: - port = pm.get_port() - - cmd = [ - yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"), - str(port) - ] - - p = yatest.common.execute(cmd, wait=False, env={"YT_LOG_LEVEL": "DEBUG"}) - time.sleep(1) - assert p.running - - try: - yield {"port": port} - finally: - p.kill() - - -def fetch_data(running_example, name): - rsp = requests.get(f"http://localhost:{running_example['port']}/{name}") - if rsp.status_code == 200: - return rsp.content - - if rsp.status_code == 500: - raise Exception(rsp.text) - - rsp.raise_for_status() - - -def test_smoke_tcmalloc(running_example): - fetch_data(running_example, "heap") - fetch_data(running_example, "allocations?d=1") - fetch_data(running_example, "peak") - fetch_data(running_example, "fragmentation") - - -async def get_async(url): - async with httpx.AsyncClient() as client: - return await client.get(url, timeout=TIMEOUT) - - -async def launch(running_example, name): - url = f"http://localhost:{running_example['port']}/{name}" - - urls = [url, url, url] - - resps = await asyncio.gather(*map(get_async, urls)) - data = [resp.status_code for resp in resps] - - assert Counter(data) == Counter([200, 429, 429]) - - -def test_async(running_example): - fetch_data(running_example, "heap") - asyncio.run(launch(running_example, "heap")) - fetch_data(running_example, "heap") - - -def test_status_handlers(running_example): - assert fetch_data(running_example, "buildid") - assert fetch_data(running_example, "version") - - -def test_cpu_profile(running_example): - if yatest.common.context.build_type != "profile": - pytest.skip() - - fetch_data(running_example, "profile?d=1") - fetch_data(running_example, "profile?d=1&freq=1000") - - -def test_spinlock_profile(running_example): - if yatest.common.context.build_type != "profile": - pytest.skip() - - fetch_data(running_example, "spinlock/lock?d=1") - fetch_data(running_example, "spinlock/lock?d=1&frac=1") - - -def test_block_profile(running_example): - if yatest.common.context.build_type != "profile": - pytest.skip() - - fetch_data(running_example, "spinlock/block?d=1") - fetch_data(running_example, "spinlock/block?d=1&frac=1") - - -def test_binary_handler(running_example): - binary = fetch_data(running_example, "binary") - - with open(yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"), "rb") as f: - real_binary = f.read() - - assert binary == real_binary - - with pytest.raises(Exception): - fetch_data(running_example, "binary?check_build_id=1234") diff --git a/yt/yt/library/ytprof/integration/ya.make b/yt/yt/library/ytprof/integration/ya.make deleted file mode 100644 index 7ba00d743cd..00000000000 --- a/yt/yt/library/ytprof/integration/ya.make +++ /dev/null @@ -1,20 +0,0 @@ -PY3TEST() - -SIZE(MEDIUM) - -INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) - -PEERDIR( - contrib/python/requests - contrib/python/httpx -) - -TEST_SRCS( - test_http.py -) - -DEPENDS( - yt/yt/library/ytprof/example -) - -END() diff --git a/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp deleted file mode 100644 index 260adff5e05..00000000000 --- a/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp +++ /dev/null @@ -1,343 +0,0 @@ -#include <dlfcn.h> - -#include <gtest/gtest.h> - -#include <yt/yt/core/concurrency/action_queue.h> -#include <yt/yt/core/concurrency/scheduler_api.h> - -#include <yt/yt/core/actions/bind.h> - -#include <yt/yt/core/tracing/trace_context.h> - -#include <yt/yt/library/ytprof/cpu_profiler.h> -#include <yt/yt/library/ytprof/symbolize.h> -#include <yt/yt/library/ytprof/profile.h> -#include <yt/yt/library/ytprof/external_pprof.h> - -#include <library/cpp/testing/common/env.h> - -#include <library/cpp/yt/memory/new.h> - -#include <util/string/cast.h> -#include <util/stream/file.h> -#include <util/datetime/base.h> -#include <util/system/shellcommand.h> - -#include <yt/yt/core/concurrency/thread_pool.h> - -namespace NYT::NYTProf { -namespace { - -using namespace NConcurrency; -using namespace NTracing; - -using TSampleFilter = TCpuProfilerOptions::TSampleFilter; - -//////////////////////////////////////////////////////////////////////////////// - -template <size_t Index> -Y_NO_INLINE void BurnCpu() -{ - THash<TString> hasher; - ui64 value = 0; - for (int i = 0; i < 10000000; i++) { - value += hasher(ToString(i)); - } - EXPECT_NE(Index, value); -} - -static std::atomic<int> Counter{0}; - -struct TNoTailCall -{ - ~TNoTailCall() - { - Counter++; - } -}; - -static Y_NO_INLINE void StaticFunction() -{ - TNoTailCall noTail; - BurnCpu<0>(); -} - -void RunUnderProfiler( - const TString& name, - std::function<void()> work, - bool checkSamples = true, - const std::vector<TSampleFilter>& filters = {}, - bool expectEmpty = false) -{ - TCpuProfilerOptions options; - options.SampleFilters = filters; - options.SamplingFrequency = 100000; - options.RecordActionRunTime = true; - -#ifdef YTPROF_DEBUG_BUILD - options.SamplingFrequency = 100; -#endif - - TCpuProfiler profiler(options); - - profiler.Start(); - - work(); - - profiler.Stop(); - - auto profile = profiler.ReadProfile(); - if (checkSamples) { - ASSERT_EQ(expectEmpty, profile.sampleSize() == 0); - } - - Symbolize(&profile, true); - AddBuildInfo(&profile, TBuildInfo::GetDefault()); - SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ - .TmpDir = GetOutputPath(), - .KeepTmpDir = true, - .RunTool = [] (const std::vector<TString>& args) { - TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}}; - command.Run(); - - EXPECT_TRUE(command.GetExitCode() == 0) - << command.GetError(); - }, - }); - - TFileOutput output(GetOutputPath() / name); - WriteCompressedProfile(&output, profile); - output.Finish(); -} - -class TCpuProfilerTest - : public ::testing::Test -{ - void SetUp() override - { - if (!IsProfileBuild()) { - GTEST_SKIP() << "rebuild with --build=profile"; - } - } -}; - -TEST_F(TCpuProfilerTest, SingleThreadRun) -{ - RunUnderProfiler("single_thread.pb.gz", [] { - BurnCpu<0>(); - }); -} - -TEST_F(TCpuProfilerTest, MultipleThreads) -{ - RunUnderProfiler("multiple_threads.pb.gz", [] { - std::thread t1([] { - BurnCpu<1>(); - }); - - std::thread t2([] { - BurnCpu<2>(); - }); - - t1.join(); - t2.join(); - }); -} - -TEST_F(TCpuProfilerTest, StaticFunction) -{ - RunUnderProfiler("static_function.pb.gz", [] { - StaticFunction(); - }); -} - -Y_NO_INLINE void RecursiveFunction(int n) -{ - TNoTailCall noTail; - if (n == 0) { - BurnCpu<0>(); - } else { - RecursiveFunction(n-1); - } -} - -TEST_F(TCpuProfilerTest, DeepRecursion) -{ - RunUnderProfiler("recursive_function.pb.gz", [] { - RecursiveFunction(1024); - }); -} - -TEST_F(TCpuProfilerTest, DlOpen) -{ - RunUnderProfiler("dlopen.pb.gz", [] { - auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso/libtestso.so"); - - auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY); - ASSERT_TRUE(dl); - - auto sym = dlsym(dl, "CallNext"); - ASSERT_TRUE(sym); - - auto callNext = reinterpret_cast<void(*)(void(*)())>(sym); - callNext(BurnCpu<0>); - }); -} - -TEST_F(TCpuProfilerTest, DlClose) -{ - RunUnderProfiler("dlclose.pb.gz", [] { - auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso1/libtestso1.so"); - - auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY); - ASSERT_TRUE(dl); - - auto sym = dlsym(dl, "CallOtherNext"); - ASSERT_TRUE(sym); - - auto callNext = reinterpret_cast<void(*)(void(*)())>(sym); - callNext(BurnCpu<0>); - - ASSERT_EQ(dlclose(dl), 0); - }); -} - -void ReadUrandom() -{ - TIFStream input("/dev/urandom"); - - std::array<char, 1 << 20> buffer; - - for (int i = 0; i < 100; i++) { - input.Read(buffer.data(), buffer.size()); - } -} - -TEST_F(TCpuProfilerTest, Syscalls) -{ - RunUnderProfiler("syscalls.pb.gz", [] { - ReadUrandom(); - }); -} - -TEST_F(TCpuProfilerTest, VDSO) -{ - RunUnderProfiler("vdso.pb.gz", [] { - auto now = TInstant::Now(); - while (TInstant::Now() < now + TDuration::MilliSeconds(100)) - { } - }, false); -} - -TEST_F(TCpuProfilerTest, ProfilerTags) -{ - auto userTag = New<TProfilerTag>("user", "prime"); - auto intTag = New<TProfilerTag>("block_size", 1024); - - RunUnderProfiler("tags.pb.gz", [&] { - { - TCpuProfilerTagGuard guard(userTag); - BurnCpu<0>(); - } - { - TCpuProfilerTagGuard guard(intTag); - BurnCpu<1>(); - } - { - TCpuProfilerTagGuard guard(userTag); - TCpuProfilerTagGuard secondGuard(intTag); - BurnCpu<2>(); - } - }); -} - -TEST_F(TCpuProfilerTest, MultipleProfilers) -{ - TCpuProfiler profiler, secondProfiler; - - profiler.Start(); - EXPECT_THROW(secondProfiler.Start(), std::exception); -} - -TEST_F(TCpuProfilerTest, TraceContext) -{ - RunUnderProfiler("trace_context.pb.gz", [] { - auto actionQueue = New<TActionQueue>("CpuProfileTest"); - - BIND([] { - auto rootTraceContext = TTraceContext::NewRoot(""); - rootTraceContext->AddProfilingTag("user", "prime"); - TCurrentTraceContextGuard guard(rootTraceContext); - - auto asyncSubrequest = BIND([&] { - TChildTraceContextGuard guard(""); - AnnotateTraceContext([] (const auto traceContext) { - traceContext->AddProfilingTag("table", "//foo"); - }); - - BurnCpu<0>(); - }) - .AsyncVia(GetCurrentInvoker()) - .Run(); - - BurnCpu<1>(); - WaitFor(asyncSubrequest) - .ThrowOnError(); - }) - .AsyncVia(actionQueue->GetInvoker()) - .Run() - .Get(); - - actionQueue->Shutdown(); - }); -} - -TEST_F(TCpuProfilerTest, SlowActions) -{ - static const TString WorkerThreadName = "Heavy"; - static const auto TraceThreshold = TDuration::MilliSeconds(50); - - const std::vector<TSampleFilter> filters = { - GetActionMinExecTimeFilter(TraceThreshold), - }; - - auto busyWait = [] (TDuration duration) { - auto now = TInstant::Now(); - while (TInstant::Now() < now + duration) - { } - }; - - auto traceContext = NTracing::TTraceContext::NewRoot("Test"); - - const bool ExpectEmptyTraces = true; - - auto threadPool = CreateThreadPool(2, WorkerThreadName); - - // No slow actions. - RunUnderProfiler("slow_actions_empty.pb.gz", [&] { - NTracing::TTraceContextGuard guard(traceContext); - auto future = BIND(busyWait, TraceThreshold / 2) - .AsyncVia(threadPool->GetInvoker()) - .Run(); - future.Get(); - }, - true, - filters, - ExpectEmptyTraces); - - // Slow actions = non empty traces. - RunUnderProfiler("slow_actions.pb.gz", [&] { - NTracing::TTraceContextGuard guard(traceContext); - auto future = BIND(busyWait, TraceThreshold * 3) - .AsyncVia(threadPool->GetInvoker()) - .Run(); - future.Get(); - }, - true, - filters); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp deleted file mode 100644 index 872284e308c..00000000000 --- a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp +++ /dev/null @@ -1,227 +0,0 @@ -#include <gtest/gtest.h> - -#include <yt/yt/library/ytprof/heap_profiler.h> -#include <yt/yt/library/ytprof/symbolize.h> -#include <yt/yt/library/ytprof/profile.h> - -#include <yt/yt/core/actions/current_invoker.h> -#include <yt/yt/core/actions/invoker_detail.h> - -#include <yt/yt/core/concurrency/action_queue.h> - -#include <yt/yt/core/misc/lazy_ptr.h> - -#include <yt/yt/core/tracing/allocation_tags.h> -#include <yt/yt/core/tracing/trace_context.h> - -#include <library/cpp/testing/common/env.h> - -#include <library/cpp/yt/backtrace/absl_unwinder/absl_unwinder.h> - -#include <tcmalloc/malloc_extension.h> - -#include <util/string/cast.h> -#include <util/stream/file.h> -#include <util/generic/hash_set.h> -#include <util/datetime/base.h> -#include <util/generic/size_literals.h> - -namespace NYT::NYTProf { -namespace { - -using namespace NTracing; - -//////////////////////////////////////////////////////////////////////////////// - -const std::string MemoryAllocationTagKey = "memory_allocation_tag"; -const std::vector<std::string> MemoryAllocationTagValues = {"0", "1", "2", "3", "4", "5", "6", "7"}; - -//////////////////////////////////////////////////////////////////////////////// - -template <size_t Index> -Y_NO_INLINE auto BlowHeap() -{ - std::vector<std::string> data; - for (int i = 0; i < 10240; i++) { - data.push_back(std::string(1_KB, 'x')); - } - return data; -} - -TEST(THeapProfilerTest, ReadProfile) -{ - NBacktrace::SetAbslStackUnwinder(); - tcmalloc::MallocExtension::SetProfileSamplingRate(256_KB); - - auto token = tcmalloc::MallocExtension::StartAllocationProfiling(); - - EnableMemoryProfilingTags(); - - auto traceContext = TTraceContext::NewRoot("Root"); - TTraceContextGuard guard(traceContext); - - traceContext->SetAllocationTags({{"user", "first"}, {"sometag", "my"}}); - - auto h0 = BlowHeap<0>(); - - int tag = 1; - traceContext->SetAllocationTags({{"user", "second"}, {"sometag", "notmy"}, {MemoryAllocationTagKey, ToString(tag)}}); - auto currentTag = traceContext->FindAllocationTag<int>(MemoryAllocationTagKey); - ASSERT_EQ(currentTag, tag); - - auto h1 = BlowHeap<1>(); - - traceContext->SetAllocationTagList(nullptr); - - auto h2 = BlowHeap<2>(); - h2.clear(); - - auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTagKey, ToString(tag)); - ASSERT_GE(usage, 5_MB); - - auto dumpProfile = [] (auto name, auto type) { - auto profile = CaptureHeapProfile(type); - - TFileOutput output(GetOutputPath() / name); - WriteCompressedProfile(&output, profile); - output.Finish(); - }; - - dumpProfile("heap.pb.gz", tcmalloc::ProfileType::kHeap); - dumpProfile("peak.pb.gz", tcmalloc::ProfileType::kPeakHeap); - dumpProfile("fragmentation.pb.gz", tcmalloc::ProfileType::kFragmentation); - dumpProfile("allocations.pb.gz", tcmalloc::ProfileType::kAllocations); - - auto profile = std::move(token).Stop(); - - TFileOutput output(GetOutputPath() / "allocations.pb.gz"); - WriteCompressedProfile(&output, TCMallocProfileToProtoProfile(profile)); - output.Finish(); -} - -TEST(THeapProfilerTest, AllocationTags) -{ - EnableMemoryProfilingTags(); - auto traceContext = TTraceContext::NewRoot("Root"); - TTraceContextGuard guard(traceContext); - - ASSERT_EQ(traceContext->FindAllocationTag<std::string>(MemoryAllocationTagKey), std::nullopt); - traceContext->SetAllocationTags({{"user", "first user"}, {MemoryAllocationTagKey, MemoryAllocationTagValues[0]}}); - ASSERT_EQ(traceContext->FindAllocationTag<std::string>("user"), "first user"); - ASSERT_EQ(traceContext->FindAllocationTag<std::string>(MemoryAllocationTagKey), MemoryAllocationTagValues[0]); - - std::vector<std::vector<std::string>> heap; - heap.push_back(BlowHeap<0>()); - - traceContext->SetAllocationTags({{"user", "second user"}, {MemoryAllocationTagKey, MemoryAllocationTagValues[1]}}); - ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTagKey), 1); - - heap.push_back(BlowHeap<1>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[0]); - - auto usage1 = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTagKey, MemoryAllocationTagValues[1]); - - ASSERT_NEAR(usage1, 12_MB, 8_MB); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[2]); - ASSERT_EQ(traceContext->FindAllocationTag<std::string>(MemoryAllocationTagKey), MemoryAllocationTagValues[2]); - - { - volatile auto h = BlowHeap<2>(); - } - - traceContext->SetAllocationTagList(nullptr); - ASSERT_EQ(traceContext->FindAllocationTag<std::string>(MemoryAllocationTagKey), std::nullopt); - - heap.push_back(BlowHeap<0>()); - - { - auto slice = CollectMemoryUsageSnapshot()->GetUsageSlice(MemoryAllocationTagKey); - ASSERT_EQ(slice[MemoryAllocationTagValues[1]], usage1); - ASSERT_LE(slice[MemoryAllocationTagValues[2]], 1_MB); - } - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[6]); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[3]); - heap.push_back(BlowHeap<3>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[4]); - heap.push_back(BlowHeap<4>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[7]); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[5]); - heap.push_back(BlowHeap<5>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[4]); - heap.push_back(BlowHeap<4>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[7]); - - traceContext->SetAllocationTagList(nullptr); - - auto slice = CollectMemoryUsageSnapshot()->GetUsageSlice(MemoryAllocationTagKey); - - constexpr auto maxDifference = 10_MB; - ASSERT_NEAR(slice[MemoryAllocationTagValues[1]], slice[MemoryAllocationTagValues[3]], maxDifference); - ASSERT_NEAR(slice[MemoryAllocationTagValues[3]], slice[MemoryAllocationTagValues[5]], maxDifference); - ASSERT_NEAR(slice[MemoryAllocationTagValues[1]], slice[MemoryAllocationTagValues[5]], maxDifference); - - ASSERT_NEAR(slice[MemoryAllocationTagValues[4]], 20_MB, 15_MB); - - ASSERT_NEAR(slice[MemoryAllocationTagValues[4]], slice[MemoryAllocationTagValues[1]] + slice[MemoryAllocationTagValues[3]], 2 * maxDifference); - ASSERT_NEAR(slice[MemoryAllocationTagValues[4]], slice[MemoryAllocationTagValues[1]] + slice[MemoryAllocationTagValues[5]], 2 * maxDifference); - ASSERT_NEAR(slice[MemoryAllocationTagValues[4]], slice[MemoryAllocationTagValues[3]] + slice[MemoryAllocationTagValues[5]], 2 * maxDifference); - - ASSERT_LE(slice[MemoryAllocationTagValues[6]], 1_MB); - ASSERT_LE(slice[MemoryAllocationTagValues[7]], 1_MB); -} - -template <size_t Index> -Y_NO_INLINE auto BlowHeap(int64_t megabytes) -{ - std::vector<std::string> data; - megabytes <<= 10; - for (int64_t i = 0; i < megabytes; i++) { - data.push_back(std::string(1_KB, 'x')); - } - return data; -} - -TEST(THeapProfilerTest, HugeAllocationsTagsWithMemoryTag) -{ - EnableMemoryProfilingTags(); - auto traceContext = TTraceContext::NewRoot("Root"); - TCurrentTraceContextGuard guard(traceContext); - - std::vector<std::vector<std::string>> heap; - - heap.push_back(BlowHeap<0>()); - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[1]); - ASSERT_EQ(traceContext->FindAllocationTag<int>(MemoryAllocationTagKey), 1); - - heap.push_back(BlowHeap<1>(100)); - - { - traceContext->SetAllocationTagList(nullptr); - auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTagKey, MemoryAllocationTagValues[1]); - ASSERT_GE(usage, 100_MB); - ASSERT_LE(usage, 150_MB); - } - - traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[2]); - heap.push_back(BlowHeap<1>(1000)); - - traceContext->SetAllocationTagList(nullptr); - auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTagKey, MemoryAllocationTagValues[2]); - ASSERT_GE(usage, 1000_MB); - ASSERT_LE(usage, 1300_MB); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/queue_ut.cpp b/yt/yt/library/ytprof/unittests/queue_ut.cpp deleted file mode 100644 index f44ae8da32d..00000000000 --- a/yt/yt/library/ytprof/unittests/queue_ut.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include <gtest/gtest.h> - -#include <yt/yt/library/ytprof/queue.h> - -namespace NYT::NYTProf { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -TEST(StaticQueue, PushPop) -{ - TStaticQueue queue(10); - - int a, b, c; - void* aptr = reinterpret_cast<void*>(&a); - void* bptr = reinterpret_cast<void*>(&b); - void* cptr = reinterpret_cast<void*>(&c); - - std::vector<void*> backtrace; - auto getBacktrace = [&] () -> std::pair<void*, bool> { - if (backtrace.empty()) { - return {nullptr, false}; - } - - auto ip = backtrace.front(); - backtrace.erase(backtrace.begin()); - return {ip, true}; - }; - - ASSERT_TRUE(queue.TryPush(getBacktrace)); - - backtrace = {aptr, bptr, cptr}; - ASSERT_TRUE(queue.TryPush(getBacktrace)); - - backtrace.push_back(aptr); - ASSERT_TRUE(queue.TryPush(getBacktrace)); - - auto readBacktrace = [&] (void *ip) { - backtrace.push_back(ip); - }; - - ASSERT_TRUE(queue.TryPop(readBacktrace)); - ASSERT_EQ(backtrace, std::vector<void*>{}); - - backtrace.clear(); - ASSERT_TRUE(queue.TryPop(readBacktrace)); - ASSERT_EQ(backtrace, (std::vector<void*>{aptr, bptr, cptr})); - - backtrace.clear(); - ASSERT_TRUE(queue.TryPop(readBacktrace)); - ASSERT_EQ(backtrace, (std::vector<void*>{aptr})); - - ASSERT_FALSE(queue.TryPop(readBacktrace)); -} - -TEST(StaticQueue, Overflow) -{ - TStaticQueue queue(10); - - ASSERT_FALSE(queue.TryPush([] () -> std::pair<void*, bool> { - return {nullptr, true}; - })); -} - -} // namespace -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp deleted file mode 100644 index 39e60eb67d0..00000000000 --- a/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include <gtest/gtest.h> - -#include <yt/yt/library/ytprof/spinlock_profiler.h> -#include <yt/yt/library/ytprof/symbolize.h> -#include <yt/yt/library/ytprof/profile.h> -#include <yt/yt/library/ytprof/external_pprof.h> - -#include <tcmalloc/malloc_extension.h> - -#include <library/cpp/testing/common/env.h> - -#include <library/cpp/yt/threading/spin_lock.h> - -#include <util/string/cast.h> -#include <util/stream/file.h> -#include <util/datetime/base.h> -#include <util/system/compiler.h> -#include <util/system/shellcommand.h> -#include <util/thread/lfstack.h> -#include <util/generic/size_literals.h> - -namespace NYT::NYTProf { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -template <class TProfiler> -void RunUnderProfiler(const TString& name, std::function<void()> work, bool checkSamples = true) -{ - TSpinlockProfilerOptions options; - options.ProfileFraction = 10; - - TProfiler profiler(options); - - profiler.Start(); - - work(); - - profiler.Stop(); - - auto profile = profiler.ReadProfile(); - if (checkSamples) { - ASSERT_NE(0, profile.sample_size()); - } - - Symbolize(&profile, true); - AddBuildInfo(&profile, TBuildInfo::GetDefault()); - SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ - .TmpDir = GetOutputPath(), - .KeepTmpDir = true, - .RunTool = [] (const std::vector<TString>& args) { - TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}}; - command.Run(); - - EXPECT_TRUE(command.GetExitCode() == 0) - << command.GetError(); - }, - }); - - TFileOutput output(GetOutputPath() / name); - WriteCompressedProfile(&output, profile); - output.Finish(); -} - -class TSpinlockProfilerTest - : public ::testing::Test -{ - void SetUp() override - { - if (!IsProfileBuild()) { - GTEST_SKIP() << "rebuild with --build=profile"; - } - } -}; - -TEST_F(TSpinlockProfilerTest, PageHeapLock) -{ - RunUnderProfiler<TSpinlockProfiler>("pageheap_lock.pb.gz", [] { - std::atomic<bool> Stop = false; - - std::thread release([&] { - while (!Stop) { - tcmalloc::MallocExtension::ReleaseMemoryToSystem(1_GB); - } - }); - - std::vector<std::thread> allocators; - for (int i = 0; i < 16; i++) { - allocators.emplace_back([&] { - while (!Stop) { - auto ptr = malloc(4_MB); - DoNotOptimizeAway(ptr); - free(ptr); - } - }); - } - - Sleep(TDuration::Seconds(5)); - Stop = true; - - release.join(); - for (auto& t : allocators) { - t.join(); - } - }); -} - - -TEST_F(TSpinlockProfilerTest, TransferCacheLock) -{ - RunUnderProfiler<TSpinlockProfiler>("transfer_cache_lock.pb.gz", [] { - std::atomic<bool> Stop = false; - - TLockFreeStack<int> stack; - std::thread producer([&] { - while (!Stop) { - stack.Enqueue(1); - } - }); - - std::thread consumer([&] { - while (!Stop) { - int value; - stack.Dequeue(&value); - } - }); - - Sleep(TDuration::Seconds(5)); - - Stop = true; - producer.join(); - consumer.join(); - }); -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST_F(TSpinlockProfilerTest, YTLocks) -{ - RunUnderProfiler<TBlockingProfiler>("ytlock.pb.gz", [] { - std::atomic<bool> Stop = false; - - NThreading::TSpinLock lock; - std::thread slow([&] { - while (!Stop) { - lock.Acquire(); - Sleep(TDuration::MilliSeconds(10)); - lock.Release(); - Sleep(TDuration::MilliSeconds(10)); - } - }); - - std::thread fast([&] { - while (!Stop) { - lock.Acquire(); - lock.Release(); - } - }); - - Sleep(TDuration::Seconds(5)); - - Stop = true; - slow.join(); - fast.join(); - }); -} - -//////////////////////////////////////////////////////////////////////////////// - - -} // namespace -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp b/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp deleted file mode 100644 index 646ecaafad6..00000000000 --- a/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include <gtest/gtest.h> - -#include <yt/yt/library/ytprof/symbolize.h> -#include <yt/yt/library/ytprof/build_info.h> - -namespace NYT::NYTProf { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -Y_NO_INLINE void* GetIP() -{ - return __builtin_return_address(0); -} - -TEST(Symbolize, EmptyProfile) -{ - NProto::Profile profile; - profile.add_string_table(); - - Symbolize(&profile); - AddBuildInfo(&profile, TBuildInfo::GetDefault()); -} - -TEST(Symbolize, SingleLocation) -{ - NProto::Profile profile; - profile.add_string_table(); - - auto thisIP = GetIP(); - - { - auto location = profile.add_location(); - location->set_address(reinterpret_cast<ui64>(thisIP)); - - auto line = location->add_line(); - line->set_function_id(reinterpret_cast<ui64>(thisIP)); - - auto function = profile.add_function(); - function->set_id(reinterpret_cast<ui64>(thisIP)); - } - - Symbolize(&profile); - - ASSERT_EQ(1, profile.function_size()); - auto function = profile.function(0); - - auto name = profile.string_table(function.name()); - ASSERT_TRUE(name.find("SingleLocation") != TString::npos) - << "function name is " << name; -} - -TEST(Symbolize, GetBuildId) -{ - if (!IsProfileBuild()) { - GTEST_SKIP(); - } - - return; - - auto buildId = GetBuildId(); - ASSERT_TRUE(buildId); - ASSERT_NE(*buildId, TString{""}); -} - -TEST(BuildInfo, Test) -{ - if (!IsProfileBuild()) { - GTEST_SKIP(); - } - - auto info = TBuildInfo::GetDefault(); - if (IsProfileBuild()) { - ASSERT_EQ(info.BuildType, "profile"); - } - - ASSERT_NE(info.ArcRevision, ""); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/testso/testso.cpp b/yt/yt/library/ytprof/unittests/testso/testso.cpp deleted file mode 100644 index 4209d5f6cca..00000000000 --- a/yt/yt/library/ytprof/unittests/testso/testso.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include <atomic> - -static std::atomic<int> CallCount; - -extern "C" void CallNext(void (*next)()) -{ - next(); - CallCount++; -} diff --git a/yt/yt/library/ytprof/unittests/testso/ya.make b/yt/yt/library/ytprof/unittests/testso/ya.make deleted file mode 100644 index 8851d8eed63..00000000000 --- a/yt/yt/library/ytprof/unittests/testso/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -DLL(testso) - -SRCS( - testso.cpp -) - -END() diff --git a/yt/yt/library/ytprof/unittests/testso1/testso.cpp b/yt/yt/library/ytprof/unittests/testso1/testso.cpp deleted file mode 100644 index 56e348c0201..00000000000 --- a/yt/yt/library/ytprof/unittests/testso1/testso.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include <atomic> - -static std::atomic<int> CallCount; - -extern "C" void CallOtherNext(void (*next)()) -{ - next(); - CallCount++; -} diff --git a/yt/yt/library/ytprof/unittests/testso1/ya.make b/yt/yt/library/ytprof/unittests/testso1/ya.make deleted file mode 100644 index d0d2f901439..00000000000 --- a/yt/yt/library/ytprof/unittests/testso1/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -DLL(testso1) - -SRCS( - testso.cpp -) - -END() diff --git a/yt/yt/library/ytprof/unittests/ya.make b/yt/yt/library/ytprof/unittests/ya.make deleted file mode 100644 index 3514382c20f..00000000000 --- a/yt/yt/library/ytprof/unittests/ya.make +++ /dev/null @@ -1,44 +0,0 @@ -GTEST() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - symbolizer_ut.cpp - cpu_profiler_ut.cpp - heap_profiler_ut.cpp - spinlock_profiler_ut.cpp - queue_ut.cpp -) - -INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) - -PEERDIR( - yt/yt/library/ytprof - yt/yt/library/profiling - yt/yt/core - library/cpp/yt/backtrace/absl_unwinder -) - -IF (OS_LINUX) - LDFLAGS("-Wl,--build-id=sha1") -ENDIF() - -DEPENDS( - yt/yt/library/ytprof/unittests/testso - yt/yt/library/ytprof/unittests/testso1 -) - -IF (BUILD_TYPE != "release" AND BUILD_TYPE != "relwithdebinfo") - CFLAGS(-DYTPROF_DEBUG_BUILD) -ENDIF() - -SIZE(MEDIUM) - -ALLOCATOR(TCMALLOC_256K) - -END() - -RECURSE( - testso - testso1 -) |