diff options
author | Alexander Smirnov <alex@ydb.tech> | 2025-01-14 05:03:04 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2025-01-14 05:03:04 +0000 |
commit | 6661afff40562c23e888fe58ea6636ebae7130ef (patch) | |
tree | c85124e3e4f9b2b7170675085e450aa575bb0226 | |
parent | fc474d1ba278418a10604d48266980c0f290f24c (diff) | |
parent | 77108821cdaa9cad85b2b37223cf998f6dcccf46 (diff) | |
download | ydb-6661afff40562c23e888fe58ea6636ebae7130ef.tar.gz |
Merge branch 'rightlib' into merge-libs-250114-0501
306 files changed, 11878 insertions, 31674 deletions
diff --git a/build/external_resources/gdb/a.yaml b/build/external_resources/gdb/a.yaml index bd74f6c2043..050c7f48961 100644 --- a/build/external_resources/gdb/a.yaml +++ b/build/external_resources/gdb/a.yaml @@ -47,7 +47,7 @@ ci: jobs: build: title: Build - task: projects/devtools/contrib/toolchain_registry/build + task: projects/devtools/toolchain_registry/build input: <<: *base-input revision: ${flow-vars.revision} @@ -69,7 +69,7 @@ ci: release: title: Release - task: projects/devtools/contrib/toolchain_registry/release + task: projects/devtools/toolchain_registry/release needs: update-mapping input: <<: *base-input diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 1ef4b391df6..f8921a6186c 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7714082087" + "uri": "sbr:7805549382" }, "darwin-arm64": { - "uri": "sbr:7714081769" + "uri": "sbr:7805547756" }, "linux": { - "uri": "sbr:7714082818" + "uri": "sbr:7805552802" }, "linux-aarch64": { - "uri": "sbr:7714081292" + "uri": "sbr:7805545270" }, "win32-clang-cl": { - "uri": "sbr:7714082412" + "uri": "sbr:7805551244" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index c958e7d1960..722c9caf8f0 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7714069562" + "uri": "sbr:7805523567" }, "darwin-arm64": { - "uri": "sbr:7714068983" + "uri": "sbr:7805523059" }, "linux": { - "uri": "sbr:7714070680" + "uri": "sbr:7805525384" }, "linux-aarch64": { - "uri": "sbr:7714068462" + "uri": "sbr:7805522645" }, "win32-clang-cl": { - "uri": "sbr:7714070213" + "uri": "sbr:7805524504" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index caee26bb248..d38da1796c6 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -480,6 +480,8 @@ "7717387253": "https://devtools-registry.s3.yandex.net/7717387253", "7750415386": "https://devtools-registry.s3.yandex.net/7750415386", "7750425697": "https://devtools-registry.s3.yandex.net/7750425697", + "7808687279": "https://devtools-registry.s3.yandex.net/7808687279", + "7808709138": "https://devtools-registry.s3.yandex.net/7808709138", "5486731632": "https://devtools-registry.s3.yandex.net/5486731632", "5514350352": "https://devtools-registry.s3.yandex.net/5514350352", "5514360398": "https://devtools-registry.s3.yandex.net/5514360398", @@ -664,6 +666,7 @@ "7648122773": "https://devtools-registry.s3.yandex.net/7648122773", "7666559958": "https://devtools-registry.s3.yandex.net/7666559958", "7714082087": "https://devtools-registry.s3.yandex.net/7714082087", + "7805549382": "https://devtools-registry.s3.yandex.net/7805549382", "5766171800": "https://devtools-registry.s3.yandex.net/5766171800", "5805430761": "https://devtools-registry.s3.yandex.net/5805430761", "5829025456": "https://devtools-registry.s3.yandex.net/5829025456", @@ -723,6 +726,7 @@ "7648121342": "https://devtools-registry.s3.yandex.net/7648121342", "7666558962": "https://devtools-registry.s3.yandex.net/7666558962", "7714081769": "https://devtools-registry.s3.yandex.net/7714081769", + "7805547756": "https://devtools-registry.s3.yandex.net/7805547756", "5766173070": "https://devtools-registry.s3.yandex.net/5766173070", "5805432830": "https://devtools-registry.s3.yandex.net/5805432830", "5829031598": "https://devtools-registry.s3.yandex.net/5829031598", @@ -782,6 +786,7 @@ "7648125070": "https://devtools-registry.s3.yandex.net/7648125070", "7666562407": "https://devtools-registry.s3.yandex.net/7666562407", "7714082818": "https://devtools-registry.s3.yandex.net/7714082818", + "7805552802": "https://devtools-registry.s3.yandex.net/7805552802", "5766171341": "https://devtools-registry.s3.yandex.net/5766171341", "5805430188": "https://devtools-registry.s3.yandex.net/5805430188", "5829023352": "https://devtools-registry.s3.yandex.net/5829023352", @@ -841,6 +846,7 @@ "7648120215": "https://devtools-registry.s3.yandex.net/7648120215", "7666556873": "https://devtools-registry.s3.yandex.net/7666556873", "7714081292": "https://devtools-registry.s3.yandex.net/7714081292", + "7805545270": "https://devtools-registry.s3.yandex.net/7805545270", "5766172695": "https://devtools-registry.s3.yandex.net/5766172695", "5805432230": "https://devtools-registry.s3.yandex.net/5805432230", "5829029743": "https://devtools-registry.s3.yandex.net/5829029743", @@ -900,6 +906,7 @@ "7648124003": "https://devtools-registry.s3.yandex.net/7648124003", "7666560929": "https://devtools-registry.s3.yandex.net/7666560929", "7714082412": "https://devtools-registry.s3.yandex.net/7714082412", + "7805551244": "https://devtools-registry.s3.yandex.net/7805551244", "4307890075": "https://devtools-registry.s3.yandex.net/4307890075", "5517245192": "https://devtools-registry.s3.yandex.net/5517245192", "4307901240": "https://devtools-registry.s3.yandex.net/4307901240", @@ -1635,6 +1642,8 @@ "7717387253": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "7750415386": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "7750425697": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "7808687279": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "7808709138": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -1819,6 +1828,7 @@ "7648122773": "devtools/ymake/bin/ymake for darwin", "7666559958": "devtools/ymake/bin/ymake for darwin", "7714082087": "devtools/ymake/bin/ymake for darwin", + "7805549382": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -1878,6 +1888,7 @@ "7648121342": "devtools/ymake/bin/ymake for darwin-arm64", "7666558962": "devtools/ymake/bin/ymake for darwin-arm64", "7714081769": "devtools/ymake/bin/ymake for darwin-arm64", + "7805547756": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -1937,6 +1948,7 @@ "7648125070": "devtools/ymake/bin/ymake for linux", "7666562407": "devtools/ymake/bin/ymake for linux", "7714082818": "devtools/ymake/bin/ymake for linux", + "7805552802": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -1996,6 +2008,7 @@ "7648120215": "devtools/ymake/bin/ymake for linux-aarch64", "7666556873": "devtools/ymake/bin/ymake for linux-aarch64", "7714081292": "devtools/ymake/bin/ymake for linux-aarch64", + "7805545270": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -2055,6 +2068,7 @@ "7648124003": "devtools/ymake/bin/ymake for win32-clang-cl", "7666560929": "devtools/ymake/bin/ymake for win32-clang-cl", "7714082412": "devtools/ymake/bin/ymake for win32-clang-cl", + "7805551244": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 36a9e3bd356..bcdce412a73 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750424052) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808708280) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750422985) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808707972) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750425697) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808709138) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750421881) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808707633) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750424913) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808708809) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index cd7c6be19ab..3813fb01915 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750413507) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808686430) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750412435) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808685838) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750415386) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808687279) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750411520) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808685532) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7750414496) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7808686875) ENDIF() diff --git a/build/sysincl/macro.yml b/build/sysincl/macro.yml index e8481561c0b..ffdea0e4ad0 100644 --- a/build/sysincl/macro.yml +++ b/build/sysincl/macro.yml @@ -1128,8 +1128,6 @@ - THRUST_MR_STD_MR_HEADER: # this macro could be resolved to <memory_resource> or <experimental/memory_resource> of stl - contrib/libs/cxxsupp/libcxx/include/memory_resource - - contrib/libs/cxxsupp/libcxxcuda11/include/memory_resource - - contrib/libs/cxxsupp/libcxxcuda11/include/experimental/memory_resource - source_filter: "^contrib/libs/unixodbc/libltdl" includes: diff --git a/build/sysincl/stl-to-libcxxcuda11.yml b/build/sysincl/stl-to-libcxxcuda11.yml index d763cbb704b..2860ef276ef 100644 --- a/build/sysincl/stl-to-libcxxcuda11.yml +++ b/build/sysincl/stl-to-libcxxcuda11.yml @@ -57,6 +57,12 @@ - experimental/system_error: DO_NOT_INCLUDE_NON_STANDARD_EXPERIMENTAL_SYSTEM_ERROR - experimental/tuple: DO_NOT_INCLUDE_NON_STANDARD_EXPERIMENTAL_TUPLE +- includes: + - THRUST_MR_STD_MR_HEADER: + # this macro could be resolved to <memory_resource> or <experimental/memory_resource> of stl + - contrib/libs/cxxsupp/libcxxcuda11/include/memory_resource + - contrib/libs/cxxsupp/libcxxcuda11/include/experimental/memory_resource + # This includes all headers needed to resolve includes in c-headers from libcxx listed above # GENERATED BY YM2 diff --git a/contrib/libs/apache/orc/c++/src/Reader.cc b/contrib/libs/apache/orc/c++/src/Reader.cc index 8f96914b947..82e77e47059 100644 --- a/contrib/libs/apache/orc/c++/src/Reader.cc +++ b/contrib/libs/apache/orc/c++/src/Reader.cc @@ -691,7 +691,7 @@ namespace orc { std::string ReaderImpl::getMetadataValue(const std::string& key) const { for (int i = 0; i < footer->metadata_size(); ++i) { - if (footer->metadata(i).name() == TProtobufString(key)) { + if (footer->metadata(i).name() == key) { return footer->metadata(i).value(); } } @@ -741,7 +741,7 @@ namespace orc { bool ReaderImpl::hasMetadataValue(const std::string& key) const { for (int i = 0; i < footer->metadata_size(); ++i) { - if (footer->metadata(i).name() == TProtobufString(key)) { + if (footer->metadata(i).name() == key) { return true; } } @@ -1367,7 +1367,7 @@ namespace orc { if (serializedFooter.length() != 0) { // Parse the file tail from the serialized one. proto::FileTail tail; - if (!tail.ParseFromString(TProtobufString(serializedFooter))) { + if (!tail.ParseFromString(serializedFooter)) { throw ParseError("Failed to parse the file tail from string"); } contents->postscript = std::make_unique<proto::PostScript>(tail.postscript()); diff --git a/contrib/libs/apache/orc/c++/src/Statistics.hh b/contrib/libs/apache/orc/c++/src/Statistics.hh index 94e19417c3a..e585bf971cc 100644 --- a/contrib/libs/apache/orc/c++/src/Statistics.hh +++ b/contrib/libs/apache/orc/c++/src/Statistics.hh @@ -711,14 +711,14 @@ namespace orc { proto::DecimalStatistics* decStats = pbStats.mutable_decimal_statistics(); if (_stats.hasMinimum()) { - decStats->set_minimum(TProtobufString(_stats.getMinimum().toString(true))); - decStats->set_maximum(TProtobufString(_stats.getMaximum().toString(true))); + decStats->set_minimum(_stats.getMinimum().toString(true)); + decStats->set_maximum(_stats.getMaximum().toString(true)); } else { decStats->clear_minimum(); decStats->clear_maximum(); } if (_stats.hasSum()) { - decStats->set_sum(TProtobufString(_stats.getSum().toString(true))); + decStats->set_sum(_stats.getSum().toString(true)); } else { decStats->clear_sum(); } @@ -1225,8 +1225,8 @@ namespace orc { proto::StringStatistics* strStats = pbStats.mutable_string_statistics(); if (_stats.hasMinimum()) { - strStats->set_minimum(TProtobufString(_stats.getMinimum())); - strStats->set_maximum(TProtobufString(_stats.getMaximum())); + strStats->set_minimum(_stats.getMinimum()); + strStats->set_maximum(_stats.getMaximum()); } else { strStats->clear_minimum(); strStats->clear_maximum(); diff --git a/contrib/libs/apache/orc/c++/src/Writer.cc b/contrib/libs/apache/orc/c++/src/Writer.cc index 8a8044f0291..89eb3781cfa 100644 --- a/contrib/libs/apache/orc/c++/src/Writer.cc +++ b/contrib/libs/apache/orc/c++/src/Writer.cc @@ -429,8 +429,8 @@ namespace orc { void WriterImpl::addUserMetadata(const std::string& name, const std::string& value) { proto::UserMetadataItem* userMetadataItem = fileFooter.add_metadata(); - userMetadataItem->set_name(TProtobufString(name)); - userMetadataItem->set_value(TProtobufString(value)); + userMetadataItem->set_name(name); + userMetadataItem->set_value(value); } void WriterImpl::init() { @@ -510,7 +510,7 @@ namespace orc { *stripeFooter.add_columns() = encodings[i]; } - stripeFooter.set_writer_timezone(TProtobufString(options.getTimezoneName())); + stripeFooter.set_writer_timezone(options.getTimezoneName()); // add stripe statistics to metadata proto::StripeStatistics* stripeStats = metadata.add_stripe_stats(); @@ -679,8 +679,8 @@ namespace orc { for (auto& key : t.getAttributeKeys()) { const auto& value = t.getAttributeValue(key); auto protoAttr = protoType.add_attributes(); - protoAttr->set_key(TProtobufString(key)); - protoAttr->set_value(TProtobufString(value)); + protoAttr->set_key(key); + protoAttr->set_value(value); } int pos = static_cast<int>(index); @@ -689,7 +689,7 @@ namespace orc { for (uint64_t i = 0; i < t.getSubtypeCount(); ++i) { // only add subtypes' field names if this type is STRUCT if (t.getKind() == STRUCT) { - footer.mutable_types(pos)->add_field_names(TProtobufString(t.getFieldName(i))); + footer.mutable_types(pos)->add_field_names(t.getFieldName(i)); } footer.mutable_types(pos)->add_subtypes(++index); buildFooterType(*t.getSubtype(i), footer, index); diff --git a/contrib/libs/jinja2cpp/src/error_info.cpp b/contrib/libs/jinja2cpp/src/error_info.cpp index 25ff5dfee70..a5125e95a96 100644 --- a/contrib/libs/jinja2cpp/src/error_info.cpp +++ b/contrib/libs/jinja2cpp/src/error_info.cpp @@ -126,53 +126,53 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT> switch (errCode) { case ErrorCode::Unspecified: - format_to(std::back_inserter(out), UNIVERSAL_STR("Parse error").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Parse error").GetValue<CharT>()); break; case ErrorCode::UnexpectedException: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected exception occurred during template processing. Exception: {}").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected exception occurred during template processing. Exception: {}").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::MetadataParseError: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Error occurred during template metadata parsing. Error: {}").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Error occurred during template metadata parsing. Error: {}").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::YetUnsupported: - format_to(std::back_inserter(out), UNIVERSAL_STR("This feature has not been supported yet").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("This feature has not been supported yet").GetValue<CharT>()); break; case ErrorCode::FileNotFound: - format_to(std::back_inserter(out), UNIVERSAL_STR("File not found").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("File not found").GetValue<CharT>()); break; case ErrorCode::ExpectedStringLiteral: - format_to(std::back_inserter(out), UNIVERSAL_STR("String expected").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("String expected").GetValue<CharT>()); break; case ErrorCode::ExpectedIdentifier: - format_to(std::back_inserter(out), UNIVERSAL_STR("Identifier expected").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Identifier expected").GetValue<CharT>()); break; case ErrorCode::ExpectedSquareBracket: - format_to(std::back_inserter(out), UNIVERSAL_STR("']' expected").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("']' expected").GetValue<CharT>()); break; case ErrorCode::ExpectedRoundBracket: - format_to(std::back_inserter(out), UNIVERSAL_STR("')' expected").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("')' expected").GetValue<CharT>()); break; case ErrorCode::ExpectedCurlyBracket: - format_to(std::back_inserter(out), UNIVERSAL_STR("'}}' expected").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("'}}' expected").GetValue<CharT>()); break; case ErrorCode::ExpectedToken: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected token '{}'").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected token '{}'").GetValue<CharT>(), extraParams[0]); if (extraParams.size() > 1) { - format_to(std::back_inserter(out), UNIVERSAL_STR(". Expected: ").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR(". Expected: ").GetValue<CharT>()); for (std::size_t i = 1; i < extraParams.size(); ++ i) { if (i != 1) - format_to(std::back_inserter(out), UNIVERSAL_STR(", ").GetValue<CharT>()); - format_to(std::back_inserter(out), UNIVERSAL_STR("\'{}\'").GetValue<CharT>(), extraParams[i]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR(", ").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("\'{}\'").GetValue<CharT>(), extraParams[i]); } } break; @@ -180,83 +180,83 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT> case ErrorCode::ExpectedExpression: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Expected expression, got: '{}'").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Expected expression, got: '{}'").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::ExpectedEndOfStatement: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of statement, got: '{}'").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of statement, got: '{}'").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::ExpectedRawEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of raw block").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of raw block").GetValue<CharT>()); break; case ErrorCode::ExpectedMetaEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of meta block").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Expected end of meta block").GetValue<CharT>()); break; case ErrorCode::UnexpectedToken: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected token: '{}'").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected token: '{}'").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::UnexpectedStatement: { auto& extraParams = errInfo.GetExtraParams(); - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement: '{}'").GetValue<CharT>(), extraParams[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement: '{}'").GetValue<CharT>(), extraParams[0]); break; } case ErrorCode::UnexpectedCommentBegin: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected comment begin").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected comment begin").GetValue<CharT>()); break; case ErrorCode::UnexpectedCommentEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected comment end").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected comment end").GetValue<CharT>()); break; case ErrorCode::UnexpectedRawBegin: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected raw block begin").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected raw block begin").GetValue<CharT>()); break; case ErrorCode::UnexpectedRawEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected raw block end").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected raw block end").GetValue<CharT>()); break; case ErrorCode::UnexpectedMetaBegin: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected meta block begin").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected meta block begin").GetValue<CharT>()); break; case ErrorCode::UnexpectedMetaEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected meta block end").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected meta block end").GetValue<CharT>()); break; case ErrorCode::UnexpectedExprBegin: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected expression block begin").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected expression block begin").GetValue<CharT>()); break; case ErrorCode::UnexpectedExprEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected expression block end").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected expression block end").GetValue<CharT>()); break; case ErrorCode::UnexpectedStmtBegin: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement block begin").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement block begin").GetValue<CharT>()); break; case ErrorCode::UnexpectedStmtEnd: - format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement block end").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Unexpected statement block end").GetValue<CharT>()); break; case ErrorCode::TemplateNotParsed: - format_to(std::back_inserter(out), UNIVERSAL_STR("Template not parsed").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Template not parsed").GetValue<CharT>()); break; case ErrorCode::TemplateNotFound: - format_to(std::back_inserter(out), UNIVERSAL_STR("Template(s) not found: {}").GetValue<CharT>(), errInfo.GetExtraParams()[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Template(s) not found: {}").GetValue<CharT>(), errInfo.GetExtraParams()[0]); break; case ErrorCode::InvalidTemplateName: - format_to(std::back_inserter(out), UNIVERSAL_STR("Invalid template name: {}").GetValue<CharT>(), errInfo.GetExtraParams()[0]); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Invalid template name: {}").GetValue<CharT>(), errInfo.GetExtraParams()[0]); break; case ErrorCode::InvalidValueType: - format_to(std::back_inserter(out), UNIVERSAL_STR("Invalid value type").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Invalid value type").GetValue<CharT>()); break; case ErrorCode::ExtensionDisabled: - format_to(std::back_inserter(out), UNIVERSAL_STR("Extension disabled").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Extension disabled").GetValue<CharT>()); break; case ErrorCode::TemplateEnvAbsent: - format_to(std::back_inserter(out), UNIVERSAL_STR("Template environment doesn't set").GetValue<CharT>()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("Template environment doesn't set").GetValue<CharT>()); break; } - format_to(std::back_inserter(out), UNIVERSAL_STR("\n{}").GetValue<CharT>(), errInfo.GetLocationDescr()); + fmt::format_to(std::back_inserter(out), UNIVERSAL_STR("\n{}").GetValue<CharT>(), errInfo.GetLocationDescr()); result = to_string(out); } diff --git a/contrib/libs/poco/Foundation/src/Format.cpp b/contrib/libs/poco/Foundation/src/Format.cpp index ed5db5c21b2..2f7900df551 100644 --- a/contrib/libs/poco/Foundation/src/Format.cpp +++ b/contrib/libs/poco/Foundation/src/Format.cpp @@ -228,7 +228,7 @@ namespace std::string format(const std::string& fmt, const Any& value) { std::string result; - format(result, fmt, value); + Poco::format(result, fmt, value); return result; } @@ -236,7 +236,7 @@ std::string format(const std::string& fmt, const Any& value) std::string format(const std::string& fmt, const Any& value1, const Any& value2) { std::string result; - format(result, fmt, value1, value2); + Poco::format(result, fmt, value1, value2); return result; } @@ -244,7 +244,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2) std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3) { std::string result; - format(result, fmt, value1, value2, value3); + Poco::format(result, fmt, value1, value2, value3); return result; } @@ -252,7 +252,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4) { std::string result; - format(result, fmt, value1, value2, value3, value4); + Poco::format(result, fmt, value1, value2, value3, value4); return result; } @@ -260,7 +260,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5); + Poco::format(result, fmt, value1, value2, value3, value4, value5); return result; } @@ -268,7 +268,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5, const Any& value6) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5, value6); + Poco::format(result, fmt, value1, value2, value3, value4, value5, value6); return result; } @@ -276,7 +276,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5, const Any& value6, const Any& value7) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5, value6, value7); + Poco::format(result, fmt, value1, value2, value3, value4, value5, value6, value7); return result; } @@ -284,7 +284,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5, const Any& value6, const Any& value7, const Any& value8) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8); + Poco::format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8); return result; } @@ -292,7 +292,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5, const Any& value6, const Any& value7, const Any& value8, const Any& value9) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8, value9); + Poco::format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8, value9); return result; } @@ -300,7 +300,7 @@ std::string format(const std::string& fmt, const Any& value1, const Any& value2, std::string format(const std::string& fmt, const Any& value1, const Any& value2, const Any& value3, const Any& value4, const Any& value5, const Any& value6, const Any& value7, const Any& value8, const Any& value9, const Any& value10) { std::string result; - format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8, value9, value10); + Poco::format(result, fmt, value1, value2, value3, value4, value5, value6, value7, value8, value9, value10); return result; } @@ -309,7 +309,7 @@ void format(std::string& result, const std::string& fmt, const Any& value) { std::vector<Any> args; args.push_back(value); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -318,7 +318,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons std::vector<Any> args; args.push_back(value1); args.push_back(value2); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -328,7 +328,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value1); args.push_back(value2); args.push_back(value3); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -339,7 +339,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value2); args.push_back(value3); args.push_back(value4); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -351,7 +351,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value3); args.push_back(value4); args.push_back(value5); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -364,7 +364,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value4); args.push_back(value5); args.push_back(value6); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -378,7 +378,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value5); args.push_back(value6); args.push_back(value7); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -393,7 +393,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value6); args.push_back(value7); args.push_back(value8); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -409,7 +409,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value7); args.push_back(value8); args.push_back(value9); - format(result, fmt, args); + Poco::format(result, fmt, args); } @@ -426,7 +426,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons args.push_back(value8); args.push_back(value9); args.push_back(value10); - format(result, fmt, args); + Poco::format(result, fmt, args); } diff --git a/contrib/python/jaraco.context/.dist-info/METADATA b/contrib/python/jaraco.context/.dist-info/METADATA new file mode 100644 index 00000000000..7d817563623 --- /dev/null +++ b/contrib/python/jaraco.context/.dist-info/METADATA @@ -0,0 +1,75 @@ +Metadata-Version: 2.1 +Name: jaraco.context +Version: 6.0.1 +Summary: Useful decorators and context managers +Author-email: "Jason R. Coombs" <jaraco@jaraco.com> +Project-URL: Source, https://github.com/jaraco/jaraco.context +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: backports.tarfile ; python_version < "3.12" +Provides-Extra: doc +Requires-Dist: sphinx >=3.5 ; extra == 'doc' +Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' +Requires-Dist: rst.linker >=1.9 ; extra == 'doc' +Requires-Dist: furo ; extra == 'doc' +Requires-Dist: sphinx-lint ; extra == 'doc' +Requires-Dist: jaraco.tidelift >=1.4 ; extra == 'doc' +Provides-Extra: test +Requires-Dist: pytest !=8.1.*,>=6 ; extra == 'test' +Requires-Dist: pytest-checkdocs >=2.4 ; extra == 'test' +Requires-Dist: pytest-cov ; extra == 'test' +Requires-Dist: pytest-mypy ; extra == 'test' +Requires-Dist: pytest-enabler >=2.2 ; extra == 'test' +Requires-Dist: portend ; extra == 'test' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'test' + +.. image:: https://img.shields.io/pypi/v/jaraco.context.svg + :target: https://pypi.org/project/jaraco.context + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.context.svg + +.. image:: https://github.com/jaraco/jaraco.context/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.context/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracocontext/badge/?version=latest + :target: https://jaracocontext.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.context + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.context?utm_source=pypi-jaraco.context&utm_medium=readme + + +Highlights +========== + +See the docs linked from the badge above for the full details, but here are some features that may be of interest. + +- ``ExceptionTrap`` provides a general-purpose wrapper for trapping exceptions and then acting on the outcome. Includes ``passes`` and ``raises`` decorators to replace the result of a wrapped function by a boolean indicating the outcome of the exception trap. See `this keyring commit <https://github.com/jaraco/keyring/commit/a85a7cbc6c909f8121660ed1f7b487f99a1c2bf7>`_ for an example of it in production. +- ``suppress`` simply enables ``contextlib.suppress`` as a decorator. +- ``on_interrupt`` is a decorator used by CLI entry points to affect the handling of a ``KeyboardInterrupt``. Inspired by `Lucretiel/autocommand#18 <https://github.com/Lucretiel/autocommand/issues/18>`_. +- ``pushd`` is similar to pytest's ``monkeypatch.chdir`` or path's `default context <https://path.readthedocs.io/en/latest/api.html>`_, changes the current working directory for the duration of the context. +- ``tarball`` will download a tarball, extract it, change directory, yield, then clean up after. Convenient when working with web assets. +- ``null`` is there for those times when one code branch needs a context and the other doesn't; this null context provides symmetry across those branches. + + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.context?utm_source=pypi-jaraco.context&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/jaraco.context/.dist-info/top_level.txt b/contrib/python/jaraco.context/.dist-info/top_level.txt new file mode 100644 index 00000000000..f6205a5f19a --- /dev/null +++ b/contrib/python/jaraco.context/.dist-info/top_level.txt @@ -0,0 +1 @@ +jaraco diff --git a/contrib/python/jaraco.context/LICENSE b/contrib/python/jaraco.context/LICENSE new file mode 100644 index 00000000000..1bb5a44356f --- /dev/null +++ b/contrib/python/jaraco.context/LICENSE @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/contrib/python/jaraco.context/README.rst b/contrib/python/jaraco.context/README.rst new file mode 100644 index 00000000000..1dedb0bbe3a --- /dev/null +++ b/contrib/python/jaraco.context/README.rst @@ -0,0 +1,44 @@ +.. image:: https://img.shields.io/pypi/v/jaraco.context.svg + :target: https://pypi.org/project/jaraco.context + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.context.svg + +.. image:: https://github.com/jaraco/jaraco.context/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.context/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracocontext/badge/?version=latest + :target: https://jaracocontext.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.context + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.context?utm_source=pypi-jaraco.context&utm_medium=readme + + +Highlights +========== + +See the docs linked from the badge above for the full details, but here are some features that may be of interest. + +- ``ExceptionTrap`` provides a general-purpose wrapper for trapping exceptions and then acting on the outcome. Includes ``passes`` and ``raises`` decorators to replace the result of a wrapped function by a boolean indicating the outcome of the exception trap. See `this keyring commit <https://github.com/jaraco/keyring/commit/a85a7cbc6c909f8121660ed1f7b487f99a1c2bf7>`_ for an example of it in production. +- ``suppress`` simply enables ``contextlib.suppress`` as a decorator. +- ``on_interrupt`` is a decorator used by CLI entry points to affect the handling of a ``KeyboardInterrupt``. Inspired by `Lucretiel/autocommand#18 <https://github.com/Lucretiel/autocommand/issues/18>`_. +- ``pushd`` is similar to pytest's ``monkeypatch.chdir`` or path's `default context <https://path.readthedocs.io/en/latest/api.html>`_, changes the current working directory for the duration of the context. +- ``tarball`` will download a tarball, extract it, change directory, yield, then clean up after. Convenient when working with web assets. +- ``null`` is there for those times when one code branch needs a context and the other doesn't; this null context provides symmetry across those branches. + + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.context?utm_source=pypi-jaraco.context&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/context.py b/contrib/python/jaraco.context/jaraco/context/__init__.py index 0322c45d4ab..55f1fb26f06 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/context.py +++ b/contrib/python/jaraco.context/jaraco/context/__init__.py @@ -1,20 +1,22 @@ from __future__ import annotations import contextlib +import errno import functools import operator import os +import platform import shutil +import stat import subprocess import sys import tempfile import urllib.request -import warnings from typing import Iterator if sys.version_info < (3, 12): - from setuptools.extern.backports import tarfile + from backports import tarfile else: import tarfile @@ -41,7 +43,16 @@ def tarball( url, target_dir: str | os.PathLike | None = None ) -> Iterator[str | os.PathLike]: """ - Get a tarball, extract it, yield, then clean up. + Get a URL to a tarball, download, extract, yield, then clean up. + + Assumes everything in the tarball is prefixed with a common + directory. That common path is stripped and the contents + are extracted to ``target_dir``, similar to passing + ``-C {target} --strip-components 1`` to the ``tar`` command. + + Uses the streaming protocol to extract the contents from a + stream in a single pass without loading the whole file into + memory. >>> import urllib.request >>> url = getfixture('tarfile_served') @@ -51,13 +62,18 @@ def tarball( >>> with tb as extracted: ... contents = pathlib.Path(extracted, 'contents.txt').read_text(encoding='utf-8') >>> assert not os.path.exists(extracted) + + If the target is not specified, contents are extracted to a + directory relative to the current working directory named after + the name of the file as extracted from the URL. + + >>> target = getfixture('tmp_path') + >>> with pushd(target), tarball(url): + ... target.joinpath('served').is_dir() + True """ if target_dir is None: target_dir = os.path.basename(url).replace('.tar.gz', '').replace('.tgz', '') - # In the tar command, use --strip-components=1 to strip the first path and - # then - # use -C to cause the files to be extracted to {target_dir}. This ensures - # that we always know where the files were extracted. os.mkdir(target_dir) try: req = urllib.request.urlopen(url) @@ -107,43 +123,31 @@ def _compose(*cmgrs): tarball_cwd = _compose(pushd, tarball) +""" +A tarball context with the current working directory pointing to the contents. +""" -@contextlib.contextmanager -def tarball_context(*args, **kwargs): - warnings.warn( - "tarball_context is deprecated. Use tarball or tarball_cwd instead.", - DeprecationWarning, - stacklevel=2, - ) - pushd_ctx = kwargs.pop('pushd', pushd) - with tarball(*args, **kwargs) as tball, pushd_ctx(tball) as dir: - yield dir - - -def infer_compression(url): +def remove_readonly(func, path, exc_info): """ - Given a URL or filename, infer the compression code for tar. - - >>> infer_compression('http://foo/bar.tar.gz') - 'z' - >>> infer_compression('http://foo/bar.tgz') - 'z' - >>> infer_compression('file.bz') - 'j' - >>> infer_compression('file.xz') - 'J' + Add support for removing read-only files on Windows. """ - warnings.warn( - "infer_compression is deprecated with no replacement", - DeprecationWarning, - stacklevel=2, + _, exc, _ = exc_info + if func in (os.rmdir, os.remove, os.unlink) and exc.errno == errno.EACCES: + # change the file to be readable,writable,executable: 0777 + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + # retry + func(path) + else: + raise + + +def robust_remover(): + return ( + functools.partial(shutil.rmtree, onerror=remove_readonly) + if platform.system() == 'Windows' + else shutil.rmtree ) - # cheat and just assume it's the last two characters - compression_indicator = url[-2:] - mapping = dict(gz='z', bz='j', xz='J') - # Assume 'z' (gzip) if no match - return mapping.get(compression_indicator, 'z') @contextlib.contextmanager @@ -155,7 +159,6 @@ def temp_dir(remover=shutil.rmtree): >>> import pathlib >>> with temp_dir() as the_dir: ... assert os.path.isdir(the_dir) - ... _ = pathlib.Path(the_dir).joinpath('somefile').write_text('contents', encoding='utf-8') >>> assert not os.path.exists(the_dir) """ temp_dir = tempfile.mkdtemp() @@ -165,44 +168,34 @@ def temp_dir(remover=shutil.rmtree): remover(temp_dir) +robust_temp_dir = functools.partial(temp_dir, remover=robust_remover()) + + @contextlib.contextmanager -def repo_context(url, branch=None, quiet=True, dest_ctx=temp_dir): +def repo_context( + url, branch: str | None = None, quiet: bool = True, dest_ctx=robust_temp_dir +): """ Check out the repo indicated by url. If dest_ctx is supplied, it should be a context manager to yield the target directory for the check out. + + >>> repo = repo_context('https://github.com/jaraco/jaraco.context') + >>> with repo as dest: + ... listing = os.listdir(dest) + >>> 'README.rst' in listing + True """ exe = 'git' if 'git' in url else 'hg' with dest_ctx() as repo_dir: cmd = [exe, 'clone', url, repo_dir] - if branch: - cmd.extend(['--branch', branch]) - devnull = open(os.path.devnull, 'w') - stdout = devnull if quiet else None - subprocess.check_call(cmd, stdout=stdout) + cmd.extend(['--branch', branch] * bool(branch)) + stream = subprocess.DEVNULL if quiet else None + subprocess.check_call(cmd, stdout=stream, stderr=stream) yield repo_dir -def null(): - """ - A null context suitable to stand in for a meaningful context. - - >>> with null() as value: - ... assert value is None - - This context is most useful when dealing with two or more code - branches but only some need a context. Wrap the others in a null - context to provide symmetry across all options. - """ - warnings.warn( - "null is deprecated. Use contextlib.nullcontext", - DeprecationWarning, - stacklevel=2, - ) - return contextlib.nullcontext() - - class ExceptionTrap: """ A context manager that will catch certain exceptions and provide an @@ -329,7 +322,9 @@ class suppress(contextlib.suppress, contextlib.ContextDecorator): class on_interrupt(contextlib.ContextDecorator): """ - Replace a KeyboardInterrupt with SystemExit(1) + Replace a KeyboardInterrupt with SystemExit(1). + + Useful in conjunction with console entry point functions. >>> def do_interrupt(): ... raise KeyboardInterrupt() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/py.typed b/contrib/python/jaraco.context/jaraco/context/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/py.typed +++ b/contrib/python/jaraco.context/jaraco/context/py.typed diff --git a/contrib/python/jaraco.context/ya.make b/contrib/python/jaraco.context/ya.make new file mode 100644 index 00000000000..8affe8dd49c --- /dev/null +++ b/contrib/python/jaraco.context/ya.make @@ -0,0 +1,23 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(6.0.1) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + jaraco/context/__init__.py +) + +RESOURCE_FILES( + PREFIX contrib/python/jaraco.context/ + .dist-info/METADATA + .dist-info/top_level.txt + jaraco/context/py.typed +) + +END() diff --git a/contrib/python/jaraco.functools/py2/.dist-info/METADATA b/contrib/python/jaraco.functools/py2/.dist-info/METADATA new file mode 100644 index 00000000000..7c476cf53ab --- /dev/null +++ b/contrib/python/jaraco.functools/py2/.dist-info/METADATA @@ -0,0 +1,45 @@ +Metadata-Version: 2.1 +Name: jaraco.functools +Version: 2.0 +Summary: Functools like those found in stdlib +Home-page: https://github.com/jaraco/jaraco.functools +Author: Jason R. Coombs +Author-email: jaraco@jaraco.com +License: UNKNOWN +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=2.7 +Requires-Dist: more-itertools +Requires-Dist: backports.functools-lru-cache (>=1.0.3) ; python_version == "2.7" +Provides-Extra: docs +Requires-Dist: sphinx ; extra == 'docs' +Requires-Dist: jaraco.packaging (>=3.2) ; extra == 'docs' +Requires-Dist: rst.linker (>=1.9) ; extra == 'docs' +Provides-Extra: testing +Requires-Dist: pytest (!=3.7.3,>=3.5) ; extra == 'testing' +Requires-Dist: pytest-checkdocs ; extra == 'testing' +Requires-Dist: pytest-flake8 ; extra == 'testing' +Requires-Dist: six ; extra == 'testing' +Requires-Dist: backports.unittest-mock ; extra == 'testing' +Requires-Dist: jaraco.classes ; extra == 'testing' + +.. image:: https://img.shields.io/pypi/v/jaraco.functools.svg + :target: https://pypi.org/project/jaraco.functools + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.functools.svg + +.. image:: https://img.shields.io/travis/jaraco/jaraco.functools/master.svg + :target: https://travis-ci.org/jaraco/jaraco.functools + +.. .. image:: https://img.shields.io/appveyor/ci/jaraco/jaraco-functools/master.svg +.. :target: https://ci.appveyor.com/project/jaraco-functools/skeleton/branch/master + +.. image:: https://readthedocs.org/projects/jaracofunctools/badge/?version=latest + :target: https://jaracofunctools.readthedocs.io/en/latest/?badge=latest + +Additional functools in the spirit of stdlib's functools. + diff --git a/contrib/python/jaraco.functools/py2/.dist-info/top_level.txt b/contrib/python/jaraco.functools/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..f6205a5f19a --- /dev/null +++ b/contrib/python/jaraco.functools/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +jaraco diff --git a/contrib/python/jaraco.functools/py2/.yandex_meta/yamaker.yaml b/contrib/python/jaraco.functools/py2/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..2ff6e0e7794 --- /dev/null +++ b/contrib/python/jaraco.functools/py2/.yandex_meta/yamaker.yaml @@ -0,0 +1,3 @@ +exclude: + # Для избежания конфликтов с другими пакетами jaraco.* + - jaraco/__init__.py diff --git a/contrib/python/jaraco.functools/py2/LICENSE b/contrib/python/jaraco.functools/py2/LICENSE new file mode 100644 index 00000000000..5e795a61f30 --- /dev/null +++ b/contrib/python/jaraco.functools/py2/LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/jaraco.functools/py2/README.rst b/contrib/python/jaraco.functools/py2/README.rst new file mode 100644 index 00000000000..424f53dc0a5 --- /dev/null +++ b/contrib/python/jaraco.functools/py2/README.rst @@ -0,0 +1,15 @@ +.. image:: https://img.shields.io/pypi/v/jaraco.functools.svg + :target: https://pypi.org/project/jaraco.functools + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.functools.svg + +.. image:: https://img.shields.io/travis/jaraco/jaraco.functools/master.svg + :target: https://travis-ci.org/jaraco/jaraco.functools + +.. .. image:: https://img.shields.io/appveyor/ci/jaraco/jaraco-functools/master.svg +.. :target: https://ci.appveyor.com/project/jaraco-functools/skeleton/branch/master + +.. image:: https://readthedocs.org/projects/jaracofunctools/badge/?version=latest + :target: https://jaracofunctools.readthedocs.io/en/latest/?badge=latest + +Additional functools in the spirit of stdlib's functools.
\ No newline at end of file diff --git a/contrib/python/jaraco.functools/py2/jaraco/functools.py b/contrib/python/jaraco.functools/py2/jaraco/functools.py new file mode 100644 index 00000000000..eb7cad4e803 --- /dev/null +++ b/contrib/python/jaraco.functools/py2/jaraco/functools.py @@ -0,0 +1,467 @@ +from __future__ import ( + absolute_import, unicode_literals, print_function, division, +) + +import functools +import time +import warnings +import inspect +import collections +from itertools import count + +__metaclass__ = type + + +try: + from functools import lru_cache +except ImportError: + try: + from backports.functools_lru_cache import lru_cache + except ImportError: + try: + from functools32 import lru_cache + except ImportError: + warnings.warn("No lru_cache available") + + +import more_itertools.recipes + + +def compose(*funcs): + """ + Compose any number of unary functions into a single unary function. + + >>> import textwrap + >>> from six import text_type + >>> stripped = text_type.strip(textwrap.dedent(compose.__doc__)) + >>> compose(text_type.strip, textwrap.dedent)(compose.__doc__) == stripped + True + + Compose also allows the innermost function to take arbitrary arguments. + + >>> round_three = lambda x: round(x, ndigits=3) + >>> f = compose(round_three, int.__truediv__) + >>> [f(3*x, x+1) for x in range(1,10)] + [1.5, 2.0, 2.25, 2.4, 2.5, 2.571, 2.625, 2.667, 2.7] + """ + + def compose_two(f1, f2): + return lambda *args, **kwargs: f1(f2(*args, **kwargs)) + return functools.reduce(compose_two, funcs) + + +def method_caller(method_name, *args, **kwargs): + """ + Return a function that will call a named method on the + target object with optional positional and keyword + arguments. + + >>> lower = method_caller('lower') + >>> lower('MyString') + 'mystring' + """ + def call_method(target): + func = getattr(target, method_name) + return func(*args, **kwargs) + return call_method + + +def once(func): + """ + Decorate func so it's only ever called the first time. + + This decorator can ensure that an expensive or non-idempotent function + will not be expensive on subsequent calls and is idempotent. + + >>> add_three = once(lambda a: a+3) + >>> add_three(3) + 6 + >>> add_three(9) + 6 + >>> add_three('12') + 6 + + To reset the stored value, simply clear the property ``saved_result``. + + >>> del add_three.saved_result + >>> add_three(9) + 12 + >>> add_three(8) + 12 + + Or invoke 'reset()' on it. + + >>> add_three.reset() + >>> add_three(-3) + 0 + >>> add_three(0) + 0 + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not hasattr(wrapper, 'saved_result'): + wrapper.saved_result = func(*args, **kwargs) + return wrapper.saved_result + wrapper.reset = lambda: vars(wrapper).__delitem__('saved_result') + return wrapper + + +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = functools.partial(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + return _special_method_cache(method, cache_wrapper) or wrapper + + +def _special_method_cache(method, cache_wrapper): + """ + Because Python treats special methods differently, it's not + possible to use instance attributes to implement the cached + methods. + + Instead, install the wrapper method under a different name + and return a simple proxy to that wrapper. + + https://github.com/jaraco/jaraco.functools/issues/5 + """ + name = method.__name__ + special_names = '__getattr__', '__getitem__' + if name not in special_names: + return + + wrapper_name = '__cached' + name + + def proxy(self, *args, **kwargs): + if wrapper_name not in vars(self): + bound = functools.partial(method, self) + cache = cache_wrapper(bound) + setattr(self, wrapper_name, cache) + else: + cache = getattr(self, wrapper_name) + return cache(*args, **kwargs) + + return proxy + + +def apply(transform): + """ + Decorate a function with a transform function that is + invoked on results returned from the decorated function. + + >>> @apply(reversed) + ... def get_numbers(start): + ... return range(start, start+3) + >>> list(get_numbers(4)) + [6, 5, 4] + """ + def wrap(func): + return compose(transform, func) + return wrap + + +def result_invoke(action): + r""" + Decorate a function with an action function that is + invoked on the results returned from the decorated + function (for its side-effect), then return the original + result. + + >>> @result_invoke(print) + ... def add_two(a, b): + ... return a + b + >>> x = add_two(2, 3) + 5 + """ + def wrap(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + result = func(*args, **kwargs) + action(result) + return result + return wrapper + return wrap + + +def call_aside(f, *args, **kwargs): + """ + Call a function for its side effect after initialization. + + >>> @call_aside + ... def func(): print("called") + called + >>> func() + called + + Use functools.partial to pass parameters to the initial call + + >>> @functools.partial(call_aside, name='bingo') + ... def func(name): print("called with", name) + called with bingo + """ + f(*args, **kwargs) + return f + + +class Throttler: + """ + Rate-limit a function (or other callable) + """ + def __init__(self, func, max_rate=float('Inf')): + if isinstance(func, Throttler): + func = func.func + self.func = func + self.max_rate = max_rate + self.reset() + + def reset(self): + self.last_called = 0 + + def __call__(self, *args, **kwargs): + self._wait() + return self.func(*args, **kwargs) + + def _wait(self): + "ensure at least 1/max_rate seconds from last call" + elapsed = time.time() - self.last_called + must_wait = 1 / self.max_rate - elapsed + time.sleep(max(0, must_wait)) + self.last_called = time.time() + + def __get__(self, obj, type=None): + return first_invoke(self._wait, functools.partial(self.func, obj)) + + +def first_invoke(func1, func2): + """ + Return a function that when invoked will invoke func1 without + any parameters (for its side-effect) and then invoke func2 + with whatever parameters were passed, returning its result. + """ + def wrapper(*args, **kwargs): + func1() + return func2(*args, **kwargs) + return wrapper + + +def retry_call(func, cleanup=lambda: None, retries=0, trap=()): + """ + Given a callable func, trap the indicated exceptions + for up to 'retries' times, invoking cleanup on the + exception. On the final attempt, allow any exceptions + to propagate. + """ + attempts = count() if retries == float('inf') else range(retries) + for attempt in attempts: + try: + return func() + except trap: + cleanup() + + return func() + + +def retry(*r_args, **r_kwargs): + """ + Decorator wrapper for retry_call. Accepts arguments to retry_call + except func and then returns a decorator for the decorated function. + + Ex: + + >>> @retry(retries=3) + ... def my_func(a, b): + ... "this is my funk" + ... print(a, b) + >>> my_func.__doc__ + 'this is my funk' + """ + def decorate(func): + @functools.wraps(func) + def wrapper(*f_args, **f_kwargs): + bound = functools.partial(func, *f_args, **f_kwargs) + return retry_call(bound, *r_args, **r_kwargs) + return wrapper + return decorate + + +def print_yielded(func): + """ + Convert a generator into a function that prints all yielded elements + + >>> @print_yielded + ... def x(): + ... yield 3; yield None + >>> x() + 3 + None + """ + print_all = functools.partial(map, print) + print_results = compose(more_itertools.recipes.consume, print_all, func) + return functools.wraps(func)(print_results) + + +def pass_none(func): + """ + Wrap func so it's not called if its first param is None + + >>> print_text = pass_none(print) + >>> print_text('text') + text + >>> print_text(None) + """ + @functools.wraps(func) + def wrapper(param, *args, **kwargs): + if param is not None: + return func(param, *args, **kwargs) + return wrapper + + +def assign_params(func, namespace): + """ + Assign parameters from namespace where func solicits. + + >>> def func(x, y=3): + ... print(x, y) + >>> assigned = assign_params(func, dict(x=2, z=4)) + >>> assigned() + 2 3 + + The usual errors are raised if a function doesn't receive + its required parameters: + + >>> assigned = assign_params(func, dict(y=3, z=4)) + >>> assigned() + Traceback (most recent call last): + TypeError: func() ...argument... + + It even works on methods: + + >>> class Handler: + ... def meth(self, arg): + ... print(arg) + >>> assign_params(Handler().meth, dict(arg='crystal', foo='clear'))() + crystal + """ + try: + sig = inspect.signature(func) + params = sig.parameters.keys() + except AttributeError: + spec = inspect.getargspec(func) + params = spec.args + call_ns = { + k: namespace[k] + for k in params + if k in namespace + } + return functools.partial(func, **call_ns) + + +def save_method_args(method): + """ + Wrap a method such that when it is called, the args and kwargs are + saved on the method. + + >>> class MyClass: + ... @save_method_args + ... def method(self, a, b): + ... print(a, b) + >>> my_ob = MyClass() + >>> my_ob.method(1, 2) + 1 2 + >>> my_ob._saved_method.args + (1, 2) + >>> my_ob._saved_method.kwargs + {} + >>> my_ob.method(a=3, b='foo') + 3 foo + >>> my_ob._saved_method.args + () + >>> my_ob._saved_method.kwargs == dict(a=3, b='foo') + True + + The arguments are stored on the instance, allowing for + different instance to save different args. + + >>> your_ob = MyClass() + >>> your_ob.method({str('x'): 3}, b=[4]) + {'x': 3} [4] + >>> your_ob._saved_method.args + ({'x': 3},) + >>> my_ob._saved_method.args + () + """ + args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') + + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + attr_name = '_saved_' + method.__name__ + attr = args_and_kwargs(args, kwargs) + setattr(self, attr_name, attr) + return method(self, *args, **kwargs) + return wrapper diff --git a/contrib/python/jaraco.functools/py2/ya.make b/contrib/python/jaraco.functools/py2/ya.make new file mode 100644 index 00000000000..afd1fa109e8 --- /dev/null +++ b/contrib/python/jaraco.functools/py2/ya.make @@ -0,0 +1,27 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +VERSION(2.0) + +LICENSE(MIT) + +PEERDIR( + contrib/deprecated/python/backports.functools-lru-cache + contrib/python/more-itertools +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + jaraco/functools.py +) + +RESOURCE_FILES( + PREFIX contrib/python/jaraco.functools/py2/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/jaraco.functools/py3/.dist-info/METADATA b/contrib/python/jaraco.functools/py3/.dist-info/METADATA new file mode 100644 index 00000000000..e0a75d48073 --- /dev/null +++ b/contrib/python/jaraco.functools/py3/.dist-info/METADATA @@ -0,0 +1,67 @@ +Metadata-Version: 2.1 +Name: jaraco.functools +Version: 4.1.0 +Summary: Functools like those found in stdlib +Author-email: "Jason R. Coombs" <jaraco@jaraco.com> +Project-URL: Source, https://github.com/jaraco/jaraco.functools +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: more-itertools +Provides-Extra: check +Requires-Dist: pytest-checkdocs >=2.4 ; extra == 'check' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'check' +Provides-Extra: cover +Requires-Dist: pytest-cov ; extra == 'cover' +Provides-Extra: doc +Requires-Dist: sphinx >=3.5 ; extra == 'doc' +Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' +Requires-Dist: rst.linker >=1.9 ; extra == 'doc' +Requires-Dist: furo ; extra == 'doc' +Requires-Dist: sphinx-lint ; extra == 'doc' +Requires-Dist: jaraco.tidelift >=1.4 ; extra == 'doc' +Provides-Extra: enabler +Requires-Dist: pytest-enabler >=2.2 ; extra == 'enabler' +Provides-Extra: test +Requires-Dist: pytest !=8.1.*,>=6 ; extra == 'test' +Requires-Dist: jaraco.classes ; extra == 'test' +Provides-Extra: type +Requires-Dist: pytest-mypy ; extra == 'type' + +.. image:: https://img.shields.io/pypi/v/jaraco.functools.svg + :target: https://pypi.org/project/jaraco.functools + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.functools.svg + +.. image:: https://github.com/jaraco/jaraco.functools/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.functools/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracofunctools/badge/?version=latest + :target: https://jaracofunctools.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.functools + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.functools?utm_source=pypi-jaraco.functools&utm_medium=readme + +Additional functools in the spirit of stdlib's functools. + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.functools?utm_source=pypi-jaraco.functools&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/jaraco.functools/py3/.dist-info/top_level.txt b/contrib/python/jaraco.functools/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..f6205a5f19a --- /dev/null +++ b/contrib/python/jaraco.functools/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +jaraco diff --git a/contrib/python/jaraco.functools/py3/LICENSE b/contrib/python/jaraco.functools/py3/LICENSE new file mode 100644 index 00000000000..1bb5a44356f --- /dev/null +++ b/contrib/python/jaraco.functools/py3/LICENSE @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/contrib/python/jaraco.functools/py3/README.rst b/contrib/python/jaraco.functools/py3/README.rst new file mode 100644 index 00000000000..a7f48543a10 --- /dev/null +++ b/contrib/python/jaraco.functools/py3/README.rst @@ -0,0 +1,32 @@ +.. image:: https://img.shields.io/pypi/v/jaraco.functools.svg + :target: https://pypi.org/project/jaraco.functools + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.functools.svg + +.. image:: https://github.com/jaraco/jaraco.functools/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.functools/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracofunctools/badge/?version=latest + :target: https://jaracofunctools.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.functools + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.functools?utm_source=pypi-jaraco.functools&utm_medium=readme + +Additional functools in the spirit of stdlib's functools. + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.functools?utm_source=pypi-jaraco.functools&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.py b/contrib/python/jaraco.functools/py3/jaraco/functools/__init__.py index 130b87a4859..d510530789b 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.py +++ b/contrib/python/jaraco.functools/py3/jaraco/functools/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import collections.abc import functools import inspect @@ -7,13 +9,23 @@ import time import types import warnings -import setuptools.extern.more_itertools +from typing import Callable, TypeVar + +import more_itertools def compose(*funcs): """ Compose any number of unary functions into a single unary function. + Comparable to + `function composition <https://en.wikipedia.org/wiki/Function_composition>`_ + in mathematics: + + ``h = g ∘ f`` implies ``h(x) = g(f(x))``. + + In Python, ``h = compose(g, f)``. + >>> import textwrap >>> expected = str.strip(textwrap.dedent(compose.__doc__)) >>> strip_and_dedent = compose(str.strip, textwrap.dedent) @@ -603,10 +615,10 @@ def splat(func): simple ``map``. >>> pairs = [(-1, 1), (0, 2)] - >>> setuptools.extern.more_itertools.consume(itertools.starmap(print, pairs)) + >>> more_itertools.consume(itertools.starmap(print, pairs)) -1 1 0 2 - >>> setuptools.extern.more_itertools.consume(map(splat(print), pairs)) + >>> more_itertools.consume(map(splat(print), pairs)) -1 1 0 2 @@ -631,3 +643,43 @@ def splat(func): {'msg': 'unknown', 'code': 0} """ return functools.wraps(func)(functools.partial(_splat_inner, func=func)) + + +_T = TypeVar('_T') + + +def chainable(method: Callable[[_T, ...], None]) -> Callable[[_T, ...], _T]: + """ + Wrap an instance method to always return self. + + + >>> class Dingus: + ... @chainable + ... def set_attr(self, name, val): + ... setattr(self, name, val) + >>> d = Dingus().set_attr('a', 'eh!') + >>> d.a + 'eh!' + >>> d2 = Dingus().set_attr('a', 'eh!').set_attr('b', 'bee!') + >>> d2.a + d2.b + 'eh!bee!' + + Enforces that the return value is null. + + >>> class BorkedDingus: + ... @chainable + ... def set_attr(self, name, val): + ... setattr(self, name, val) + ... return len(name) + >>> BorkedDingus().set_attr('a', 'eh!') + Traceback (most recent call last): + ... + AssertionError + """ + + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + assert method(self, *args, **kwargs) is None + return self + + return wrapper diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.pyi b/contrib/python/jaraco.functools/py3/jaraco/functools/__init__.pyi index c2b9ab1757e..19191bf93ee 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/__init__.pyi +++ b/contrib/python/jaraco.functools/py3/jaraco/functools/__init__.pyi @@ -74,9 +74,6 @@ def result_invoke( def invoke( f: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs ) -> Callable[_P, _R]: ... -def call_aside( - f: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... class Throttler(Generic[_R]): last_called: float diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/py.typed b/contrib/python/jaraco.functools/py3/jaraco/functools/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/py.typed +++ b/contrib/python/jaraco.functools/py3/jaraco/functools/py.typed diff --git a/contrib/python/jaraco.functools/py3/ya.make b/contrib/python/jaraco.functools/py3/ya.make new file mode 100644 index 00000000000..36e37b14d58 --- /dev/null +++ b/contrib/python/jaraco.functools/py3/ya.make @@ -0,0 +1,28 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(4.1.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/more-itertools +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + jaraco/functools/__init__.py + jaraco/functools/__init__.pyi +) + +RESOURCE_FILES( + PREFIX contrib/python/jaraco.functools/py3/ + .dist-info/METADATA + .dist-info/top_level.txt + jaraco/functools/py.typed +) + +END() diff --git a/contrib/python/jaraco.functools/ya.make b/contrib/python/jaraco.functools/ya.make new file mode 100644 index 00000000000..4ff2677d2b6 --- /dev/null +++ b/contrib/python/jaraco.functools/ya.make @@ -0,0 +1,20 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +VERSION(Service-proxy-version) + +IF (PYTHON2) + PEERDIR(contrib/python/jaraco.functools/py2) +ELSE() + PEERDIR(contrib/python/jaraco.functools/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) diff --git a/contrib/python/jaraco.text/.dist-info/METADATA b/contrib/python/jaraco.text/.dist-info/METADATA new file mode 100644 index 00000000000..797b9da7334 --- /dev/null +++ b/contrib/python/jaraco.text/.dist-info/METADATA @@ -0,0 +1,96 @@ +Metadata-Version: 2.1 +Name: jaraco.text +Version: 4.0.0 +Summary: Module for text manipulation +Author-email: "Jason R. Coombs" <jaraco@jaraco.com> +Project-URL: Source, https://github.com/jaraco/jaraco.text +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: jaraco.functools +Requires-Dist: jaraco.context >=4.1 +Requires-Dist: autocommand +Requires-Dist: more-itertools +Requires-Dist: importlib-resources ; python_version < "3.9" +Provides-Extra: doc +Requires-Dist: sphinx >=3.5 ; extra == 'doc' +Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' +Requires-Dist: rst.linker >=1.9 ; extra == 'doc' +Requires-Dist: furo ; extra == 'doc' +Requires-Dist: sphinx-lint ; extra == 'doc' +Requires-Dist: jaraco.tidelift >=1.4 ; extra == 'doc' +Provides-Extra: inflect +Requires-Dist: inflect ; extra == 'inflect' +Provides-Extra: test +Requires-Dist: pytest !=8.1.*,>=6 ; extra == 'test' +Requires-Dist: pytest-checkdocs >=2.4 ; extra == 'test' +Requires-Dist: pytest-cov ; extra == 'test' +Requires-Dist: pytest-mypy ; extra == 'test' +Requires-Dist: pytest-enabler >=2.2 ; extra == 'test' +Requires-Dist: pathlib2 ; (python_version < "3.10") and extra == 'test' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'test' + +.. image:: https://img.shields.io/pypi/v/jaraco.text.svg + :target: https://pypi.org/project/jaraco.text + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.text.svg + +.. image:: https://github.com/jaraco/jaraco.text/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.text/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracotext/badge/?version=latest + :target: https://jaracotext.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.text + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.text?utm_source=pypi-jaraco.text&utm_medium=readme + + +This package provides handy routines for dealing with text, such as +wrapping, substitution, trimming, stripping, prefix and suffix removal, +line continuation, indentation, comment processing, identifier processing, +values parsing, case insensitive comparison, and more. See the docs +(linked in the badge above) for the detailed documentation and examples. + +Layouts +======= + +One of the features of this package is the layouts module, which +provides a simple example of translating keystrokes from one keyboard +layout to another:: + + echo qwerty | python -m jaraco.text.to-dvorak + ',.pyf + echo "',.pyf" | python -m jaraco.text.to-qwerty + qwerty + +Newline Reporting +================= + +Need to know what newlines appear in a file? + +:: + + $ python -m jaraco.text.show-newlines README.rst + newline is '\n' + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.text?utm_source=pypi-jaraco.text&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/jaraco.text/.dist-info/top_level.txt b/contrib/python/jaraco.text/.dist-info/top_level.txt new file mode 100644 index 00000000000..f6205a5f19a --- /dev/null +++ b/contrib/python/jaraco.text/.dist-info/top_level.txt @@ -0,0 +1 @@ +jaraco diff --git a/contrib/python/jaraco.text/.yandex_meta/yamaker.yaml b/contrib/python/jaraco.text/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..bc39e7ce986 --- /dev/null +++ b/contrib/python/jaraco.text/.yandex_meta/yamaker.yaml @@ -0,0 +1,6 @@ +requirements: + - autocommand: null +exclude: + - "jaraco/text/Lorem ipsum.txt" +keep: + - jaraco/text/Lorem_ipsum.txt diff --git a/contrib/python/jaraco.text/LICENSE b/contrib/python/jaraco.text/LICENSE new file mode 100644 index 00000000000..1bb5a44356f --- /dev/null +++ b/contrib/python/jaraco.text/LICENSE @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/contrib/python/jaraco.text/README.rst b/contrib/python/jaraco.text/README.rst new file mode 100644 index 00000000000..c5a9f1db169 --- /dev/null +++ b/contrib/python/jaraco.text/README.rst @@ -0,0 +1,59 @@ +.. image:: https://img.shields.io/pypi/v/jaraco.text.svg + :target: https://pypi.org/project/jaraco.text + +.. image:: https://img.shields.io/pypi/pyversions/jaraco.text.svg + +.. image:: https://github.com/jaraco/jaraco.text/actions/workflows/main.yml/badge.svg + :target: https://github.com/jaraco/jaraco.text/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/jaracotext/badge/?version=latest + :target: https://jaracotext.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2024-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/jaraco.text + :target: https://tidelift.com/subscription/pkg/pypi-jaraco.text?utm_source=pypi-jaraco.text&utm_medium=readme + + +This package provides handy routines for dealing with text, such as +wrapping, substitution, trimming, stripping, prefix and suffix removal, +line continuation, indentation, comment processing, identifier processing, +values parsing, case insensitive comparison, and more. See the docs +(linked in the badge above) for the detailed documentation and examples. + +Layouts +======= + +One of the features of this package is the layouts module, which +provides a simple example of translating keystrokes from one keyboard +layout to another:: + + echo qwerty | python -m jaraco.text.to-dvorak + ',.pyf + echo "',.pyf" | python -m jaraco.text.to-qwerty + qwerty + +Newline Reporting +================= + +Need to know what newlines appear in a file? + +:: + + $ python -m jaraco.text.show-newlines README.rst + newline is '\n' + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more <https://tidelift.com/subscription/pkg/pypi-jaraco.text?utm_source=pypi-jaraco.text&utm_medium=referral&utm_campaign=github>`_. diff --git a/contrib/python/jaraco.text/jaraco/text/Lorem_ipsum.txt b/contrib/python/jaraco.text/jaraco/text/Lorem_ipsum.txt new file mode 100644 index 00000000000..986f944b60b --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/Lorem_ipsum.txt @@ -0,0 +1,2 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus magna felis sollicitudin mauris. Integer in mauris eu nibh euismod gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis risus a elit. Etiam tempor. Ut ullamcorper, ligula eu tempor congue, eros est euismod turpis, id tincidunt sapien risus a quam. Maecenas fermentum consequat mi. Donec fermentum. Pellentesque malesuada nulla a mi. Duis sapien sem, aliquet nec, commodo eget, consequat quis, neque. Aliquam faucibus, elit ut dictum aliquet, felis nisl adipiscing sapien, sed malesuada diam lacus eget erat. Cras mollis scelerisque nunc. Nullam arcu. Aliquam consequat. Curabitur augue lorem, dapibus quis, laoreet et, pretium ac, nisi. Aenean magna nisl, mollis quis, molestie eu, feugiat in, orci. In hac habitasse platea dictumst. diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/text/__init__.py b/contrib/python/jaraco.text/jaraco/text/__init__.py index c466378ceba..af5e07598bb 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/text/__init__.py +++ b/contrib/python/jaraco.text/jaraco/text/__init__.py @@ -1,15 +1,17 @@ -import re +import functools import itertools +import re import textwrap -import functools + +from typing import Iterable try: from importlib.resources import files # type: ignore except ImportError: # pragma: nocover - from pkg_resources.extern.importlib_resources import files # type: ignore + from importlib_resources import files # type: ignore -from pkg_resources.extern.jaraco.functools import compose, method_cache -from pkg_resources.extern.jaraco.context import ExceptionTrap +from jaraco.context import ExceptionTrap +from jaraco.functools import compose, method_cache def substitution(old, new): @@ -66,7 +68,7 @@ class FoldedCase(str): >>> s in ["Hello World"] True - You may test for set inclusion, but candidate and elements + Allows testing for set inclusion, but candidate and elements must both be folded. >>> FoldedCase("Hello World") in {s} @@ -92,37 +94,40 @@ class FoldedCase(str): >>> FoldedCase('hello') > FoldedCase('Hello') False + + >>> FoldedCase('ß') == FoldedCase('ss') + True """ def __lt__(self, other): - return self.lower() < other.lower() + return self.casefold() < other.casefold() def __gt__(self, other): - return self.lower() > other.lower() + return self.casefold() > other.casefold() def __eq__(self, other): - return self.lower() == other.lower() + return self.casefold() == other.casefold() def __ne__(self, other): - return self.lower() != other.lower() + return self.casefold() != other.casefold() def __hash__(self): - return hash(self.lower()) + return hash(self.casefold()) def __contains__(self, other): - return super().lower().__contains__(other.lower()) + return super().casefold().__contains__(other.casefold()) def in_(self, other): "Does self appear in other?" return self in FoldedCase(other) - # cache lower since it's likely to be called frequently. + # cache casefold since it's likely to be called frequently. @method_cache - def lower(self): - return super().lower() + def casefold(self): + return super().casefold() def index(self, sub): - return self.lower().index(sub.lower()) + return self.casefold().index(sub.casefold()) def split(self, splitter=' ', maxsplit=0): pattern = re.compile(re.escape(splitter), re.I) @@ -224,9 +229,12 @@ def unwrap(s): return '\n'.join(cleaned) +lorem_ipsum: str = ( + files(__name__).joinpath('Lorem_ipsum.txt').read_text(encoding='utf-8') +) -class Splitter(object): +class Splitter: """object that will split a string with the given arguments for each call >>> s = Splitter(',') @@ -276,7 +284,7 @@ class WordSet(tuple): >>> WordSet.parse("myABCClass") ('my', 'ABC', 'Class') - The result is a WordSet, so you can get the form you need. + The result is a WordSet, providing access to other forms. >>> WordSet.parse("myABCClass").underscore_separated() 'my_ABC_Class' @@ -363,7 +371,7 @@ class WordSet(tuple): return self.trim_left(item).trim_right(item) def __getitem__(self, item): - result = super(WordSet, self).__getitem__(item) + result = super().__getitem__(item) if isinstance(item, slice): result = WordSet(result) return result @@ -548,7 +556,14 @@ def yield_lines(iterable): @yield_lines.register(str) def _(text): - return filter(_nonblank, map(str.strip, text.splitlines())) + return clean(text.splitlines()) + + +def clean(lines: Iterable[str]): + """ + Yield non-blank, non-comment elements from lines. + """ + return filter(_nonblank, map(str.strip, lines)) def drop_comment(line): @@ -578,7 +593,7 @@ def join_continuation(lines): ['foobarbaz'] Not sure why, but... - The character preceeding the backslash is also elided. + The character preceding the backslash is also elided. >>> list(join_continuation(['goo\\', 'dly'])) ['godly'] @@ -597,3 +612,36 @@ def join_continuation(lines): except StopIteration: return yield item + + +def read_newlines(filename, limit=1024): + r""" + >>> tmp_path = getfixture('tmp_path') + >>> filename = tmp_path / 'out.txt' + >>> _ = filename.write_text('foo\n', newline='', encoding='utf-8') + >>> read_newlines(filename) + '\n' + >>> _ = filename.write_text('foo\r\n', newline='', encoding='utf-8') + >>> read_newlines(filename) + '\r\n' + >>> _ = filename.write_text('foo\r\nbar\nbing\r', newline='', encoding='utf-8') + >>> read_newlines(filename) + ('\r', '\n', '\r\n') + """ + with open(filename, encoding='utf-8') as fp: + fp.read(limit) + return fp.newlines + + +def lines_from(input): + """ + Generate lines from a :class:`importlib.resources.abc.Traversable` path. + + >>> lines = lines_from(files(__name__).joinpath('Lorem ipsum.txt')) + >>> next(lines) + 'Lorem ipsum...' + >>> next(lines) + 'Curabitur pretium...' + """ + with input.open(encoding='utf-8') as stream: + yield from stream diff --git a/contrib/python/jaraco.text/jaraco/text/layouts.py b/contrib/python/jaraco.text/jaraco/text/layouts.py new file mode 100644 index 00000000000..9636f0f7b53 --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/layouts.py @@ -0,0 +1,25 @@ +qwerty = "-=qwertyuiop[]asdfghjkl;'zxcvbnm,./_+QWERTYUIOP{}ASDFGHJKL:\"ZXCVBNM<>?" +dvorak = "[]',.pyfgcrl/=aoeuidhtns-;qjkxbmwvz{}\"<>PYFGCRL?+AOEUIDHTNS_:QJKXBMWVZ" + + +to_dvorak = str.maketrans(qwerty, dvorak) +to_qwerty = str.maketrans(dvorak, qwerty) + + +def translate(input, translation): + """ + >>> translate('dvorak', to_dvorak) + 'ekrpat' + >>> translate('qwerty', to_qwerty) + 'x,dokt' + """ + return input.translate(translation) + + +def _translate_stream(stream, translation): + """ + >>> import io + >>> _translate_stream(io.StringIO('foo'), to_dvorak) + urr + """ + print(translate(stream.read(), translation)) diff --git a/contrib/python/jaraco.text/jaraco/text/show-newlines.py b/contrib/python/jaraco.text/jaraco/text/show-newlines.py new file mode 100644 index 00000000000..b78bcae50ef --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/show-newlines.py @@ -0,0 +1,28 @@ +import inflect +from more_itertools import always_iterable + +import jaraco.text + + +def report_newlines(filename): + r""" + Report the newlines in the indicated file. + + >>> tmp_path = getfixture('tmp_path') + >>> filename = tmp_path / 'out.txt' + >>> _ = filename.write_text('foo\nbar\n', newline='', encoding='utf-8') + >>> report_newlines(filename) + newline is '\n' + >>> filename = tmp_path / 'out.txt' + >>> _ = filename.write_text('foo\nbar\r\n', newline='', encoding='utf-8') + >>> report_newlines(filename) + newlines are ('\n', '\r\n') + """ + newlines = jaraco.text.read_newlines(filename) + count = len(tuple(always_iterable(newlines))) + engine = inflect.engine() + print( + engine.plural_noun("newline", count), + engine.plural_verb("is", count), + repr(newlines), + ) diff --git a/contrib/python/jaraco.text/jaraco/text/strip-prefix.py b/contrib/python/jaraco.text/jaraco/text/strip-prefix.py new file mode 100644 index 00000000000..0d7b7702dc1 --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/strip-prefix.py @@ -0,0 +1,16 @@ +import sys + +from jaraco.text import Stripper + + +def strip_prefix(): + r""" + Strip any common prefix from stdin. + + >>> import io, pytest + >>> getfixture('monkeypatch').setattr('sys.stdin', io.StringIO('abcdef\nabc123')) + >>> strip_prefix() + def + 123 + """ + sys.stdout.writelines(Stripper.strip_prefix(sys.stdin).lines) diff --git a/contrib/python/jaraco.text/jaraco/text/to-dvorak.py b/contrib/python/jaraco.text/jaraco/text/to-dvorak.py new file mode 100644 index 00000000000..14c8981e441 --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/to-dvorak.py @@ -0,0 +1,5 @@ +import sys + +from . import layouts + +__name__ == '__main__' and layouts._translate_stream(sys.stdin, layouts.to_dvorak) diff --git a/contrib/python/jaraco.text/jaraco/text/to-qwerty.py b/contrib/python/jaraco.text/jaraco/text/to-qwerty.py new file mode 100644 index 00000000000..23596fda930 --- /dev/null +++ b/contrib/python/jaraco.text/jaraco/text/to-qwerty.py @@ -0,0 +1,5 @@ +import sys + +from . import layouts + +__name__ == '__main__' and layouts._translate_stream(sys.stdin, layouts.to_qwerty) diff --git a/contrib/python/jaraco.text/patches/01-arcadia.patch b/contrib/python/jaraco.text/patches/01-arcadia.patch new file mode 100644 index 00000000000..fb34122ec72 --- /dev/null +++ b/contrib/python/jaraco.text/patches/01-arcadia.patch @@ -0,0 +1,5 @@ +--- contrib/python/jaraco.text/jaraco/text/__init__.py (index) ++++ contrib/python/jaraco.text/jaraco/text/__init__.py (working tree) +@@ -231,1 +231,1 @@ def unwrap(s): +- files(__name__).joinpath('Lorem ipsum.txt').read_text(encoding='utf-8') ++ files(__name__).joinpath('Lorem_ipsum.txt').read_text(encoding='utf-8') diff --git a/contrib/python/jaraco.text/patches/01-remove-autocommand.patch b/contrib/python/jaraco.text/patches/01-remove-autocommand.patch new file mode 100644 index 00000000000..1b6d9d463c1 --- /dev/null +++ b/contrib/python/jaraco.text/patches/01-remove-autocommand.patch @@ -0,0 +1,20 @@ +У autocommand лицензия LGPL-3 и транзитивно через jaraco.text поджигает проверку у Catboost + +В Аркидии этим ни кто не пользуется, потому просто удаляю +--- contrib/python/jaraco.text/jaraco/text/show-newlines.py (index) ++++ contrib/python/jaraco.text/jaraco/text/show-newlines.py (working tree) +@@ -1 +0,0 @@ +-import autocommand +@@ -30,3 +28,0 @@ def report_newlines(filename): +- +- +-autocommand.autocommand(__name__)(report_newlines) +--- contrib/python/jaraco.text/jaraco/text/strip-prefix.py (index) ++++ contrib/python/jaraco.text/jaraco/text/strip-prefix.py (working tree) +@@ -3,2 +2,0 @@ import sys +-import autocommand +- +@@ -19,3 +16,0 @@ def strip_prefix(): +- +- +-autocommand.autocommand(__name__)(strip_prefix) diff --git a/contrib/python/jaraco.text/ya.make b/contrib/python/jaraco.text/ya.make new file mode 100644 index 00000000000..b5e3efddec6 --- /dev/null +++ b/contrib/python/jaraco.text/ya.make @@ -0,0 +1,38 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(4.0.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/jaraco.context + contrib/python/jaraco.functools + contrib/python/more-itertools +) + +NO_LINT() + +NO_CHECK_IMPORTS( + jaraco.text.show-newlines +) + +PY_SRCS( + TOP_LEVEL + jaraco/text/__init__.py + jaraco/text/layouts.py + jaraco/text/show-newlines.py + jaraco/text/strip-prefix.py + jaraco/text/to-dvorak.py + jaraco/text/to-qwerty.py +) + +RESOURCE_FILES( + PREFIX contrib/python/jaraco.text/ + .dist-info/METADATA + .dist-info/top_level.txt + jaraco/text/Lorem_ipsum.txt +) + +END() diff --git a/contrib/python/ordered-set/.dist-info/METADATA b/contrib/python/ordered-set/.dist-info/METADATA new file mode 100644 index 00000000000..7aea136818b --- /dev/null +++ b/contrib/python/ordered-set/.dist-info/METADATA @@ -0,0 +1,158 @@ +Metadata-Version: 2.1 +Name: ordered-set +Version: 4.1.0 +Summary: An OrderedSet is a custom MutableSet that remembers its order, so that every +Author-email: Elia Robyn Lake <gh@arborelia.net> +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Dist: pytest ; extra == "dev" +Requires-Dist: black ; extra == "dev" +Requires-Dist: mypy ; extra == "dev" +Project-URL: Home, https://github.com/rspeer/ordered-set +Provides-Extra: dev + +[](https://pypi.python.org/pypi/ordered-set) + +An OrderedSet is a mutable data structure that is a hybrid of a list and a set. +It remembers the order of its entries, and every entry has an index number that +can be looked up. + +## Installation + +`ordered_set` is available on PyPI and packaged as a wheel. You can list it +as a dependency of your project, in whatever form that takes. + +To install it into your current Python environment: + + pip install ordered-set + +To install the code for development, after checking out the repository: + + pip install flit + flit install + +## Usage examples + +An OrderedSet is created and used like a set: + + >>> from ordered_set import OrderedSet + + >>> letters = OrderedSet('abracadabra') + + >>> letters + OrderedSet(['a', 'b', 'r', 'c', 'd']) + + >>> 'r' in letters + True + +It is efficient to find the index of an entry in an OrderedSet, or find an +entry by its index. To help with this use case, the `.add()` method returns +the index of the added item, whether it was already in the set or not. + + >>> letters.index('r') + 2 + + >>> letters[2] + 'r' + + >>> letters.add('r') + 2 + + >>> letters.add('x') + 5 + +OrderedSets implement the union (`|`), intersection (`&`), and difference (`-`) +operators like sets do. + + >>> letters |= OrderedSet('shazam') + + >>> letters + OrderedSet(['a', 'b', 'r', 'c', 'd', 'x', 's', 'h', 'z', 'm']) + + >>> letters & set('aeiou') + OrderedSet(['a']) + + >>> letters -= 'abcd' + + >>> letters + OrderedSet(['r', 'x', 's', 'h', 'z', 'm']) + +The `__getitem__()` and `index()` methods have been extended to accept any +iterable except a string, returning a list, to perform NumPy-like "fancy +indexing". + + >>> letters = OrderedSet('abracadabra') + + >>> letters[[0, 2, 3]] + ['a', 'r', 'c'] + + >>> letters.index(['a', 'r', 'c']) + [0, 2, 3] + +OrderedSet implements `__getstate__` and `__setstate__` so it can be pickled, +and implements the abstract base classes `collections.MutableSet` and +`collections.Sequence`. + +OrderedSet can be used as a generic collection type, similar to the collections +in the `typing` module like List, Dict, and Set. For example, you can annotate +a variable as having the type `OrderedSet[str]` or `OrderedSet[Tuple[int, +str]]`. + + +## OrderedSet in data science applications + +An OrderedSet can be used as a bi-directional mapping between a sparse +vocabulary and dense index numbers. As of version 3.1, it accepts NumPy arrays +of index numbers as well as lists. + +This combination of features makes OrderedSet a simple implementation of many +of the things that `pandas.Index` is used for, and many of its operations are +faster than the equivalent pandas operations. + +For further compatibility with pandas.Index, `get_loc` (the pandas method for +looking up a single index) and `get_indexer` (the pandas method for fancy +indexing in reverse) are both aliases for `index` (which handles both cases +in OrderedSet). + + +## Authors + +OrderedSet was implemented by Elia Robyn Lake (maiden name: Robyn Speer). +Jon Crall contributed changes and tests to make it fit the Python set API. +Roman Inflianskas added the original type annotations. + + +## Comparisons + +The original implementation of OrderedSet was a [recipe posted to ActiveState +Recipes][recipe] by Raymond Hettiger, released under the MIT license. + +[recipe]: https://code.activestate.com/recipes/576694-orderedset/ + +Hettiger's implementation kept its content in a doubly-linked list referenced by a +dict. As a result, looking up an item by its index was an O(N) operation, while +deletion was O(1). + +This version makes different trade-offs for the sake of efficient lookups. Its +content is a standard Python list instead of a doubly-linked list. This +provides O(1) lookups by index at the expense of O(N) deletion, as well as +slightly faster iteration. + +In Python 3.6 and later, the built-in `dict` type is inherently ordered. If you +ignore the dictionary values, that also gives you a simple ordered set, with +fast O(1) insertion, deletion, iteration and membership testing. However, `dict` +does not provide the list-like random access features of OrderedSet. You +would have to convert it to a list in O(N) to look up the index of an entry or +look up an entry by its index. + diff --git a/contrib/python/ordered-set/.dist-info/top_level.txt b/contrib/python/ordered-set/.dist-info/top_level.txt new file mode 100644 index 00000000000..1c191eef52b --- /dev/null +++ b/contrib/python/ordered-set/.dist-info/top_level.txt @@ -0,0 +1 @@ +ordered_set diff --git a/contrib/python/ordered-set/MIT-LICENSE b/contrib/python/ordered-set/MIT-LICENSE new file mode 100644 index 00000000000..02185ee4960 --- /dev/null +++ b/contrib/python/ordered-set/MIT-LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012-2022 Elia Robyn Lake + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/ordered-set/README.md b/contrib/python/ordered-set/README.md new file mode 100644 index 00000000000..f66fadda0fc --- /dev/null +++ b/contrib/python/ordered-set/README.md @@ -0,0 +1,133 @@ +[](https://pypi.python.org/pypi/ordered-set) + +An OrderedSet is a mutable data structure that is a hybrid of a list and a set. +It remembers the order of its entries, and every entry has an index number that +can be looked up. + +## Installation + +`ordered_set` is available on PyPI and packaged as a wheel. You can list it +as a dependency of your project, in whatever form that takes. + +To install it into your current Python environment: + + pip install ordered-set + +To install the code for development, after checking out the repository: + + pip install flit + flit install + +## Usage examples + +An OrderedSet is created and used like a set: + + >>> from ordered_set import OrderedSet + + >>> letters = OrderedSet('abracadabra') + + >>> letters + OrderedSet(['a', 'b', 'r', 'c', 'd']) + + >>> 'r' in letters + True + +It is efficient to find the index of an entry in an OrderedSet, or find an +entry by its index. To help with this use case, the `.add()` method returns +the index of the added item, whether it was already in the set or not. + + >>> letters.index('r') + 2 + + >>> letters[2] + 'r' + + >>> letters.add('r') + 2 + + >>> letters.add('x') + 5 + +OrderedSets implement the union (`|`), intersection (`&`), and difference (`-`) +operators like sets do. + + >>> letters |= OrderedSet('shazam') + + >>> letters + OrderedSet(['a', 'b', 'r', 'c', 'd', 'x', 's', 'h', 'z', 'm']) + + >>> letters & set('aeiou') + OrderedSet(['a']) + + >>> letters -= 'abcd' + + >>> letters + OrderedSet(['r', 'x', 's', 'h', 'z', 'm']) + +The `__getitem__()` and `index()` methods have been extended to accept any +iterable except a string, returning a list, to perform NumPy-like "fancy +indexing". + + >>> letters = OrderedSet('abracadabra') + + >>> letters[[0, 2, 3]] + ['a', 'r', 'c'] + + >>> letters.index(['a', 'r', 'c']) + [0, 2, 3] + +OrderedSet implements `__getstate__` and `__setstate__` so it can be pickled, +and implements the abstract base classes `collections.MutableSet` and +`collections.Sequence`. + +OrderedSet can be used as a generic collection type, similar to the collections +in the `typing` module like List, Dict, and Set. For example, you can annotate +a variable as having the type `OrderedSet[str]` or `OrderedSet[Tuple[int, +str]]`. + + +## OrderedSet in data science applications + +An OrderedSet can be used as a bi-directional mapping between a sparse +vocabulary and dense index numbers. As of version 3.1, it accepts NumPy arrays +of index numbers as well as lists. + +This combination of features makes OrderedSet a simple implementation of many +of the things that `pandas.Index` is used for, and many of its operations are +faster than the equivalent pandas operations. + +For further compatibility with pandas.Index, `get_loc` (the pandas method for +looking up a single index) and `get_indexer` (the pandas method for fancy +indexing in reverse) are both aliases for `index` (which handles both cases +in OrderedSet). + + +## Authors + +OrderedSet was implemented by Elia Robyn Lake (maiden name: Robyn Speer). +Jon Crall contributed changes and tests to make it fit the Python set API. +Roman Inflianskas added the original type annotations. + + +## Comparisons + +The original implementation of OrderedSet was a [recipe posted to ActiveState +Recipes][recipe] by Raymond Hettiger, released under the MIT license. + +[recipe]: https://code.activestate.com/recipes/576694-orderedset/ + +Hettiger's implementation kept its content in a doubly-linked list referenced by a +dict. As a result, looking up an item by its index was an O(N) operation, while +deletion was O(1). + +This version makes different trade-offs for the sake of efficient lookups. Its +content is a standard Python list instead of a doubly-linked list. This +provides O(1) lookups by index at the expense of O(N) deletion, as well as +slightly faster iteration. + +In Python 3.6 and later, the built-in `dict` type is inherently ordered. If you +ignore the dictionary values, that also gives you a simple ordered set, with +fast O(1) insertion, deletion, iteration and membership testing. However, `dict` +does not provide the list-like random access features of OrderedSet. You +would have to convert it to a list in O(N) to look up the index of an entry or +look up an entry by its index. diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/ordered_set.py b/contrib/python/ordered-set/ordered_set/__init__.py index 14876000de8..e86c70ed80e 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/ordered_set.py +++ b/contrib/python/ordered-set/ordered_set/__init__.py @@ -1,45 +1,58 @@ """ An OrderedSet is a custom MutableSet that remembers its order, so that every -entry has an index that can be looked up. +entry has an index that can be looked up. It can also act like a Sequence. Based on a recipe originally posted to ActiveState Recipes by Raymond Hettiger, and released under the MIT license. """ import itertools as it -from collections import deque - -try: - # Python 3 - from collections.abc import MutableSet, Sequence -except ImportError: - # Python 2.7 - from collections import MutableSet, Sequence +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + MutableSet, + AbstractSet, + Sequence, + Set, + TypeVar, + Union, + overload, +) SLICE_ALL = slice(None) -__version__ = "3.1" +__version__ = "4.1.0" + + +T = TypeVar("T") +# SetLike[T] is either a set of elements of type T, or a sequence, which +# we will convert to an OrderedSet by adding its elements in order. +SetLike = Union[AbstractSet[T], Sequence[T]] +OrderedSetInitializer = Union[AbstractSet[T], Sequence[T], Iterable[T]] -def is_iterable(obj): + +def _is_atomic(obj: Any) -> bool: """ - Are we being asked to look up a list of things, instead of a single thing? - We check for the `__iter__` attribute so that this can cover types that - don't have to be known by this module, such as NumPy arrays. + Returns True for objects which are iterable but should not be iterated in + the context of indexing an OrderedSet. + + When we index by an iterable, usually that means we're being asked to look + up a list of things. - Strings, however, should be considered as atomic values to look up, not - iterables. The same goes for tuples, since they are immutable and therefore - valid entries. + However, in the case of the .index() method, we shouldn't handle strings + and tuples like other iterables. They're not sequences of things to look + up, they're the single, atomic thing we're trying to find. - We don't need to check for the Python 2 `unicode` type, because it doesn't - have an `__iter__` attribute anyway. + As an example, oset.index('hello') should give the index of 'hello' in an + OrderedSet of strings. It shouldn't give the indexes of each individual + character. """ - return ( - hasattr(obj, "__iter__") - and not isinstance(obj, str) - and not isinstance(obj, tuple) - ) + return isinstance(obj, str) or isinstance(obj, tuple) -class OrderedSet(MutableSet, Sequence): +class OrderedSet(MutableSet[T], Sequence[T]): """ An OrderedSet is a custom MutableSet that remembers its order, so that every entry has an index that can be looked up. @@ -49,11 +62,14 @@ class OrderedSet(MutableSet, Sequence): OrderedSet([1, 2, 3]) """ - def __init__(self, iterable=None): - self.items = [] - self.map = {} - if iterable is not None: - self |= iterable + def __init__(self, initial: OrderedSetInitializer[T] = None): + self.items: List[T] = [] + self.map: Dict[T, int] = {} + if initial is not None: + # In terms of duck-typing, the default __ior__ is compatible with + # the types we use, but it doesn't expect all the types we + # support as values for `initial`. + self |= initial # type: ignore def __len__(self): """ @@ -67,6 +83,19 @@ class OrderedSet(MutableSet, Sequence): """ return len(self.items) + @overload + def __getitem__(self, index: slice) -> "OrderedSet[T]": + ... + + @overload + def __getitem__(self, index: Sequence[int]) -> List[T]: + ... + + @overload + def __getitem__(self, index: int) -> T: + ... + + # concrete implementation def __getitem__(self, index): """ Get the item at a given index. @@ -87,9 +116,9 @@ class OrderedSet(MutableSet, Sequence): """ if isinstance(index, slice) and index == SLICE_ALL: return self.copy() - elif is_iterable(index): + elif isinstance(index, Iterable): return [self.items[i] for i in index] - elif hasattr(index, "__index__") or isinstance(index, slice): + elif isinstance(index, slice) or hasattr(index, "__index__"): result = self.items[index] if isinstance(result, list): return self.__class__(result) @@ -98,7 +127,7 @@ class OrderedSet(MutableSet, Sequence): else: raise TypeError("Don't know how to index an OrderedSet by %r" % index) - def copy(self): + def copy(self) -> "OrderedSet[T]": """ Return a shallow copy of this object. @@ -112,9 +141,12 @@ class OrderedSet(MutableSet, Sequence): """ return self.__class__(self) + # Define the gritty details of how an OrderedSet is serialized as a pickle. + # We leave off type annotations, because the only code that should interact + # with these is a generalized tool such as pickle. def __getstate__(self): if len(self) == 0: - # The state can't be an empty list. + # In pickle, the state can't be an empty list. # We need to return a truthy value, or else __setstate__ won't be run. # # This could have been done more gracefully by always putting the state @@ -130,9 +162,9 @@ class OrderedSet(MutableSet, Sequence): else: self.__init__(state) - def __contains__(self, key): + def __contains__(self, key: Any) -> bool: """ - Test if the item is in this ordered set + Test if the item is in this ordered set. Example: >>> 1 in OrderedSet([1, 3, 2]) @@ -142,7 +174,10 @@ class OrderedSet(MutableSet, Sequence): """ return key in self.map - def add(self, key): + # Technically type-incompatible with MutableSet, because we return an + # int instead of nothing. This is also one of the things that makes + # OrderedSet convenient to use. + def add(self, key: T) -> int: """ Add `key` as an item to this OrderedSet, then return its index. @@ -163,7 +198,7 @@ class OrderedSet(MutableSet, Sequence): append = add - def update(self, sequence): + def update(self, sequence: SetLike[T]) -> int: """ Update the set with the given iterable sequence, then return the index of the last element inserted. @@ -175,7 +210,7 @@ class OrderedSet(MutableSet, Sequence): >>> print(oset) OrderedSet([1, 2, 3, 5, 4]) """ - item_index = None + item_index = 0 try: for item in sequence: item_index = self.add(item) @@ -185,6 +220,15 @@ class OrderedSet(MutableSet, Sequence): ) return item_index + @overload + def index(self, key: Sequence[T]) -> List[int]: + ... + + @overload + def index(self, key: T) -> int: + ... + + # concrete implementation def index(self, key): """ Get the index of a given entry, raising an IndexError if it's not @@ -198,7 +242,7 @@ class OrderedSet(MutableSet, Sequence): >>> oset.index(2) 1 """ - if is_iterable(key): + if isinstance(key, Iterable) and not _is_atomic(key): return [self.index(subkey) for subkey in key] return self.map[key] @@ -206,11 +250,12 @@ class OrderedSet(MutableSet, Sequence): get_loc = index get_indexer = index - def pop(self): + def pop(self, index=-1) -> T: """ - Remove and return the last element from the set. + Remove and return item at index (default last). Raises KeyError if the set is empty. + Raises IndexError if index is out of range. Example: >>> oset = OrderedSet([1, 2, 3]) @@ -220,12 +265,12 @@ class OrderedSet(MutableSet, Sequence): if not self.items: raise KeyError("Set is empty") - elem = self.items[-1] - del self.items[-1] + elem = self.items[index] + del self.items[index] del self.map[elem] return elem - def discard(self, key): + def discard(self, key: T) -> None: """ Remove an element. Do not raise an exception if absent. @@ -249,14 +294,14 @@ class OrderedSet(MutableSet, Sequence): if v >= i: self.map[k] = v - 1 - def clear(self): + def clear(self) -> None: """ Remove all items from this OrderedSet. """ del self.items[:] self.map.clear() - def __iter__(self): + def __iter__(self) -> Iterator[T]: """ Example: >>> list(iter(OrderedSet([1, 2, 3]))) @@ -264,7 +309,7 @@ class OrderedSet(MutableSet, Sequence): """ return iter(self.items) - def __reversed__(self): + def __reversed__(self) -> Iterator[T]: """ Example: >>> list(reversed(OrderedSet([1, 2, 3]))) @@ -272,12 +317,12 @@ class OrderedSet(MutableSet, Sequence): """ return reversed(self.items) - def __repr__(self): + def __repr__(self) -> str: if not self: return "%s()" % (self.__class__.__name__,) return "%s(%r)" % (self.__class__.__name__, list(self)) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: """ Returns true if the containers have the same items. If `other` is a Sequence, then order is checked, otherwise it is ignored. @@ -293,9 +338,7 @@ class OrderedSet(MutableSet, Sequence): >>> oset == OrderedSet([3, 2, 1]) False """ - # In Python 2 deque is not a Sequence, so treat it as one for - # consistent behavior with Python 3. - if isinstance(other, (Sequence, deque)): + if isinstance(other, Sequence): # Check that this OrderedSet contains the same elements, in the # same order, as the other object. return list(self) == list(other) @@ -307,7 +350,7 @@ class OrderedSet(MutableSet, Sequence): else: return set(self) == other_as_set - def union(self, *sets): + def union(self, *sets: SetLike[T]) -> "OrderedSet[T]": """ Combines all unique items. Each items order is defined by its first appearance. @@ -321,16 +364,18 @@ class OrderedSet(MutableSet, Sequence): >>> oset | {10} OrderedSet([3, 1, 4, 5, 2, 0, 10]) """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet + cls: type = OrderedSet + if isinstance(self, OrderedSet): + cls = self.__class__ containers = map(list, it.chain([self], sets)) items = it.chain.from_iterable(containers) return cls(items) - def __and__(self, other): + def __and__(self, other: SetLike[T]) -> "OrderedSet[T]": # the parent implementation of this is backwards return self.intersection(other) - def intersection(self, *sets): + def intersection(self, *sets: SetLike[T]) -> "OrderedSet[T]": """ Returns elements in common between all sets. Order is defined only by the first set. @@ -344,15 +389,16 @@ class OrderedSet(MutableSet, Sequence): >>> oset.intersection() OrderedSet([1, 2, 3]) """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet + cls: type = OrderedSet + items: OrderedSetInitializer[T] = self + if isinstance(self, OrderedSet): + cls = self.__class__ if sets: common = set.intersection(*map(set, sets)) items = (item for item in self if item in common) - else: - items = self return cls(items) - def difference(self, *sets): + def difference(self, *sets: SetLike[T]) -> "OrderedSet[T]": """ Returns all elements that are in this set but not the others. @@ -367,14 +413,13 @@ class OrderedSet(MutableSet, Sequence): OrderedSet([1, 2, 3]) """ cls = self.__class__ + items: OrderedSetInitializer[T] = self if sets: other = set.union(*map(set, sets)) items = (item for item in self if item not in other) - else: - items = self return cls(items) - def issubset(self, other): + def issubset(self, other: SetLike[T]) -> bool: """ Report whether another set contains this set. @@ -390,7 +435,7 @@ class OrderedSet(MutableSet, Sequence): return False return all(item in other for item in self) - def issuperset(self, other): + def issuperset(self, other: SetLike[T]) -> bool: """ Report whether this set contains another set. @@ -406,7 +451,7 @@ class OrderedSet(MutableSet, Sequence): return False return all(item in self for item in other) - def symmetric_difference(self, other): + def symmetric_difference(self, other: SetLike[T]) -> "OrderedSet[T]": """ Return the symmetric difference of two OrderedSets as a new set. That is, the new set will contain all elements that are in exactly @@ -421,12 +466,14 @@ class OrderedSet(MutableSet, Sequence): >>> this.symmetric_difference(other) OrderedSet([4, 5, 9, 2]) """ - cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet + cls: type = OrderedSet + if isinstance(self, OrderedSet): + cls = self.__class__ diff1 = cls(self).difference(other) diff2 = cls(other).difference(self) return diff1.union(diff2) - def _update_items(self, items): + def _update_items(self, items: list) -> None: """ Replace the 'items' list of this OrderedSet with a new one, updating self.map accordingly. @@ -434,7 +481,7 @@ class OrderedSet(MutableSet, Sequence): self.items = items self.map = {item: idx for (idx, item) in enumerate(items)} - def difference_update(self, *sets): + def difference_update(self, *sets: SetLike[T]) -> None: """ Update this OrderedSet to remove items from one or more other sets. @@ -449,12 +496,13 @@ class OrderedSet(MutableSet, Sequence): >>> print(this) OrderedSet([3, 5]) """ - items_to_remove = set() + items_to_remove = set() # type: Set[T] for other in sets: - items_to_remove |= set(other) + items_as_set = set(other) # type: Set[T] + items_to_remove |= items_as_set self._update_items([item for item in self.items if item not in items_to_remove]) - def intersection_update(self, other): + def intersection_update(self, other: SetLike[T]) -> None: """ Update this OrderedSet to keep only items in another set, preserving their order in this set. @@ -469,7 +517,7 @@ class OrderedSet(MutableSet, Sequence): other = set(other) self._update_items([item for item in self.items if item in other]) - def symmetric_difference_update(self, other): + def symmetric_difference_update(self, other: SetLike[T]) -> None: """ Update this OrderedSet to remove items from another set, then add items from the other set that were not present in this set. diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/py.typed b/contrib/python/ordered-set/ordered_set/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/py.typed +++ b/contrib/python/ordered-set/ordered_set/py.typed diff --git a/contrib/python/ordered-set/ya.make b/contrib/python/ordered-set/ya.make new file mode 100644 index 00000000000..2d052f4c9f4 --- /dev/null +++ b/contrib/python/ordered-set/ya.make @@ -0,0 +1,23 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(4.1.0) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + ordered_set/__init__.py +) + +RESOURCE_FILES( + PREFIX contrib/python/ordered-set/ + .dist-info/METADATA + .dist-info/top_level.txt + ordered_set/py.typed +) + +END() diff --git a/contrib/python/platformdirs/.dist-info/METADATA b/contrib/python/platformdirs/.dist-info/METADATA new file mode 100644 index 00000000000..91c59c9a280 --- /dev/null +++ b/contrib/python/platformdirs/.dist-info/METADATA @@ -0,0 +1,327 @@ +Metadata-Version: 2.3 +Name: platformdirs +Version: 4.3.6 +Summary: A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`. +Project-URL: Changelog, https://github.com/tox-dev/platformdirs/releases +Project-URL: Documentation, https://platformdirs.readthedocs.io +Project-URL: Homepage, https://github.com/tox-dev/platformdirs +Project-URL: Source, https://github.com/tox-dev/platformdirs +Project-URL: Tracker, https://github.com/tox-dev/platformdirs/issues +Maintainer-email: Bernát Gábor <gaborjbernat@gmail.com>, Julian Berman <Julian@GrayVines.com>, Ofek Lev <oss@ofek.dev>, Ronny Pfannschmidt <opensource@ronnypfannschmidt.de> +License-Expression: MIT +License-File: LICENSE +Keywords: appdirs,application,cache,directory,log,user +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.8 +Provides-Extra: docs +Requires-Dist: furo>=2024.8.6; extra == 'docs' +Requires-Dist: proselint>=0.14; extra == 'docs' +Requires-Dist: sphinx-autodoc-typehints>=2.4; extra == 'docs' +Requires-Dist: sphinx>=8.0.2; extra == 'docs' +Provides-Extra: test +Requires-Dist: appdirs==1.4.4; extra == 'test' +Requires-Dist: covdefaults>=2.3; extra == 'test' +Requires-Dist: pytest-cov>=5; extra == 'test' +Requires-Dist: pytest-mock>=3.14; extra == 'test' +Requires-Dist: pytest>=8.3.2; extra == 'test' +Provides-Extra: type +Requires-Dist: mypy>=1.11.2; extra == 'type' +Description-Content-Type: text/x-rst + +The problem +=========== + +.. image:: https://badge.fury.io/py/platformdirs.svg + :target: https://badge.fury.io/py/platformdirs +.. image:: https://img.shields.io/pypi/pyversions/platformdirs.svg + :target: https://pypi.python.org/pypi/platformdirs/ +.. image:: https://github.com/tox-dev/platformdirs/actions/workflows/check.yaml/badge.svg + :target: https://github.com/platformdirs/platformdirs/actions +.. image:: https://static.pepy.tech/badge/platformdirs/month + :target: https://pepy.tech/project/platformdirs + +When writing desktop application, finding the right location to store user data +and configuration varies per platform. Even for single-platform apps, there +may by plenty of nuances in figuring out the right location. + +For example, if running on macOS, you should use:: + + ~/Library/Application Support/<AppName> + +If on Windows (at least English Win) that should be:: + + C:\Documents and Settings\<User>\Application Data\Local Settings\<AppAuthor>\<AppName> + +or possibly:: + + C:\Documents and Settings\<User>\Application Data\<AppAuthor>\<AppName> + +for `roaming profiles <https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-vista/cc766489(v=ws.10)>`_ but that is another story. + +On Linux (and other Unices), according to the `XDG Basedir Spec`_, it should be:: + + ~/.local/share/<AppName> + +.. _XDG Basedir Spec: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +``platformdirs`` to the rescue +============================== + +This kind of thing is what the ``platformdirs`` package is for. +``platformdirs`` will help you choose an appropriate: + +- user data dir (``user_data_dir``) +- user config dir (``user_config_dir``) +- user cache dir (``user_cache_dir``) +- site data dir (``site_data_dir``) +- site config dir (``site_config_dir``) +- user log dir (``user_log_dir``) +- user documents dir (``user_documents_dir``) +- user downloads dir (``user_downloads_dir``) +- user pictures dir (``user_pictures_dir``) +- user videos dir (``user_videos_dir``) +- user music dir (``user_music_dir``) +- user desktop dir (``user_desktop_dir``) +- user runtime dir (``user_runtime_dir``) + +And also: + +- Is slightly opinionated on the directory names used. Look for "OPINION" in + documentation and code for when an opinion is being applied. + +Example output +============== + +On macOS: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/Users/trentm/Library/Application Support/SuperApp' + >>> site_data_dir(appname, appauthor) + '/Library/Application Support/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/Users/trentm/Library/Caches/SuperApp' + >>> user_log_dir(appname, appauthor) + '/Users/trentm/Library/Logs/SuperApp' + >>> user_documents_dir() + '/Users/trentm/Documents' + >>> user_downloads_dir() + '/Users/trentm/Downloads' + >>> user_pictures_dir() + '/Users/trentm/Pictures' + >>> user_videos_dir() + '/Users/trentm/Movies' + >>> user_music_dir() + '/Users/trentm/Music' + >>> user_desktop_dir() + '/Users/trentm/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp' + +On Windows: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' + >>> user_data_dir(appname, appauthor, roaming=True) + 'C:\\Users\\trentm\\AppData\\Roaming\\Acme\\SuperApp' + >>> user_cache_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Cache' + >>> user_log_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Logs' + >>> user_documents_dir() + 'C:\\Users\\trentm\\Documents' + >>> user_downloads_dir() + 'C:\\Users\\trentm\\Downloads' + >>> user_pictures_dir() + 'C:\\Users\\trentm\\Pictures' + >>> user_videos_dir() + 'C:\\Users\\trentm\\Videos' + >>> user_music_dir() + 'C:\\Users\\trentm\\Music' + >>> user_desktop_dir() + 'C:\\Users\\trentm\\Desktop' + >>> user_runtime_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Temp\\Acme\\SuperApp' + +On Linux: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/home/trentm/.local/share/SuperApp' + >>> site_data_dir(appname, appauthor) + '/usr/local/share/SuperApp' + >>> site_data_dir(appname, appauthor, multipath=True) + '/usr/local/share/SuperApp:/usr/share/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/home/trentm/.cache/SuperApp' + >>> user_log_dir(appname, appauthor) + '/home/trentm/.local/state/SuperApp/log' + >>> user_config_dir(appname) + '/home/trentm/.config/SuperApp' + >>> user_documents_dir() + '/home/trentm/Documents' + >>> user_downloads_dir() + '/home/trentm/Downloads' + >>> user_pictures_dir() + '/home/trentm/Pictures' + >>> user_videos_dir() + '/home/trentm/Videos' + >>> user_music_dir() + '/home/trentm/Music' + >>> user_desktop_dir() + '/home/trentm/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/run/user/{os.getuid()}/SuperApp' + >>> site_config_dir(appname) + '/etc/xdg/SuperApp' + >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" + >>> site_config_dir(appname, multipath=True) + '/etc/SuperApp:/usr/local/etc/SuperApp' + +On Android:: + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/data/data/com.myApp/files/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp' + >>> user_log_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp/log' + >>> user_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' + >>> user_documents_dir() + '/storage/emulated/0/Documents' + >>> user_downloads_dir() + '/storage/emulated/0/Downloads' + >>> user_pictures_dir() + '/storage/emulated/0/Pictures' + >>> user_videos_dir() + '/storage/emulated/0/DCIM/Camera' + >>> user_music_dir() + '/storage/emulated/0/Music' + >>> user_desktop_dir() + '/storage/emulated/0/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp/tmp' + +Note: Some android apps like Termux and Pydroid are used as shells. These +apps are used by the end user to emulate Linux environment. Presence of +``SHELL`` environment variable is used by Platformdirs to differentiate +between general android apps and android apps used as shells. Shell android +apps also support ``XDG_*`` environment variables. + + +``PlatformDirs`` for convenience +================================ + +.. code-block:: pycon + + >>> from platformdirs import PlatformDirs + >>> dirs = PlatformDirs("SuperApp", "Acme") + >>> dirs.user_data_dir + '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp' + >>> dirs.user_log_dir + '/Users/trentm/Library/Logs/SuperApp' + >>> dirs.user_documents_dir + '/Users/trentm/Documents' + >>> dirs.user_downloads_dir + '/Users/trentm/Downloads' + >>> dirs.user_pictures_dir + '/Users/trentm/Pictures' + >>> dirs.user_videos_dir + '/Users/trentm/Movies' + >>> dirs.user_music_dir + '/Users/trentm/Music' + >>> dirs.user_desktop_dir + '/Users/trentm/Desktop' + >>> dirs.user_runtime_dir + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp' + +Per-version isolation +===================== + +If you have multiple versions of your app in use that you want to be +able to run side-by-side, then you may want version-isolation for these +dirs:: + + >>> from platformdirs import PlatformDirs + >>> dirs = PlatformDirs("SuperApp", "Acme", version="1.0") + >>> dirs.user_data_dir + '/Users/trentm/Library/Application Support/SuperApp/1.0' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp/1.0' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp/1.0' + >>> dirs.user_log_dir + '/Users/trentm/Library/Logs/SuperApp/1.0' + >>> dirs.user_documents_dir + '/Users/trentm/Documents' + >>> dirs.user_downloads_dir + '/Users/trentm/Downloads' + >>> dirs.user_pictures_dir + '/Users/trentm/Pictures' + >>> dirs.user_videos_dir + '/Users/trentm/Movies' + >>> dirs.user_music_dir + '/Users/trentm/Music' + >>> dirs.user_desktop_dir + '/Users/trentm/Desktop' + >>> dirs.user_runtime_dir + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp/1.0' + +Be wary of using this for configuration files though; you'll need to handle +migrating configuration files manually. + +Why this Fork? +============== + +This repository is a friendly fork of the wonderful work started by +`ActiveState <https://github.com/ActiveState/appdirs>`_ who created +``appdirs``, this package's ancestor. + +Maintaining an open source project is no easy task, particularly +from within an organization, and the Python community is indebted +to ``appdirs`` (and to Trent Mick and Jeff Rouse in particular) for +creating an incredibly useful simple module, as evidenced by the wide +number of users it has attracted over the years. + +Nonetheless, given the number of long-standing open issues +and pull requests, and no clear path towards `ensuring +that maintenance of the package would continue or grow +<https://github.com/ActiveState/appdirs/issues/79>`_, this fork was +created. + +Contributions are most welcome. diff --git a/contrib/python/platformdirs/.dist-info/top_level.txt b/contrib/python/platformdirs/.dist-info/top_level.txt new file mode 100644 index 00000000000..67fd014bbdd --- /dev/null +++ b/contrib/python/platformdirs/.dist-info/top_level.txt @@ -0,0 +1 @@ +platformdirs diff --git a/contrib/python/platformdirs/LICENSE b/contrib/python/platformdirs/LICENSE new file mode 100644 index 00000000000..f35fed9191b --- /dev/null +++ b/contrib/python/platformdirs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2010-202x The platformdirs developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/python/platformdirs/README.rst b/contrib/python/platformdirs/README.rst new file mode 100644 index 00000000000..1562ecb5906 --- /dev/null +++ b/contrib/python/platformdirs/README.rst @@ -0,0 +1,283 @@ +The problem +=========== + +.. image:: https://badge.fury.io/py/platformdirs.svg + :target: https://badge.fury.io/py/platformdirs +.. image:: https://img.shields.io/pypi/pyversions/platformdirs.svg + :target: https://pypi.python.org/pypi/platformdirs/ +.. image:: https://github.com/tox-dev/platformdirs/actions/workflows/check.yaml/badge.svg + :target: https://github.com/platformdirs/platformdirs/actions +.. image:: https://static.pepy.tech/badge/platformdirs/month + :target: https://pepy.tech/project/platformdirs + +When writing desktop application, finding the right location to store user data +and configuration varies per platform. Even for single-platform apps, there +may by plenty of nuances in figuring out the right location. + +For example, if running on macOS, you should use:: + + ~/Library/Application Support/<AppName> + +If on Windows (at least English Win) that should be:: + + C:\Documents and Settings\<User>\Application Data\Local Settings\<AppAuthor>\<AppName> + +or possibly:: + + C:\Documents and Settings\<User>\Application Data\<AppAuthor>\<AppName> + +for `roaming profiles <https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-vista/cc766489(v=ws.10)>`_ but that is another story. + +On Linux (and other Unices), according to the `XDG Basedir Spec`_, it should be:: + + ~/.local/share/<AppName> + +.. _XDG Basedir Spec: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +``platformdirs`` to the rescue +============================== + +This kind of thing is what the ``platformdirs`` package is for. +``platformdirs`` will help you choose an appropriate: + +- user data dir (``user_data_dir``) +- user config dir (``user_config_dir``) +- user cache dir (``user_cache_dir``) +- site data dir (``site_data_dir``) +- site config dir (``site_config_dir``) +- user log dir (``user_log_dir``) +- user documents dir (``user_documents_dir``) +- user downloads dir (``user_downloads_dir``) +- user pictures dir (``user_pictures_dir``) +- user videos dir (``user_videos_dir``) +- user music dir (``user_music_dir``) +- user desktop dir (``user_desktop_dir``) +- user runtime dir (``user_runtime_dir``) + +And also: + +- Is slightly opinionated on the directory names used. Look for "OPINION" in + documentation and code for when an opinion is being applied. + +Example output +============== + +On macOS: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/Users/trentm/Library/Application Support/SuperApp' + >>> site_data_dir(appname, appauthor) + '/Library/Application Support/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/Users/trentm/Library/Caches/SuperApp' + >>> user_log_dir(appname, appauthor) + '/Users/trentm/Library/Logs/SuperApp' + >>> user_documents_dir() + '/Users/trentm/Documents' + >>> user_downloads_dir() + '/Users/trentm/Downloads' + >>> user_pictures_dir() + '/Users/trentm/Pictures' + >>> user_videos_dir() + '/Users/trentm/Movies' + >>> user_music_dir() + '/Users/trentm/Music' + >>> user_desktop_dir() + '/Users/trentm/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp' + +On Windows: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' + >>> user_data_dir(appname, appauthor, roaming=True) + 'C:\\Users\\trentm\\AppData\\Roaming\\Acme\\SuperApp' + >>> user_cache_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Cache' + >>> user_log_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Logs' + >>> user_documents_dir() + 'C:\\Users\\trentm\\Documents' + >>> user_downloads_dir() + 'C:\\Users\\trentm\\Downloads' + >>> user_pictures_dir() + 'C:\\Users\\trentm\\Pictures' + >>> user_videos_dir() + 'C:\\Users\\trentm\\Videos' + >>> user_music_dir() + 'C:\\Users\\trentm\\Music' + >>> user_desktop_dir() + 'C:\\Users\\trentm\\Desktop' + >>> user_runtime_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Temp\\Acme\\SuperApp' + +On Linux: + +.. code-block:: pycon + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/home/trentm/.local/share/SuperApp' + >>> site_data_dir(appname, appauthor) + '/usr/local/share/SuperApp' + >>> site_data_dir(appname, appauthor, multipath=True) + '/usr/local/share/SuperApp:/usr/share/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/home/trentm/.cache/SuperApp' + >>> user_log_dir(appname, appauthor) + '/home/trentm/.local/state/SuperApp/log' + >>> user_config_dir(appname) + '/home/trentm/.config/SuperApp' + >>> user_documents_dir() + '/home/trentm/Documents' + >>> user_downloads_dir() + '/home/trentm/Downloads' + >>> user_pictures_dir() + '/home/trentm/Pictures' + >>> user_videos_dir() + '/home/trentm/Videos' + >>> user_music_dir() + '/home/trentm/Music' + >>> user_desktop_dir() + '/home/trentm/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/run/user/{os.getuid()}/SuperApp' + >>> site_config_dir(appname) + '/etc/xdg/SuperApp' + >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" + >>> site_config_dir(appname, multipath=True) + '/etc/SuperApp:/usr/local/etc/SuperApp' + +On Android:: + + >>> from platformdirs import * + >>> appname = "SuperApp" + >>> appauthor = "Acme" + >>> user_data_dir(appname, appauthor) + '/data/data/com.myApp/files/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp' + >>> user_log_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp/log' + >>> user_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' + >>> user_documents_dir() + '/storage/emulated/0/Documents' + >>> user_downloads_dir() + '/storage/emulated/0/Downloads' + >>> user_pictures_dir() + '/storage/emulated/0/Pictures' + >>> user_videos_dir() + '/storage/emulated/0/DCIM/Camera' + >>> user_music_dir() + '/storage/emulated/0/Music' + >>> user_desktop_dir() + '/storage/emulated/0/Desktop' + >>> user_runtime_dir(appname, appauthor) + '/data/data/com.myApp/cache/SuperApp/tmp' + +Note: Some android apps like Termux and Pydroid are used as shells. These +apps are used by the end user to emulate Linux environment. Presence of +``SHELL`` environment variable is used by Platformdirs to differentiate +between general android apps and android apps used as shells. Shell android +apps also support ``XDG_*`` environment variables. + + +``PlatformDirs`` for convenience +================================ + +.. code-block:: pycon + + >>> from platformdirs import PlatformDirs + >>> dirs = PlatformDirs("SuperApp", "Acme") + >>> dirs.user_data_dir + '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp' + >>> dirs.user_log_dir + '/Users/trentm/Library/Logs/SuperApp' + >>> dirs.user_documents_dir + '/Users/trentm/Documents' + >>> dirs.user_downloads_dir + '/Users/trentm/Downloads' + >>> dirs.user_pictures_dir + '/Users/trentm/Pictures' + >>> dirs.user_videos_dir + '/Users/trentm/Movies' + >>> dirs.user_music_dir + '/Users/trentm/Music' + >>> dirs.user_desktop_dir + '/Users/trentm/Desktop' + >>> dirs.user_runtime_dir + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp' + +Per-version isolation +===================== + +If you have multiple versions of your app in use that you want to be +able to run side-by-side, then you may want version-isolation for these +dirs:: + + >>> from platformdirs import PlatformDirs + >>> dirs = PlatformDirs("SuperApp", "Acme", version="1.0") + >>> dirs.user_data_dir + '/Users/trentm/Library/Application Support/SuperApp/1.0' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp/1.0' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp/1.0' + >>> dirs.user_log_dir + '/Users/trentm/Library/Logs/SuperApp/1.0' + >>> dirs.user_documents_dir + '/Users/trentm/Documents' + >>> dirs.user_downloads_dir + '/Users/trentm/Downloads' + >>> dirs.user_pictures_dir + '/Users/trentm/Pictures' + >>> dirs.user_videos_dir + '/Users/trentm/Movies' + >>> dirs.user_music_dir + '/Users/trentm/Music' + >>> dirs.user_desktop_dir + '/Users/trentm/Desktop' + >>> dirs.user_runtime_dir + '/Users/trentm/Library/Caches/TemporaryItems/SuperApp/1.0' + +Be wary of using this for configuration files though; you'll need to handle +migrating configuration files manually. + +Why this Fork? +============== + +This repository is a friendly fork of the wonderful work started by +`ActiveState <https://github.com/ActiveState/appdirs>`_ who created +``appdirs``, this package's ancestor. + +Maintaining an open source project is no easy task, particularly +from within an organization, and the Python community is indebted +to ``appdirs`` (and to Trent Mick and Jeff Rouse in particular) for +creating an incredibly useful simple module, as evidenced by the wide +number of users it has attracted over the years. + +Nonetheless, given the number of long-standing open issues +and pull requests, and no clear path towards `ensuring +that maintenance of the package would continue or grow +<https://github.com/ActiveState/appdirs/issues/79>`_, this fork was +created. + +Contributions are most welcome. diff --git a/contrib/python/platformdirs/platformdirs/__init__.py b/contrib/python/platformdirs/platformdirs/__init__.py new file mode 100644 index 00000000000..afe8351d203 --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/__init__.py @@ -0,0 +1,631 @@ +""" +Utilities for determining application-specific dirs. + +See <https://github.com/platformdirs/platformdirs> for details and usage. + +""" + +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING + +from .api import PlatformDirsABC +from .version import __version__ +from .version import __version_tuple__ as __version_info__ + +if TYPE_CHECKING: + from pathlib import Path + from typing import Literal + +if sys.platform == "win32": + from platformdirs.windows import Windows as _Result +elif sys.platform == "darwin": + from platformdirs.macos import MacOS as _Result +else: + from platformdirs.unix import Unix as _Result + + +def _set_platform_dir_class() -> type[PlatformDirsABC]: + if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system": + if os.getenv("SHELL") or os.getenv("PREFIX"): + return _Result + + from platformdirs.android import _android_folder # noqa: PLC0415 + + if _android_folder() is not None: + from platformdirs.android import Android # noqa: PLC0415 + + return Android # return to avoid redefinition of a result + + return _Result + + +if TYPE_CHECKING: + # Work around mypy issue: https://github.com/python/mypy/issues/10962 + PlatformDirs = _Result +else: + PlatformDirs = _set_platform_dir_class() #: Currently active platform +AppDirs = PlatformDirs #: Backwards compatibility with appdirs + + +def user_data_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: data directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_data_dir + + +def site_data_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: data directory shared by users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + multipath=multipath, + ensure_exists=ensure_exists, + ).site_data_dir + + +def user_config_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: config directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_config_dir + + +def site_config_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: config directory shared by the users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + multipath=multipath, + ensure_exists=ensure_exists, + ).site_config_dir + + +def user_cache_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: cache directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_cache_dir + + +def site_cache_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: cache directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).site_cache_dir + + +def user_state_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: state directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_state_dir + + +def user_log_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: log directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_log_dir + + +def user_documents_dir() -> str: + """:returns: documents directory tied to the user""" + return PlatformDirs().user_documents_dir + + +def user_downloads_dir() -> str: + """:returns: downloads directory tied to the user""" + return PlatformDirs().user_downloads_dir + + +def user_pictures_dir() -> str: + """:returns: pictures directory tied to the user""" + return PlatformDirs().user_pictures_dir + + +def user_videos_dir() -> str: + """:returns: videos directory tied to the user""" + return PlatformDirs().user_videos_dir + + +def user_music_dir() -> str: + """:returns: music directory tied to the user""" + return PlatformDirs().user_music_dir + + +def user_desktop_dir() -> str: + """:returns: desktop directory tied to the user""" + return PlatformDirs().user_desktop_dir + + +def user_runtime_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: runtime directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_runtime_dir + + +def site_runtime_dir( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> str: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: runtime directory shared by users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).site_runtime_dir + + +def user_data_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: data path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_data_path + + +def site_data_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param multipath: See `multipath <platformdirs.api.PlatformDirsABC.multipath>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: data path shared by users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + multipath=multipath, + ensure_exists=ensure_exists, + ).site_data_path + + +def user_config_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: config path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_config_path + + +def site_config_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: config path shared by the users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + multipath=multipath, + ensure_exists=ensure_exists, + ).site_config_path + + +def site_cache_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: cache directory tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).site_cache_path + + +def user_cache_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: cache path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_cache_path + + +def user_state_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.roaming>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: state path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + roaming=roaming, + ensure_exists=ensure_exists, + ).user_state_path + + +def user_log_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: log path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_log_path + + +def user_documents_path() -> Path: + """:returns: documents a path tied to the user""" + return PlatformDirs().user_documents_path + + +def user_downloads_path() -> Path: + """:returns: downloads path tied to the user""" + return PlatformDirs().user_downloads_path + + +def user_pictures_path() -> Path: + """:returns: pictures path tied to the user""" + return PlatformDirs().user_pictures_path + + +def user_videos_path() -> Path: + """:returns: videos path tied to the user""" + return PlatformDirs().user_videos_path + + +def user_music_path() -> Path: + """:returns: music path tied to the user""" + return PlatformDirs().user_music_path + + +def user_desktop_path() -> Path: + """:returns: desktop path tied to the user""" + return PlatformDirs().user_desktop_path + + +def user_runtime_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: runtime path tied to the user + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).user_runtime_path + + +def site_runtime_path( + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 +) -> Path: + """ + :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. + :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. + :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. + :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. + :param ensure_exists: See `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + :returns: runtime path shared by users + """ + return PlatformDirs( + appname=appname, + appauthor=appauthor, + version=version, + opinion=opinion, + ensure_exists=ensure_exists, + ).site_runtime_path + + +__all__ = [ + "AppDirs", + "PlatformDirs", + "PlatformDirsABC", + "__version__", + "__version_info__", + "site_cache_dir", + "site_cache_path", + "site_config_dir", + "site_config_path", + "site_data_dir", + "site_data_path", + "site_runtime_dir", + "site_runtime_path", + "user_cache_dir", + "user_cache_path", + "user_config_dir", + "user_config_path", + "user_data_dir", + "user_data_path", + "user_desktop_dir", + "user_desktop_path", + "user_documents_dir", + "user_documents_path", + "user_downloads_dir", + "user_downloads_path", + "user_log_dir", + "user_log_path", + "user_music_dir", + "user_music_path", + "user_pictures_dir", + "user_pictures_path", + "user_runtime_dir", + "user_runtime_path", + "user_state_dir", + "user_state_path", + "user_videos_dir", + "user_videos_path", +] diff --git a/contrib/python/platformdirs/platformdirs/__main__.py b/contrib/python/platformdirs/platformdirs/__main__.py new file mode 100644 index 00000000000..922c521358e --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/__main__.py @@ -0,0 +1,55 @@ +"""Main entry point.""" + +from __future__ import annotations + +from platformdirs import PlatformDirs, __version__ + +PROPS = ( + "user_data_dir", + "user_config_dir", + "user_cache_dir", + "user_state_dir", + "user_log_dir", + "user_documents_dir", + "user_downloads_dir", + "user_pictures_dir", + "user_videos_dir", + "user_music_dir", + "user_runtime_dir", + "site_data_dir", + "site_config_dir", + "site_cache_dir", + "site_runtime_dir", +) + + +def main() -> None: + """Run the main entry point.""" + app_name = "MyApp" + app_author = "MyCompany" + + print(f"-- platformdirs {__version__} --") # noqa: T201 + + print("-- app dirs (with optional 'version')") # noqa: T201 + dirs = PlatformDirs(app_name, app_author, version="1.0") + for prop in PROPS: + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 + + print("\n-- app dirs (without optional 'version')") # noqa: T201 + dirs = PlatformDirs(app_name, app_author) + for prop in PROPS: + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 + + print("\n-- app dirs (without optional 'appauthor')") # noqa: T201 + dirs = PlatformDirs(app_name) + for prop in PROPS: + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 + + print("\n-- app dirs (with disabled 'appauthor')") # noqa: T201 + dirs = PlatformDirs(app_name, appauthor=False) + for prop in PROPS: + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/contrib/python/platformdirs/platformdirs/android.py b/contrib/python/platformdirs/platformdirs/android.py new file mode 100644 index 00000000000..7004a852422 --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/android.py @@ -0,0 +1,249 @@ +"""Android.""" + +from __future__ import annotations + +import os +import re +import sys +from functools import lru_cache +from typing import TYPE_CHECKING, cast + +from .api import PlatformDirsABC + + +class Android(PlatformDirsABC): + """ + Follows the guidance `from here <https://android.stackexchange.com/a/216132>`_. + + Makes use of the `appname <platformdirs.api.PlatformDirsABC.appname>`, `version + <platformdirs.api.PlatformDirsABC.version>`, `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + + """ + + @property + def user_data_dir(self) -> str: + """:return: data directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/files/<AppName>``""" + return self._append_app_name_and_version(cast(str, _android_folder()), "files") + + @property + def site_data_dir(self) -> str: + """:return: data directory shared by users, same as `user_data_dir`""" + return self.user_data_dir + + @property + def user_config_dir(self) -> str: + """ + :return: config directory tied to the user, e.g. \ + ``/data/user/<userid>/<packagename>/shared_prefs/<AppName>`` + """ + return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") + + @property + def site_config_dir(self) -> str: + """:return: config directory shared by the users, same as `user_config_dir`""" + return self.user_config_dir + + @property + def user_cache_dir(self) -> str: + """:return: cache directory tied to the user, e.g.,``/data/user/<userid>/<packagename>/cache/<AppName>``""" + return self._append_app_name_and_version(cast(str, _android_folder()), "cache") + + @property + def site_cache_dir(self) -> str: + """:return: cache directory shared by users, same as `user_cache_dir`""" + return self.user_cache_dir + + @property + def user_state_dir(self) -> str: + """:return: state directory tied to the user, same as `user_data_dir`""" + return self.user_data_dir + + @property + def user_log_dir(self) -> str: + """ + :return: log directory tied to the user, same as `user_cache_dir` if not opinionated else ``log`` in it, + e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/log`` + """ + path = self.user_cache_dir + if self.opinion: + path = os.path.join(path, "log") # noqa: PTH118 + return path + + @property + def user_documents_dir(self) -> str: + """:return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents``""" + return _android_documents_folder() + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``/storage/emulated/0/Downloads``""" + return _android_downloads_folder() + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``/storage/emulated/0/Pictures``""" + return _android_pictures_folder() + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``/storage/emulated/0/DCIM/Camera``""" + return _android_videos_folder() + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``/storage/emulated/0/Music``""" + return _android_music_folder() + + @property + def user_desktop_dir(self) -> str: + """:return: desktop directory tied to the user e.g. ``/storage/emulated/0/Desktop``""" + return "/storage/emulated/0/Desktop" + + @property + def user_runtime_dir(self) -> str: + """ + :return: runtime directory tied to the user, same as `user_cache_dir` if not opinionated else ``tmp`` in it, + e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/tmp`` + """ + path = self.user_cache_dir + if self.opinion: + path = os.path.join(path, "tmp") # noqa: PTH118 + return path + + @property + def site_runtime_dir(self) -> str: + """:return: runtime directory shared by users, same as `user_runtime_dir`""" + return self.user_runtime_dir + + +@lru_cache(maxsize=1) +def _android_folder() -> str | None: # noqa: C901 + """:return: base folder for the Android OS or None if it cannot be found""" + result: str | None = None + # type checker isn't happy with our "import android", just don't do this when type checking see + # https://stackoverflow.com/a/61394121 + if not TYPE_CHECKING: + try: + # First try to get a path to android app using python4android (if available)... + from android import mActivity # noqa: PLC0415 + + context = cast("android.content.Context", mActivity.getApplicationContext()) # noqa: F821 + result = context.getFilesDir().getParentFile().getAbsolutePath() + except Exception: # noqa: BLE001 + result = None + if result is None: + try: + # ...and fall back to using plain pyjnius, if python4android isn't available or doesn't deliver any useful + # result... + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + result = context.getFilesDir().getParentFile().getAbsolutePath() + except Exception: # noqa: BLE001 + result = None + if result is None: + # and if that fails, too, find an android folder looking at path on the sys.path + # warning: only works for apps installed under /data, not adopted storage etc. + pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files") + for path in sys.path: + if pattern.match(path): + result = path.split("/files")[0] + break + else: + result = None + if result is None: + # one last try: find an android folder looking at path on the sys.path taking adopted storage paths into + # account + pattern = re.compile(r"/mnt/expand/[a-fA-F0-9-]{36}/(data|user/\d+)/(.+)/files") + for path in sys.path: + if pattern.match(path): + result = path.split("/files")[0] + break + else: + result = None + return result + + +@lru_cache(maxsize=1) +def _android_documents_folder() -> str: + """:return: documents folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + documents_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOCUMENTS).getAbsolutePath() + except Exception: # noqa: BLE001 + documents_dir = "/storage/emulated/0/Documents" + + return documents_dir + + +@lru_cache(maxsize=1) +def _android_downloads_folder() -> str: + """:return: downloads folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + downloads_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOWNLOADS).getAbsolutePath() + except Exception: # noqa: BLE001 + downloads_dir = "/storage/emulated/0/Downloads" + + return downloads_dir + + +@lru_cache(maxsize=1) +def _android_pictures_folder() -> str: + """:return: pictures folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + pictures_dir: str = context.getExternalFilesDir(environment.DIRECTORY_PICTURES).getAbsolutePath() + except Exception: # noqa: BLE001 + pictures_dir = "/storage/emulated/0/Pictures" + + return pictures_dir + + +@lru_cache(maxsize=1) +def _android_videos_folder() -> str: + """:return: videos folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + videos_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DCIM).getAbsolutePath() + except Exception: # noqa: BLE001 + videos_dir = "/storage/emulated/0/DCIM/Camera" + + return videos_dir + + +@lru_cache(maxsize=1) +def _android_music_folder() -> str: + """:return: music folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass # noqa: PLC0415 + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + music_dir: str = context.getExternalFilesDir(environment.DIRECTORY_MUSIC).getAbsolutePath() + except Exception: # noqa: BLE001 + music_dir = "/storage/emulated/0/Music" + + return music_dir + + +__all__ = [ + "Android", +] diff --git a/contrib/python/platformdirs/platformdirs/api.py b/contrib/python/platformdirs/platformdirs/api.py new file mode 100644 index 00000000000..18d660e4f8c --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/api.py @@ -0,0 +1,298 @@ +"""Base API.""" + +from __future__ import annotations + +import os +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterator, Literal + + +class PlatformDirsABC(ABC): # noqa: PLR0904 + """Abstract base class for platform directories.""" + + def __init__( # noqa: PLR0913, PLR0917 + self, + appname: str | None = None, + appauthor: str | None | Literal[False] = None, + version: str | None = None, + roaming: bool = False, # noqa: FBT001, FBT002 + multipath: bool = False, # noqa: FBT001, FBT002 + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 + ) -> None: + """ + Create a new platform directory. + + :param appname: See `appname`. + :param appauthor: See `appauthor`. + :param version: See `version`. + :param roaming: See `roaming`. + :param multipath: See `multipath`. + :param opinion: See `opinion`. + :param ensure_exists: See `ensure_exists`. + + """ + self.appname = appname #: The name of application. + self.appauthor = appauthor + """ + The name of the app author or distributing body for this application. + + Typically, it is the owning company name. Defaults to `appname`. You may pass ``False`` to disable it. + + """ + self.version = version + """ + An optional version path element to append to the path. + + You might want to use this if you want multiple versions of your app to be able to run independently. If used, + this would typically be ``<major>.<minor>``. + + """ + self.roaming = roaming + """ + Whether to use the roaming appdata directory on Windows. + + That means that for users on a Windows network setup for roaming profiles, this user data will be synced on + login (see + `here <https://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>`_). + + """ + self.multipath = multipath + """ + An optional parameter which indicates that the entire list of data dirs should be returned. + + By default, the first item would only be returned. + + """ + self.opinion = opinion #: A flag to indicating to use opinionated values. + self.ensure_exists = ensure_exists + """ + Optionally create the directory (and any missing parents) upon access if it does not exist. + + By default, no directories are created. + + """ + + def _append_app_name_and_version(self, *base: str) -> str: + params = list(base[1:]) + if self.appname: + params.append(self.appname) + if self.version: + params.append(self.version) + path = os.path.join(base[0], *params) # noqa: PTH118 + self._optionally_create_directory(path) + return path + + def _optionally_create_directory(self, path: str) -> None: + if self.ensure_exists: + Path(path).mkdir(parents=True, exist_ok=True) + + def _first_item_as_path_if_multipath(self, directory: str) -> Path: + if self.multipath: + # If multipath is True, the first path is returned. + directory = directory.split(os.pathsep)[0] + return Path(directory) + + @property + @abstractmethod + def user_data_dir(self) -> str: + """:return: data directory tied to the user""" + + @property + @abstractmethod + def site_data_dir(self) -> str: + """:return: data directory shared by users""" + + @property + @abstractmethod + def user_config_dir(self) -> str: + """:return: config directory tied to the user""" + + @property + @abstractmethod + def site_config_dir(self) -> str: + """:return: config directory shared by the users""" + + @property + @abstractmethod + def user_cache_dir(self) -> str: + """:return: cache directory tied to the user""" + + @property + @abstractmethod + def site_cache_dir(self) -> str: + """:return: cache directory shared by users""" + + @property + @abstractmethod + def user_state_dir(self) -> str: + """:return: state directory tied to the user""" + + @property + @abstractmethod + def user_log_dir(self) -> str: + """:return: log directory tied to the user""" + + @property + @abstractmethod + def user_documents_dir(self) -> str: + """:return: documents directory tied to the user""" + + @property + @abstractmethod + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user""" + + @property + @abstractmethod + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user""" + + @property + @abstractmethod + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user""" + + @property + @abstractmethod + def user_music_dir(self) -> str: + """:return: music directory tied to the user""" + + @property + @abstractmethod + def user_desktop_dir(self) -> str: + """:return: desktop directory tied to the user""" + + @property + @abstractmethod + def user_runtime_dir(self) -> str: + """:return: runtime directory tied to the user""" + + @property + @abstractmethod + def site_runtime_dir(self) -> str: + """:return: runtime directory shared by users""" + + @property + def user_data_path(self) -> Path: + """:return: data path tied to the user""" + return Path(self.user_data_dir) + + @property + def site_data_path(self) -> Path: + """:return: data path shared by users""" + return Path(self.site_data_dir) + + @property + def user_config_path(self) -> Path: + """:return: config path tied to the user""" + return Path(self.user_config_dir) + + @property + def site_config_path(self) -> Path: + """:return: config path shared by the users""" + return Path(self.site_config_dir) + + @property + def user_cache_path(self) -> Path: + """:return: cache path tied to the user""" + return Path(self.user_cache_dir) + + @property + def site_cache_path(self) -> Path: + """:return: cache path shared by users""" + return Path(self.site_cache_dir) + + @property + def user_state_path(self) -> Path: + """:return: state path tied to the user""" + return Path(self.user_state_dir) + + @property + def user_log_path(self) -> Path: + """:return: log path tied to the user""" + return Path(self.user_log_dir) + + @property + def user_documents_path(self) -> Path: + """:return: documents a path tied to the user""" + return Path(self.user_documents_dir) + + @property + def user_downloads_path(self) -> Path: + """:return: downloads path tied to the user""" + return Path(self.user_downloads_dir) + + @property + def user_pictures_path(self) -> Path: + """:return: pictures path tied to the user""" + return Path(self.user_pictures_dir) + + @property + def user_videos_path(self) -> Path: + """:return: videos path tied to the user""" + return Path(self.user_videos_dir) + + @property + def user_music_path(self) -> Path: + """:return: music path tied to the user""" + return Path(self.user_music_dir) + + @property + def user_desktop_path(self) -> Path: + """:return: desktop path tied to the user""" + return Path(self.user_desktop_dir) + + @property + def user_runtime_path(self) -> Path: + """:return: runtime path tied to the user""" + return Path(self.user_runtime_dir) + + @property + def site_runtime_path(self) -> Path: + """:return: runtime path shared by users""" + return Path(self.site_runtime_dir) + + def iter_config_dirs(self) -> Iterator[str]: + """:yield: all user and site configuration directories.""" + yield self.user_config_dir + yield self.site_config_dir + + def iter_data_dirs(self) -> Iterator[str]: + """:yield: all user and site data directories.""" + yield self.user_data_dir + yield self.site_data_dir + + def iter_cache_dirs(self) -> Iterator[str]: + """:yield: all user and site cache directories.""" + yield self.user_cache_dir + yield self.site_cache_dir + + def iter_runtime_dirs(self) -> Iterator[str]: + """:yield: all user and site runtime directories.""" + yield self.user_runtime_dir + yield self.site_runtime_dir + + def iter_config_paths(self) -> Iterator[Path]: + """:yield: all user and site configuration paths.""" + for path in self.iter_config_dirs(): + yield Path(path) + + def iter_data_paths(self) -> Iterator[Path]: + """:yield: all user and site data paths.""" + for path in self.iter_data_dirs(): + yield Path(path) + + def iter_cache_paths(self) -> Iterator[Path]: + """:yield: all user and site cache paths.""" + for path in self.iter_cache_dirs(): + yield Path(path) + + def iter_runtime_paths(self) -> Iterator[Path]: + """:yield: all user and site runtime paths.""" + for path in self.iter_runtime_dirs(): + yield Path(path) diff --git a/contrib/python/platformdirs/platformdirs/macos.py b/contrib/python/platformdirs/platformdirs/macos.py new file mode 100644 index 00000000000..e4b0391abd7 --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/macos.py @@ -0,0 +1,144 @@ +"""macOS.""" + +from __future__ import annotations + +import os.path +import sys +from typing import TYPE_CHECKING + +from .api import PlatformDirsABC + +if TYPE_CHECKING: + from pathlib import Path + + +class MacOS(PlatformDirsABC): + """ + Platform directories for the macOS operating system. + + Follows the guidance from + `Apple documentation <https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html>`_. + Makes use of the `appname <platformdirs.api.PlatformDirsABC.appname>`, + `version <platformdirs.api.PlatformDirsABC.version>`, + `ensure_exists <platformdirs.api.PlatformDirsABC.ensure_exists>`. + + """ + + @property + def user_data_dir(self) -> str: + """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``""" + return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support")) # noqa: PTH111 + + @property + def site_data_dir(self) -> str: + """ + :return: data directory shared by users, e.g. ``/Library/Application Support/$appname/$version``. + If we're using a Python binary managed by `Homebrew <https://brew.sh>`_, the directory + will be under the Homebrew prefix, e.g. ``/opt/homebrew/share/$appname/$version``. + If `multipath <platformdirs.api.PlatformDirsABC.multipath>` is enabled, and we're in Homebrew, + the response is a multi-path string separated by ":", e.g. + ``/opt/homebrew/share/$appname/$version:/Library/Application Support/$appname/$version`` + """ + is_homebrew = sys.prefix.startswith("/opt/homebrew") + path_list = [self._append_app_name_and_version("/opt/homebrew/share")] if is_homebrew else [] + path_list.append(self._append_app_name_and_version("/Library/Application Support")) + if self.multipath: + return os.pathsep.join(path_list) + return path_list[0] + + @property + def site_data_path(self) -> Path: + """:return: data path shared by users. Only return the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_data_dir) + + @property + def user_config_dir(self) -> str: + """:return: config directory tied to the user, same as `user_data_dir`""" + return self.user_data_dir + + @property + def site_config_dir(self) -> str: + """:return: config directory shared by the users, same as `site_data_dir`""" + return self.site_data_dir + + @property + def user_cache_dir(self) -> str: + """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``""" + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) # noqa: PTH111 + + @property + def site_cache_dir(self) -> str: + """ + :return: cache directory shared by users, e.g. ``/Library/Caches/$appname/$version``. + If we're using a Python binary managed by `Homebrew <https://brew.sh>`_, the directory + will be under the Homebrew prefix, e.g. ``/opt/homebrew/var/cache/$appname/$version``. + If `multipath <platformdirs.api.PlatformDirsABC.multipath>` is enabled, and we're in Homebrew, + the response is a multi-path string separated by ":", e.g. + ``/opt/homebrew/var/cache/$appname/$version:/Library/Caches/$appname/$version`` + """ + is_homebrew = sys.prefix.startswith("/opt/homebrew") + path_list = [self._append_app_name_and_version("/opt/homebrew/var/cache")] if is_homebrew else [] + path_list.append(self._append_app_name_and_version("/Library/Caches")) + if self.multipath: + return os.pathsep.join(path_list) + return path_list[0] + + @property + def site_cache_path(self) -> Path: + """:return: cache path shared by users. Only return the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_cache_dir) + + @property + def user_state_dir(self) -> str: + """:return: state directory tied to the user, same as `user_data_dir`""" + return self.user_data_dir + + @property + def user_log_dir(self) -> str: + """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``""" + return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) # noqa: PTH111 + + @property + def user_documents_dir(self) -> str: + """:return: documents directory tied to the user, e.g. ``~/Documents``""" + return os.path.expanduser("~/Documents") # noqa: PTH111 + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return os.path.expanduser("~/Downloads") # noqa: PTH111 + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return os.path.expanduser("~/Pictures") # noqa: PTH111 + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Movies``""" + return os.path.expanduser("~/Movies") # noqa: PTH111 + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return os.path.expanduser("~/Music") # noqa: PTH111 + + @property + def user_desktop_dir(self) -> str: + """:return: desktop directory tied to the user, e.g. ``~/Desktop``""" + return os.path.expanduser("~/Desktop") # noqa: PTH111 + + @property + def user_runtime_dir(self) -> str: + """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``""" + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) # noqa: PTH111 + + @property + def site_runtime_dir(self) -> str: + """:return: runtime directory shared by users, same as `user_runtime_dir`""" + return self.user_runtime_dir + + +__all__ = [ + "MacOS", +] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/py.typed b/contrib/python/platformdirs/platformdirs/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/py.typed +++ b/contrib/python/platformdirs/platformdirs/py.typed diff --git a/contrib/python/platformdirs/platformdirs/unix.py b/contrib/python/platformdirs/platformdirs/unix.py new file mode 100644 index 00000000000..f1942e92ef4 --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/unix.py @@ -0,0 +1,269 @@ +"""Unix.""" + +from __future__ import annotations + +import os +import sys +from configparser import ConfigParser +from pathlib import Path +from typing import Iterator, NoReturn + +from .api import PlatformDirsABC + +if sys.platform == "win32": + + def getuid() -> NoReturn: + msg = "should only be used on Unix" + raise RuntimeError(msg) + +else: + from os import getuid + + +class Unix(PlatformDirsABC): # noqa: PLR0904 + """ + On Unix/Linux, we follow the `XDG Basedir Spec <https://specifications.freedesktop.org/basedir-spec/basedir-spec- + latest.html>`_. + + The spec allows overriding directories with environment variables. The examples shown are the default values, + alongside the name of the environment variable that overrides them. Makes use of the `appname + <platformdirs.api.PlatformDirsABC.appname>`, `version <platformdirs.api.PlatformDirsABC.version>`, `multipath + <platformdirs.api.PlatformDirsABC.multipath>`, `opinion <platformdirs.api.PlatformDirsABC.opinion>`, `ensure_exists + <platformdirs.api.PlatformDirsABC.ensure_exists>`. + + """ + + @property + def user_data_dir(self) -> str: + """ + :return: data directory tied to the user, e.g. ``~/.local/share/$appname/$version`` or + ``$XDG_DATA_HOME/$appname/$version`` + """ + path = os.environ.get("XDG_DATA_HOME", "") + if not path.strip(): + path = os.path.expanduser("~/.local/share") # noqa: PTH111 + return self._append_app_name_and_version(path) + + @property + def _site_data_dirs(self) -> list[str]: + path = os.environ.get("XDG_DATA_DIRS", "") + if not path.strip(): + path = f"/usr/local/share{os.pathsep}/usr/share" + return [self._append_app_name_and_version(p) for p in path.split(os.pathsep)] + + @property + def site_data_dir(self) -> str: + """ + :return: data directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` is + enabled and ``XDG_DATA_DIRS`` is set and a multi path the response is also a multi path separated by the + OS path separator), e.g. ``/usr/local/share/$appname/$version`` or ``/usr/share/$appname/$version`` + """ + # XDG default for $XDG_DATA_DIRS; only first, if multipath is False + dirs = self._site_data_dirs + if not self.multipath: + return dirs[0] + return os.pathsep.join(dirs) + + @property + def user_config_dir(self) -> str: + """ + :return: config directory tied to the user, e.g. ``~/.config/$appname/$version`` or + ``$XDG_CONFIG_HOME/$appname/$version`` + """ + path = os.environ.get("XDG_CONFIG_HOME", "") + if not path.strip(): + path = os.path.expanduser("~/.config") # noqa: PTH111 + return self._append_app_name_and_version(path) + + @property + def _site_config_dirs(self) -> list[str]: + path = os.environ.get("XDG_CONFIG_DIRS", "") + if not path.strip(): + path = "/etc/xdg" + return [self._append_app_name_and_version(p) for p in path.split(os.pathsep)] + + @property + def site_config_dir(self) -> str: + """ + :return: config directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` + is enabled and ``XDG_CONFIG_DIRS`` is set and a multi path the response is also a multi path separated by + the OS path separator), e.g. ``/etc/xdg/$appname/$version`` + """ + # XDG default for $XDG_CONFIG_DIRS only first, if multipath is False + dirs = self._site_config_dirs + if not self.multipath: + return dirs[0] + return os.pathsep.join(dirs) + + @property + def user_cache_dir(self) -> str: + """ + :return: cache directory tied to the user, e.g. ``~/.cache/$appname/$version`` or + ``~/$XDG_CACHE_HOME/$appname/$version`` + """ + path = os.environ.get("XDG_CACHE_HOME", "") + if not path.strip(): + path = os.path.expanduser("~/.cache") # noqa: PTH111 + return self._append_app_name_and_version(path) + + @property + def site_cache_dir(self) -> str: + """:return: cache directory shared by users, e.g. ``/var/cache/$appname/$version``""" + return self._append_app_name_and_version("/var/cache") + + @property + def user_state_dir(self) -> str: + """ + :return: state directory tied to the user, e.g. ``~/.local/state/$appname/$version`` or + ``$XDG_STATE_HOME/$appname/$version`` + """ + path = os.environ.get("XDG_STATE_HOME", "") + if not path.strip(): + path = os.path.expanduser("~/.local/state") # noqa: PTH111 + return self._append_app_name_and_version(path) + + @property + def user_log_dir(self) -> str: + """:return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it""" + path = self.user_state_dir + if self.opinion: + path = os.path.join(path, "log") # noqa: PTH118 + self._optionally_create_directory(path) + return path + + @property + def user_documents_dir(self) -> str: + """:return: documents directory tied to the user, e.g. ``~/Documents``""" + return _get_user_media_dir("XDG_DOCUMENTS_DIR", "~/Documents") + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return _get_user_media_dir("XDG_DOWNLOAD_DIR", "~/Downloads") + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return _get_user_media_dir("XDG_PICTURES_DIR", "~/Pictures") + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Videos``""" + return _get_user_media_dir("XDG_VIDEOS_DIR", "~/Videos") + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return _get_user_media_dir("XDG_MUSIC_DIR", "~/Music") + + @property + def user_desktop_dir(self) -> str: + """:return: desktop directory tied to the user, e.g. ``~/Desktop``""" + return _get_user_media_dir("XDG_DESKTOP_DIR", "~/Desktop") + + @property + def user_runtime_dir(self) -> str: + """ + :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or + ``$XDG_RUNTIME_DIR/$appname/$version``. + + For FreeBSD/OpenBSD/NetBSD, it would return ``/var/run/user/$(id -u)/$appname/$version`` if + exists, otherwise ``/tmp/runtime-$(id -u)/$appname/$version``, if``$XDG_RUNTIME_DIR`` + is not set. + """ + path = os.environ.get("XDG_RUNTIME_DIR", "") + if not path.strip(): + if sys.platform.startswith(("freebsd", "openbsd", "netbsd")): + path = f"/var/run/user/{getuid()}" + if not Path(path).exists(): + path = f"/tmp/runtime-{getuid()}" # noqa: S108 + else: + path = f"/run/user/{getuid()}" + return self._append_app_name_and_version(path) + + @property + def site_runtime_dir(self) -> str: + """ + :return: runtime directory shared by users, e.g. ``/run/$appname/$version`` or \ + ``$XDG_RUNTIME_DIR/$appname/$version``. + + Note that this behaves almost exactly like `user_runtime_dir` if ``$XDG_RUNTIME_DIR`` is set, but will + fall back to paths associated to the root user instead of a regular logged-in user if it's not set. + + If you wish to ensure that a logged-in root user path is returned e.g. ``/run/user/0``, use `user_runtime_dir` + instead. + + For FreeBSD/OpenBSD/NetBSD, it would return ``/var/run/$appname/$version`` if ``$XDG_RUNTIME_DIR`` is not set. + """ + path = os.environ.get("XDG_RUNTIME_DIR", "") + if not path.strip(): + if sys.platform.startswith(("freebsd", "openbsd", "netbsd")): + path = "/var/run" + else: + path = "/run" + return self._append_app_name_and_version(path) + + @property + def site_data_path(self) -> Path: + """:return: data path shared by users. Only return the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_data_dir) + + @property + def site_config_path(self) -> Path: + """:return: config path shared by the users, returns the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_config_dir) + + @property + def site_cache_path(self) -> Path: + """:return: cache path shared by users. Only return the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_cache_dir) + + def iter_config_dirs(self) -> Iterator[str]: + """:yield: all user and site configuration directories.""" + yield self.user_config_dir + yield from self._site_config_dirs + + def iter_data_dirs(self) -> Iterator[str]: + """:yield: all user and site data directories.""" + yield self.user_data_dir + yield from self._site_data_dirs + + +def _get_user_media_dir(env_var: str, fallback_tilde_path: str) -> str: + media_dir = _get_user_dirs_folder(env_var) + if media_dir is None: + media_dir = os.environ.get(env_var, "").strip() + if not media_dir: + media_dir = os.path.expanduser(fallback_tilde_path) # noqa: PTH111 + + return media_dir + + +def _get_user_dirs_folder(key: str) -> str | None: + """ + Return directory from user-dirs.dirs config file. + + See https://freedesktop.org/wiki/Software/xdg-user-dirs/. + + """ + user_dirs_config_path = Path(Unix().user_config_dir) / "user-dirs.dirs" + if user_dirs_config_path.exists(): + parser = ConfigParser() + + with user_dirs_config_path.open() as stream: + # Add fake section header, so ConfigParser doesn't complain + parser.read_string(f"[top]\n{stream.read()}") + + if key not in parser["top"]: + return None + + path = parser["top"][key].strip('"') + # Handle relative home paths + return path.replace("$HOME", os.path.expanduser("~")) # noqa: PTH111 + + return None + + +__all__ = [ + "Unix", +] diff --git a/contrib/python/platformdirs/platformdirs/version.py b/contrib/python/platformdirs/platformdirs/version.py new file mode 100644 index 00000000000..afb49243e3d --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/version.py @@ -0,0 +1,16 @@ +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '4.3.6' +__version_tuple__ = version_tuple = (4, 3, 6) diff --git a/contrib/python/platformdirs/platformdirs/windows.py b/contrib/python/platformdirs/platformdirs/windows.py new file mode 100644 index 00000000000..d7bc96091a2 --- /dev/null +++ b/contrib/python/platformdirs/platformdirs/windows.py @@ -0,0 +1,272 @@ +"""Windows.""" + +from __future__ import annotations + +import os +import sys +from functools import lru_cache +from typing import TYPE_CHECKING + +from .api import PlatformDirsABC + +if TYPE_CHECKING: + from collections.abc import Callable + + +class Windows(PlatformDirsABC): + """ + `MSDN on where to store app data files <https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid>`_. + + Makes use of the `appname <platformdirs.api.PlatformDirsABC.appname>`, `appauthor + <platformdirs.api.PlatformDirsABC.appauthor>`, `version <platformdirs.api.PlatformDirsABC.version>`, `roaming + <platformdirs.api.PlatformDirsABC.roaming>`, `opinion <platformdirs.api.PlatformDirsABC.opinion>`, `ensure_exists + <platformdirs.api.PlatformDirsABC.ensure_exists>`. + + """ + + @property + def user_data_dir(self) -> str: + """ + :return: data directory tied to the user, e.g. + ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname`` (not roaming) or + ``%USERPROFILE%\\AppData\\Roaming\\$appauthor\\$appname`` (roaming) + """ + const = "CSIDL_APPDATA" if self.roaming else "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(get_win_folder(const)) + return self._append_parts(path) + + def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str: + params = [] + if self.appname: + if self.appauthor is not False: + author = self.appauthor or self.appname + params.append(author) + params.append(self.appname) + if opinion_value is not None and self.opinion: + params.append(opinion_value) + if self.version: + params.append(self.version) + path = os.path.join(path, *params) # noqa: PTH118 + self._optionally_create_directory(path) + return path + + @property + def site_data_dir(self) -> str: + """:return: data directory shared by users, e.g. ``C:\\ProgramData\\$appauthor\\$appname``""" + path = os.path.normpath(get_win_folder("CSIDL_COMMON_APPDATA")) + return self._append_parts(path) + + @property + def user_config_dir(self) -> str: + """:return: config directory tied to the user, same as `user_data_dir`""" + return self.user_data_dir + + @property + def site_config_dir(self) -> str: + """:return: config directory shared by the users, same as `site_data_dir`""" + return self.site_data_dir + + @property + def user_cache_dir(self) -> str: + """ + :return: cache directory tied to the user (if opinionated with ``Cache`` folder within ``$appname``) e.g. + ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname\\Cache\\$version`` + """ + path = os.path.normpath(get_win_folder("CSIDL_LOCAL_APPDATA")) + return self._append_parts(path, opinion_value="Cache") + + @property + def site_cache_dir(self) -> str: + """:return: cache directory shared by users, e.g. ``C:\\ProgramData\\$appauthor\\$appname\\Cache\\$version``""" + path = os.path.normpath(get_win_folder("CSIDL_COMMON_APPDATA")) + return self._append_parts(path, opinion_value="Cache") + + @property + def user_state_dir(self) -> str: + """:return: state directory tied to the user, same as `user_data_dir`""" + return self.user_data_dir + + @property + def user_log_dir(self) -> str: + """:return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it""" + path = self.user_data_dir + if self.opinion: + path = os.path.join(path, "Logs") # noqa: PTH118 + self._optionally_create_directory(path) + return path + + @property + def user_documents_dir(self) -> str: + """:return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents``""" + return os.path.normpath(get_win_folder("CSIDL_PERSONAL")) + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``%USERPROFILE%\\Downloads``""" + return os.path.normpath(get_win_folder("CSIDL_DOWNLOADS")) + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``%USERPROFILE%\\Pictures``""" + return os.path.normpath(get_win_folder("CSIDL_MYPICTURES")) + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``%USERPROFILE%\\Videos``""" + return os.path.normpath(get_win_folder("CSIDL_MYVIDEO")) + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``%USERPROFILE%\\Music``""" + return os.path.normpath(get_win_folder("CSIDL_MYMUSIC")) + + @property + def user_desktop_dir(self) -> str: + """:return: desktop directory tied to the user, e.g. ``%USERPROFILE%\\Desktop``""" + return os.path.normpath(get_win_folder("CSIDL_DESKTOPDIRECTORY")) + + @property + def user_runtime_dir(self) -> str: + """ + :return: runtime directory tied to the user, e.g. + ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname`` + """ + path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) # noqa: PTH118 + return self._append_parts(path) + + @property + def site_runtime_dir(self) -> str: + """:return: runtime directory shared by users, same as `user_runtime_dir`""" + return self.user_runtime_dir + + +def get_win_folder_from_env_vars(csidl_name: str) -> str: + """Get folder from environment variables.""" + result = get_win_folder_if_csidl_name_not_env_var(csidl_name) + if result is not None: + return result + + env_var_name = { + "CSIDL_APPDATA": "APPDATA", + "CSIDL_COMMON_APPDATA": "ALLUSERSPROFILE", + "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA", + }.get(csidl_name) + if env_var_name is None: + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) + result = os.environ.get(env_var_name) + if result is None: + msg = f"Unset environment variable: {env_var_name}" + raise ValueError(msg) + return result + + +def get_win_folder_if_csidl_name_not_env_var(csidl_name: str) -> str | None: + """Get a folder for a CSIDL name that does not exist as an environment variable.""" + if csidl_name == "CSIDL_PERSONAL": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") # noqa: PTH118 + + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Downloads") # noqa: PTH118 + + if csidl_name == "CSIDL_MYPICTURES": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Pictures") # noqa: PTH118 + + if csidl_name == "CSIDL_MYVIDEO": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Videos") # noqa: PTH118 + + if csidl_name == "CSIDL_MYMUSIC": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Music") # noqa: PTH118 + return None + + +def get_win_folder_from_registry(csidl_name: str) -> str: + """ + Get folder from the registry. + + This is a fallback technique at best. I'm not sure if using the registry for these guarantees us the correct answer + for all CSIDL_* names. + + """ + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + "CSIDL_PERSONAL": "Personal", + "CSIDL_DOWNLOADS": "{374DE290-123F-4565-9164-39C4925E467B}", + "CSIDL_MYPICTURES": "My Pictures", + "CSIDL_MYVIDEO": "My Video", + "CSIDL_MYMUSIC": "My Music", + }.get(csidl_name) + if shell_folder_name is None: + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) + if sys.platform != "win32": # only needed for mypy type checker to know that this code runs only on Windows + raise NotImplementedError + import winreg # noqa: PLC0415 + + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") + directory, _ = winreg.QueryValueEx(key, shell_folder_name) + return str(directory) + + +def get_win_folder_via_ctypes(csidl_name: str) -> str: + """Get folder with ctypes.""" + # There is no 'CSIDL_DOWNLOADS'. + # Use 'CSIDL_PROFILE' (40) and append the default folder 'Downloads' instead. + # https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid + + import ctypes # noqa: PLC0415 + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + "CSIDL_PERSONAL": 5, + "CSIDL_MYPICTURES": 39, + "CSIDL_MYVIDEO": 14, + "CSIDL_MYMUSIC": 13, + "CSIDL_DOWNLOADS": 40, + "CSIDL_DESKTOPDIRECTORY": 16, + }.get(csidl_name) + if csidl_const is None: + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) + + buf = ctypes.create_unicode_buffer(1024) + windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker + windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if it has high-bit chars. + if any(ord(c) > 255 for c in buf): # noqa: PLR2004 + buf2 = ctypes.create_unicode_buffer(1024) + if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(buf.value, "Downloads") # noqa: PTH118 + + return buf.value + + +def _pick_get_win_folder() -> Callable[[str], str]: + try: + import ctypes # noqa: PLC0415 + except ImportError: + pass + else: + if hasattr(ctypes, "windll"): + return get_win_folder_via_ctypes + try: + import winreg # noqa: PLC0415, F401 + except ImportError: + return get_win_folder_from_env_vars + else: + return get_win_folder_from_registry + + +get_win_folder = lru_cache(maxsize=None)(_pick_get_win_folder()) + +__all__ = [ + "Windows", +] diff --git a/contrib/python/platformdirs/ya.make b/contrib/python/platformdirs/ya.make new file mode 100644 index 00000000000..66109cf6a8b --- /dev/null +++ b/contrib/python/platformdirs/ya.make @@ -0,0 +1,30 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(4.3.6) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + platformdirs/__init__.py + platformdirs/__main__.py + platformdirs/android.py + platformdirs/api.py + platformdirs/macos.py + platformdirs/unix.py + platformdirs/version.py + platformdirs/windows.py +) + +RESOURCE_FILES( + PREFIX contrib/python/platformdirs/ + .dist-info/METADATA + .dist-info/top_level.txt + platformdirs/py.typed +) + +END() diff --git a/contrib/python/setuptools/py3/.dist-info/METADATA b/contrib/python/setuptools/py3/.dist-info/METADATA index 90eca520d03..55b94d8d5a0 100644 --- a/contrib/python/setuptools/py3/.dist-info/METADATA +++ b/contrib/python/setuptools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: setuptools -Version: 70.3.0 +Version: 71.1.0 Summary: Easily download, build, install, upgrade, and uninstall Python packages Author-email: Python Packaging Authority <distutils-sig@python.org> Project-URL: Source, https://github.com/pypa/setuptools @@ -20,6 +20,16 @@ Requires-Python: >=3.8 Description-Content-Type: text/x-rst License-File: LICENSE Provides-Extra: certs +Provides-Extra: core +Requires-Dist: packaging >=24 ; extra == 'core' +Requires-Dist: ordered-set >=3.1.1 ; extra == 'core' +Requires-Dist: more-itertools >=8.8 ; extra == 'core' +Requires-Dist: jaraco.text >=3.7 ; extra == 'core' +Requires-Dist: wheel >=0.43.0 ; extra == 'core' +Requires-Dist: platformdirs >=2.6.2 ; extra == 'core' +Requires-Dist: importlib-metadata >=6 ; (python_version < "3.10") and extra == 'core' +Requires-Dist: tomli >=2.0.1 ; (python_version < "3.11") and extra == 'core' +Requires-Dist: importlib-resources >=5.10.2 ; (python_version < "3.9") and extra == 'core' Provides-Extra: doc Requires-Dist: sphinx >=3.5 ; extra == 'doc' Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' @@ -54,15 +64,17 @@ Requires-Dist: ini2toml[lite] >=0.14 ; extra == 'test' Requires-Dist: tomli-w >=1.0.0 ; extra == 'test' Requires-Dist: pytest-timeout ; extra == 'test' Requires-Dist: pytest-home >=0.5 ; extra == 'test' -Requires-Dist: mypy ==1.10.0 ; extra == 'test' +Requires-Dist: mypy ==1.11.* ; extra == 'test' Requires-Dist: tomli ; extra == 'test' Requires-Dist: importlib-metadata ; extra == 'test' Requires-Dist: pytest-subprocess ; extra == 'test' Requires-Dist: pyproject-hooks !=1.1 ; extra == 'test' Requires-Dist: jaraco.test ; extra == 'test' +Requires-Dist: pytest-ruff <0.4 ; (platform_system == "Windows") and extra == 'test' Requires-Dist: jaraco.develop >=7.21 ; (python_version >= "3.9" and sys_platform != "cygwin") and extra == 'test' -Requires-Dist: pytest-ruff >=0.3.2 ; (sys_platform != "cygwin") and extra == 'test' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'test' Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'test' +Requires-Dist: pytest-ruff >=0.3.2 ; (sys_platform != "cygwin") and extra == 'test' .. |pypi-version| image:: https://img.shields.io/pypi/v/setuptools.svg :target: https://pypi.org/project/setuptools diff --git a/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml b/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml index 82e5cdb5fff..ffe47f851aa 100644 --- a/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml +++ b/contrib/python/setuptools/py3/.yandex_meta/yamaker.yaml @@ -1,4 +1,16 @@ requirements: + - jaraco.context + - jaraco.functools + - jaraco.text + - ordered-set + - more-itertools + - packaging + - platformdirs + - typing-extensions + - typeguard + - wheel - library/python/resource mark_as_sources: - setuptools/command/test.py +exclude: + - setuptools/_vendor/* diff --git a/contrib/python/setuptools/py3/pkg_resources/__init__.py b/contrib/python/setuptools/py3/pkg_resources/__init__.py index bb8cbf88fdb..3f3956b6144 100644 --- a/contrib/python/setuptools/py3/pkg_resources/__init__.py +++ b/contrib/python/setuptools/py3/pkg_resources/__init__.py @@ -34,6 +34,7 @@ import re import types from typing import ( Any, + BinaryIO, Literal, Dict, Iterator, @@ -74,6 +75,10 @@ from pkgutil import get_importer import _imp +sys.path.extend(((vendor_path := os.path.join(os.path.dirname(os.path.dirname(__file__)), 'setuptools', '_vendor')) not in sys.path) * [vendor_path]) # fmt: skip +# workaround for #4476 +sys.modules.pop('backports', None) + # capture these to bypass sandboxing from os import utime from os import open as os_open @@ -87,16 +92,17 @@ except ImportError: # no write support, probably under GAE WRITE_SUPPORT = False -from pkg_resources.extern.jaraco.text import ( +import packaging.specifiers +from jaraco.text import ( yield_lines, drop_comment, join_continuation, ) -from pkg_resources.extern.packaging import markers as _packaging_markers -from pkg_resources.extern.packaging import requirements as _packaging_requirements -from pkg_resources.extern.packaging import utils as _packaging_utils -from pkg_resources.extern.packaging import version as _packaging_version -from pkg_resources.extern.platformdirs import user_cache_dir as _user_cache_dir +from packaging import markers as _packaging_markers +from packaging import requirements as _packaging_requirements +from packaging import utils as _packaging_utils +from packaging import version as _packaging_version +from platformdirs import user_cache_dir as _user_cache_dir if TYPE_CHECKING: from _typeshed import BytesPath, StrPath, StrOrBytesPath @@ -109,7 +115,6 @@ warnings.warn( stacklevel=2, ) - _T = TypeVar("_T") _DistributionT = TypeVar("_DistributionT", bound="Distribution") # Type aliases @@ -153,7 +158,6 @@ class PEP440Warning(RuntimeWarning): parse_version = _packaging_version.Version - _state_vars: dict[str, str] = {} @@ -335,7 +339,9 @@ class VersionConflict(ResolutionError): def report(self): return self._template.format(**locals()) - def with_context(self, required_by: set[Distribution | str]): + def with_context( + self, required_by: set[Distribution | str] + ) -> Self | ContextualVersionConflict: """ If required_by is non-empty, return a version of self that is a ContextualVersionConflict. @@ -404,7 +410,7 @@ DEVELOP_DIST = -1 def register_loader_type( loader_type: type[_ModuleLike], provider_factory: _ProviderFactoryType -): +) -> None: """Register `provider_factory` to make providers for `loader_type` `loader_type` is the type or class of a PEP 302 ``module.__loader__``, @@ -480,7 +486,7 @@ darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") get_platform = get_build_platform -def compatible_platforms(provided: str | None, required: str | None): +def compatible_platforms(provided: str | None, required: str | None) -> bool: """Can code for the `provided` platform run on the `required` platform? Returns true if either platform is ``None``, or the platforms are equal. @@ -538,8 +544,7 @@ def get_distribution(dist: Distribution | _PkgReqType) -> Distribution: if isinstance(dist, str): dist = Requirement.parse(dist) if isinstance(dist, Requirement): - # Bad type narrowing, dist has to be a Requirement here, so get_provider has to return Distribution - dist = get_provider(dist) # type: ignore[assignment] + dist = get_provider(dist) if not isinstance(dist, Distribution): raise TypeError("Expected str, Requirement, or Distribution", dist) return dist @@ -561,7 +566,7 @@ def get_entry_map(dist: _EPDistType, group: str | None = None): return get_distribution(dist).get_entry_map(group) -def get_entry_info(dist: _EPDistType, group: str, name: str): +def get_entry_info(dist: _EPDistType, group: str, name: str) -> EntryPoint | None: """Return the EntryPoint object for `group`+`name`, or ``None``""" return get_distribution(dist).get_entry_info(group, name) @@ -682,7 +687,7 @@ class WorkingSet: sys.path[:] = ws.entries return ws - def add_entry(self, entry: str): + def add_entry(self, entry: str) -> None: """Add a path item to ``.entries``, finding any distributions on it ``find_distributions(entry, True)`` is used to find distributions @@ -725,7 +730,9 @@ class WorkingSet: raise VersionConflict(dist, req) return dist - def iter_entry_points(self, group: str, name: str | None = None): + def iter_entry_points( + self, group: str, name: str | None = None + ) -> Iterator[EntryPoint]: """Yield entry point objects from `group` matching `name` If `name` is None, yields all entry points in `group` from all @@ -739,7 +746,7 @@ class WorkingSet: if name is None or name == entry.name ) - def run_script(self, requires: str, script_name: str): + def run_script(self, requires: str, script_name: str) -> None: """Locate distribution for `requires` and run `script_name` script""" ns = sys._getframe(1).f_globals name = ns['__name__'] @@ -770,7 +777,7 @@ class WorkingSet: entry: str | None = None, insert: bool = True, replace: bool = False, - ): + ) -> None: """Add `dist` to working set, associated with `entry` If `entry` is unspecified, it defaults to the ``.location`` of `dist`. @@ -1050,7 +1057,7 @@ class WorkingSet: return sorted_distributions, error_info - def require(self, *requirements: _NestedStr): + def require(self, *requirements: _NestedStr) -> list[Distribution]: """Ensure that distributions matching `requirements` are activated `requirements` must be a string or a (possibly-nested) sequence @@ -1068,7 +1075,7 @@ class WorkingSet: def subscribe( self, callback: Callable[[Distribution], object], existing: bool = True - ): + ) -> None: """Invoke `callback` for all distributions If `existing=True` (default), @@ -1117,11 +1124,10 @@ class _ReqExtras(Dict["Requirement", Tuple[str, ...]]): Return False if the req has a marker and fails evaluation. Otherwise, return True. """ - extra_evals = ( + return not req.marker or any( req.marker.evaluate({'extra': extra}) - for extra in self.get(req, ()) + (extras or (None,)) + for extra in self.get(req, ()) + (extras or ("",)) ) - return not req.marker or any(extra_evals) class Environment: @@ -1154,7 +1160,7 @@ class Environment: self.python = python self.scan(search_path) - def can_add(self, dist: Distribution): + def can_add(self, dist: Distribution) -> bool: """Is distribution `dist` acceptable for this environment? The distribution must match the platform and python version @@ -1168,11 +1174,11 @@ class Environment: ) return py_compat and compatible_platforms(dist.platform, self.platform) - def remove(self, dist: Distribution): + def remove(self, dist: Distribution) -> None: """Remove `dist` from the environment""" self._distmap[dist.key].remove(dist) - def scan(self, search_path: Iterable[str] | None = None): + def scan(self, search_path: Iterable[str] | None = None) -> None: """Scan `search_path` for distributions usable in this environment Any distributions found are added to the environment. @@ -1198,7 +1204,7 @@ class Environment: distribution_key = project_name.lower() return self._distmap.get(distribution_key, []) - def add(self, dist: Distribution): + def add(self, dist: Distribution) -> None: """Add `dist` if we ``can_add()`` it and it has not already been added""" if self.can_add(dist) and dist.has_version(): dists = self._distmap.setdefault(dist.key, []) @@ -1349,23 +1355,29 @@ class ResourceManager: def __init__(self): self.cached_files = {} - def resource_exists(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_exists( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> bool: """Does the named resource exist?""" return get_provider(package_or_requirement).has_resource(resource_name) - def resource_isdir(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_isdir( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> bool: """Is the named resource an existing directory?""" return get_provider(package_or_requirement).resource_isdir(resource_name) def resource_filename( self, package_or_requirement: _PkgReqType, resource_name: str - ): + ) -> str: """Return a true filesystem path for specified resource""" return get_provider(package_or_requirement).get_resource_filename( self, resource_name ) - def resource_stream(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_stream( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> _ResourceStream: """Return a readable file-like object for specified resource""" return get_provider(package_or_requirement).get_resource_stream( self, resource_name @@ -1379,7 +1391,9 @@ class ResourceManager: self, resource_name ) - def resource_listdir(self, package_or_requirement: _PkgReqType, resource_name: str): + def resource_listdir( + self, package_or_requirement: _PkgReqType, resource_name: str + ) -> list[str]: """List the contents of the named resource directory""" return get_provider(package_or_requirement).resource_listdir(resource_name) @@ -1413,7 +1427,7 @@ class ResourceManager: err.original_error = old_exc raise err - def get_cache_path(self, archive_name: str, names: Iterable[StrPath] = ()): + def get_cache_path(self, archive_name: str, names: Iterable[StrPath] = ()) -> str: """Return absolute location in cache for `archive_name` and `names` The parent directory of the resulting path will be created if it does @@ -1465,7 +1479,7 @@ class ResourceManager: ).format(**locals()) warnings.warn(msg, UserWarning) - def postprocess(self, tempname: StrOrBytesPath, filename: StrOrBytesPath): + def postprocess(self, tempname: StrOrBytesPath, filename: StrOrBytesPath) -> None: """Perform any platform-specific postprocessing of `tempname` This is where Mac header rewrites should be done; other platforms don't @@ -1485,7 +1499,7 @@ class ResourceManager: mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777 os.chmod(tempname, mode) - def set_extraction_path(self, path: str): + def set_extraction_path(self, path: str) -> None: """Set the base path where resources will be extracted to, if needed. If you do not call this routine before any extractions take place, the @@ -1533,7 +1547,7 @@ def get_default_cache() -> str: return os.environ.get('PYTHON_EGG_CACHE') or _user_cache_dir(appname='Python-Eggs') -def safe_name(name: str): +def safe_name(name: str) -> str: """Convert an arbitrary string to a standard distribution name Any runs of non-alphanumeric/. characters are replaced with a single '-'. @@ -1541,7 +1555,7 @@ def safe_name(name: str): return re.sub('[^A-Za-z0-9.]+', '-', name) -def safe_version(version: str): +def safe_version(version: str) -> str: """ Convert an arbitrary string to a standard version string """ @@ -1585,7 +1599,7 @@ def _safe_segment(segment): return re.sub(r'\.[^A-Za-z0-9]+', '.', segment).strip(".-") -def safe_extra(extra: str): +def safe_extra(extra: str) -> str: """Convert an arbitrary string to a standard 'extra' name Any runs of non-alphanumeric characters are replaced with a single '_', @@ -1594,7 +1608,7 @@ def safe_extra(extra: str): return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() -def to_filename(name: str): +def to_filename(name: str) -> str: """Convert a project or version name to its filename-escaped form Any '-' characters are currently replaced with '_'. @@ -1602,7 +1616,7 @@ def to_filename(name: str): return name.replace('-', '_') -def invalid_marker(text: str): +def invalid_marker(text: str) -> SyntaxError | Literal[False]: """ Validate text as a PEP 508 environment marker; return an exception if invalid or False otherwise. @@ -1642,10 +1656,14 @@ class NullProvider: self.loader = getattr(module, '__loader__', None) self.module_path = os.path.dirname(getattr(module, '__file__', '')) - def get_resource_filename(self, manager: ResourceManager, resource_name: str): + def get_resource_filename( + self, manager: ResourceManager, resource_name: str + ) -> str: return self._fn(self.module_path, resource_name) - def get_resource_stream(self, manager: ResourceManager, resource_name: str): + def get_resource_stream( + self, manager: ResourceManager, resource_name: str + ) -> BinaryIO: return io.BytesIO(self.get_resource_string(manager, resource_name)) def get_resource_string( @@ -1653,7 +1671,7 @@ class NullProvider: ) -> bytes: return self._get(self._fn(self.module_path, resource_name)) - def has_resource(self, resource_name: str): + def has_resource(self, resource_name: str) -> bool: return self._has(self._fn(self.module_path, resource_name)) def _get_metadata_path(self, name): @@ -1666,7 +1684,7 @@ class NullProvider: path = self._get_metadata_path(name) return self._has(path) - def get_metadata(self, name: str): + def get_metadata(self, name: str) -> str: if not self.egg_info: return "" path = self._get_metadata_path(name) @@ -1682,13 +1700,13 @@ class NullProvider: def get_metadata_lines(self, name: str) -> Iterator[str]: return yield_lines(self.get_metadata(name)) - def resource_isdir(self, resource_name: str): + def resource_isdir(self, resource_name: str) -> bool: return self._isdir(self._fn(self.module_path, resource_name)) def metadata_isdir(self, name: str) -> bool: return bool(self.egg_info and self._isdir(self._fn(self.egg_info, name))) - def resource_listdir(self, resource_name: str): + def resource_listdir(self, resource_name: str) -> list[str]: return self._listdir(self._fn(self.module_path, resource_name)) def metadata_listdir(self, name: str) -> list[str]: @@ -1696,7 +1714,7 @@ class NullProvider: return self._listdir(self._fn(self.egg_info, name)) return [] - def run_script(self, script_name: str, namespace: dict[str, Any]): + def run_script(self, script_name: str, namespace: dict[str, Any]) -> None: script = 'scripts/' + script_name if not self.has_metadata(script): raise ResolutionError( @@ -1880,7 +1898,9 @@ class DefaultProvider(EggProvider): def _listdir(self, path): return os.listdir(path) - def get_resource_stream(self, manager: object, resource_name: str): + def get_resource_stream( + self, manager: object, resource_name: str + ) -> io.BufferedReader: return open(self._fn(self.module_path, resource_name), 'rb') def _get(self, path) -> bytes: @@ -1929,7 +1949,7 @@ class ZipManifests(Dict[str, "MemoizedZipManifests.manifest_mod"]): # `path` could be `StrPath | IO[bytes]` but that violates the LSP for `MemoizedZipManifests.load` @classmethod - def build(cls, path: str): + def build(cls, path: str) -> dict[str, zipfile.ZipInfo]: """ Build a dictionary similar to the zipimport directory caches, except instead of tuples, store ZipInfo objects. @@ -2007,7 +2027,9 @@ class ZipProvider(EggProvider): def zipinfo(self): return self._zip_manifests.load(self.loader.archive) - def get_resource_filename(self, manager: ResourceManager, resource_name: str): + def get_resource_filename( + self, manager: ResourceManager, resource_name: str + ) -> str: if not self.egg_name: raise NotImplementedError( "resource_filename() only supported for .egg, not .zip" @@ -2167,7 +2189,7 @@ class FileMetadata(EmptyProvider): def has_metadata(self, name: str) -> bool: return name == 'PKG-INFO' and os.path.isfile(self.path) - def get_metadata(self, name: str): + def get_metadata(self, name: str) -> str: if name != 'PKG-INFO': raise KeyError("No metadata except PKG-INFO is available") @@ -2232,7 +2254,9 @@ _distribution_finders: dict[type, _DistFinderType[Any]] = _declare_state( ) -def register_finder(importer_type: type[_T], distribution_finder: _DistFinderType[_T]): +def register_finder( + importer_type: type[_T], distribution_finder: _DistFinderType[_T] +) -> None: """Register `distribution_finder` to find distributions in sys.path items `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item @@ -2242,7 +2266,7 @@ def register_finder(importer_type: type[_T], distribution_finder: _DistFinderTyp _distribution_finders[importer_type] = distribution_finder -def find_distributions(path_item: str, only: bool = False): +def find_distributions(path_item: str, only: bool = False) -> Iterable[Distribution]: """Yield distributions accessible via `path_item`""" importer = get_importer(path_item) finder = _find_adapter(_distribution_finders, importer) @@ -2416,7 +2440,7 @@ _namespace_packages: dict[str | None, list[str]] = _declare_state( def register_namespace_handler( importer_type: type[_T], namespace_handler: _NSHandlerType[_T] -): +) -> None: """Register `namespace_handler` to declare namespace packages `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item @@ -2505,7 +2529,7 @@ def _rebuild_mod_path(orig_path, package_name, module: types.ModuleType): module.__path__ = new_path -def declare_namespace(packageName: str): +def declare_namespace(packageName: str) -> None: """Declare that package 'packageName' is a namespace package""" msg = ( @@ -2548,7 +2572,7 @@ def declare_namespace(packageName: str): _imp.release_lock() -def fixup_namespace_packages(path_item: str, parent: str | None = None): +def fixup_namespace_packages(path_item: str, parent: str | None = None) -> None: """Ensure that previously-declared namespace packages include path_item""" _imp.acquire_lock() try: @@ -2625,6 +2649,7 @@ if TYPE_CHECKING: @overload def _normalize_cached(filename: BytesPath) -> bytes: ... def _normalize_cached(filename: StrOrBytesPath) -> str | bytes: ... + else: @functools.lru_cache(maxsize=None) @@ -2759,7 +2784,7 @@ class EntryPoint: self, env: Environment | None = None, installer: _InstallerType | None = None, - ): + ) -> None: if not self.dist: error_cls = UnknownExtra if self.extras else AttributeError raise error_cls("Can't require() without a distribution", self) @@ -2783,7 +2808,7 @@ class EntryPoint: ) @classmethod - def parse(cls, src: str, dist: Distribution | None = None): + def parse(cls, src: str, dist: Distribution | None = None) -> Self: """Parse a single entry point from string `src` Entry point syntax follows the form:: @@ -2817,7 +2842,7 @@ class EntryPoint: group: str, lines: _NestedStr, dist: Distribution | None = None, - ): + ) -> dict[str, Self]: """Parse an entry point group""" if not MODULE(group): raise ValueError("Invalid group name", group) @@ -2834,7 +2859,7 @@ class EntryPoint: cls, data: str | Iterable[str] | dict[str, str | Iterable[str]], dist: Distribution | None = None, - ): + ) -> dict[str, dict[str, Self]]: """Parse a map of entry point groups""" _data: Iterable[tuple[str | None, str | Iterable[str]]] if isinstance(data, dict): @@ -3039,7 +3064,9 @@ class Distribution: return self.__dep_map @staticmethod - def _filter_extras(dm: dict[str | None, list[Requirement]]): + def _filter_extras( + dm: dict[str | None, list[Requirement]], + ) -> dict[str | None, list[Requirement]]: """ Given a mapping of extras to dependencies, strip off environment markers and filter out any dependencies @@ -3066,7 +3093,7 @@ class Distribution: dm.setdefault(extra, []).extend(parse_requirements(reqs)) return dm - def requires(self, extras: Iterable[str] = ()): + def requires(self, extras: Iterable[str] = ()) -> list[Requirement]: """List of Requirements needed for this distro if `extras` are used""" dm = self._dep_map deps: list[Requirement] = [] @@ -3105,7 +3132,7 @@ class Distribution: lines = self._get_metadata(self.PKG_INFO) return _version_from_file(lines) - def activate(self, path: list[str] | None = None, replace: bool = False): + def activate(self, path: list[str] | None = None, replace: bool = False) -> None: """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: path = sys.path @@ -3160,7 +3187,7 @@ class Distribution: filename: StrPath, metadata: _MetadataType = None, **kw: int, # We could set `precedence` explicitly, but keeping this as `**kw` for full backwards and subclassing compatibility - ): + ) -> Distribution: return cls.from_location( _normalize_cached(filename), os.path.basename(filename), metadata, **kw ) @@ -3195,7 +3222,7 @@ class Distribution: return self._ep_map.get(group, {}) return self._ep_map - def get_entry_info(self, group: str, name: str): + def get_entry_info(self, group: str, name: str) -> EntryPoint | None: """Return the EntryPoint object for `group`+`name`, or ``None``""" return self.get_entry_map(group).get(name) @@ -3205,7 +3232,7 @@ class Distribution: path: list[str], loc=None, replace: bool = False, - ): + ) -> None: """Ensure self.location is on path If replace=False (default): @@ -3310,7 +3337,7 @@ class Distribution: return False return True - def clone(self, **kw: str | int | IResourceProvider | None): + def clone(self, **kw: str | int | IResourceProvider | None) -> Self: """Copy this distribution, substituting in any changed keyword args""" names = 'project_name version py_version platform location precedence' for attr in names.split(): @@ -3416,7 +3443,7 @@ def issue_warning(*args, **kw): warnings.warn(stacklevel=level + 1, *args, **kw) -def parse_requirements(strs: _NestedStr): +def parse_requirements(strs: _NestedStr) -> map[Requirement]: """ Yield ``Requirement`` objects for each specification in `strs`. @@ -3430,6 +3457,10 @@ class RequirementParseError(_packaging_requirements.InvalidRequirement): class Requirement(_packaging_requirements.Requirement): + # prefer variable length tuple to set (as found in + # packaging.requirements.Requirement) + extras: tuple[str, ...] # type: ignore[assignment] + def __init__(self, requirement_string: str): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" super().__init__(requirement_string) @@ -3437,8 +3468,7 @@ class Requirement(_packaging_requirements.Requirement): project_name = safe_name(self.name) self.project_name, self.key = project_name, project_name.lower() self.specs = [(spec.operator, spec.version) for spec in self.specifier] - # packaging.requirements.Requirement uses a set for its extras. We use a variable-length tuple - self.extras: tuple[str] = tuple(map(safe_extra, self.extras)) + self.extras = tuple(map(safe_extra, self.extras)) self.hashCmp = ( self.key, self.url, @@ -3454,17 +3484,24 @@ class Requirement(_packaging_requirements.Requirement): def __ne__(self, other): return not self == other - def __contains__(self, item: Distribution | str | tuple[str, ...]) -> bool: + def __contains__( + self, item: Distribution | packaging.specifiers.UnparsedVersion + ) -> bool: if isinstance(item, Distribution): if item.key != self.key: return False - item = item.version + version = item.version + else: + version = item # Allow prereleases always in order to match the previous behavior of # this method. In the future this should be smarter and follow PEP 440 # more accurately. - return self.specifier.contains(item, prereleases=True) + return self.specifier.contains( + version, + prereleases=True, + ) def __hash__(self): return self.__hash @@ -3473,7 +3510,7 @@ class Requirement(_packaging_requirements.Requirement): return "Requirement.parse(%r)" % str(self) @staticmethod - def parse(s: str | Iterable[str]): + def parse(s: str | Iterable[str]) -> Requirement: (req,) = parse_requirements(s) return req @@ -3499,7 +3536,7 @@ def _find_adapter(registry: Mapping[type, _AdapterT], ob: object) -> _AdapterT: raise TypeError(f"Could not find adapter for {registry} and {ob}") -def ensure_directory(path: StrOrBytesPath): +def ensure_directory(path: StrOrBytesPath) -> None: """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) os.makedirs(dirname, exist_ok=True) @@ -3701,6 +3738,7 @@ class PkgResourcesDeprecationWarning(Warning): _LOCALE_ENCODING = "locale" if sys.version_info >= (3, 10) else None +# This must go before calls to `_call_aside`. See https://github.com/pypa/setuptools/pull/4422 def _read_utf8_with_fallback(file: str, fallback_encoding=_LOCALE_ENCODING) -> str: """See setuptools.unicode_utils._read_utf8_with_fallback""" try: diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py deleted file mode 100644 index a7a9a6e7b94..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/tarfile.py +++ /dev/null @@ -1,2900 +0,0 @@ -#!/usr/bin/env python3 -#------------------------------------------------------------------- -# tarfile.py -#------------------------------------------------------------------- -# Copyright (C) 2002 Lars Gustaebel <lars@gustaebel.de> -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -"""Read from and write to tar format archives. -""" - -version = "0.9.0" -__author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" -__credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." - -#--------- -# Imports -#--------- -from builtins import open as bltn_open -import sys -import os -import io -import shutil -import stat -import time -import struct -import copy -import re -import warnings - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -# os.symlink on Windows prior to 6.0 raises NotImplementedError -# OSError (winerror=1314) will be raised if the caller does not hold the -# SeCreateSymbolicLinkPrivilege privilege -symlink_exception = (AttributeError, NotImplementedError, OSError) - -# from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", - "CompressionError", "StreamError", "ExtractError", "HeaderError", - "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", - "tar_filter", "FilterError", "AbsoluteLinkError", - "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] - - -#--------------------------------------------------------- -# tar constants -#--------------------------------------------------------- -NUL = b"\0" # the null character -BLOCKSIZE = 512 # length of processing blocks -RECORDSIZE = BLOCKSIZE * 20 # length of records -GNU_MAGIC = b"ustar \0" # magic gnu tar string -POSIX_MAGIC = b"ustar\x0000" # magic posix tar string - -LENGTH_NAME = 100 # maximum length of a filename -LENGTH_LINK = 100 # maximum length of a linkname -LENGTH_PREFIX = 155 # maximum length of the prefix field - -REGTYPE = b"0" # regular file -AREGTYPE = b"\0" # regular file -LNKTYPE = b"1" # link (inside tarfile) -SYMTYPE = b"2" # symbolic link -CHRTYPE = b"3" # character special device -BLKTYPE = b"4" # block special device -DIRTYPE = b"5" # directory -FIFOTYPE = b"6" # fifo special device -CONTTYPE = b"7" # contiguous file - -GNUTYPE_LONGNAME = b"L" # GNU tar longname -GNUTYPE_LONGLINK = b"K" # GNU tar longlink -GNUTYPE_SPARSE = b"S" # GNU tar sparse file - -XHDTYPE = b"x" # POSIX.1-2001 extended header -XGLTYPE = b"g" # POSIX.1-2001 global header -SOLARIS_XHDTYPE = b"X" # Solaris extended header - -USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format -GNU_FORMAT = 1 # GNU tar format -PAX_FORMAT = 2 # POSIX.1-2001 (pax) format -DEFAULT_FORMAT = PAX_FORMAT - -#--------------------------------------------------------- -# tarfile constants -#--------------------------------------------------------- -# File types that tarfile supports: -SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, - SYMTYPE, DIRTYPE, FIFOTYPE, - CONTTYPE, CHRTYPE, BLKTYPE, - GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# File types that will be treated as a regular file. -REGULAR_TYPES = (REGTYPE, AREGTYPE, - CONTTYPE, GNUTYPE_SPARSE) - -# File types that are part of the GNU tar format. -GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# Fields from a pax header that override a TarInfo attribute. -PAX_FIELDS = ("path", "linkpath", "size", "mtime", - "uid", "gid", "uname", "gname") - -# Fields from a pax header that are affected by hdrcharset. -PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"} - -# Fields in a pax header that are numbers, all other fields -# are treated as strings. -PAX_NUMBER_FIELDS = { - "atime": float, - "ctime": float, - "mtime": float, - "uid": int, - "gid": int, - "size": int -} - -#--------------------------------------------------------- -# initialization -#--------------------------------------------------------- -if os.name == "nt": - ENCODING = "utf-8" -else: - ENCODING = sys.getfilesystemencoding() - -#--------------------------------------------------------- -# Some useful functions -#--------------------------------------------------------- - -def stn(s, length, encoding, errors): - """Convert a string to a null-terminated bytes object. - """ - if s is None: - raise ValueError("metadata cannot contain None") - s = s.encode(encoding, errors) - return s[:length] + (length - len(s)) * NUL - -def nts(s, encoding, errors): - """Convert a null-terminated bytes object to a string. - """ - p = s.find(b"\0") - if p != -1: - s = s[:p] - return s.decode(encoding, errors) - -def nti(s): - """Convert a number field to a python number. - """ - # There are two possible encodings for a number field, see - # itn() below. - if s[0] in (0o200, 0o377): - n = 0 - for i in range(len(s) - 1): - n <<= 8 - n += s[i + 1] - if s[0] == 0o377: - n = -(256 ** (len(s) - 1) - n) - else: - try: - s = nts(s, "ascii", "strict") - n = int(s.strip() or "0", 8) - except ValueError: - raise InvalidHeaderError("invalid header") - return n - -def itn(n, digits=8, format=DEFAULT_FORMAT): - """Convert a python number to a number field. - """ - # POSIX 1003.1-1988 requires numbers to be encoded as a string of - # octal digits followed by a null-byte, this allows values up to - # (8**(digits-1))-1. GNU tar allows storing numbers greater than - # that if necessary. A leading 0o200 or 0o377 byte indicate this - # particular encoding, the following digits-1 bytes are a big-endian - # base-256 representation. This allows values up to (256**(digits-1))-1. - # A 0o200 byte indicates a positive number, a 0o377 byte a negative - # number. - original_n = n - n = int(n) - if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL - elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): - if n >= 0: - s = bytearray([0o200]) - else: - s = bytearray([0o377]) - n = 256 ** digits + n - - for i in range(digits - 1): - s.insert(1, n & 0o377) - n >>= 8 - else: - raise ValueError("overflow in number field") - - return s - -def calc_chksums(buf): - """Calculate the checksum for a member's header by summing up all - characters except for the chksum field which is treated as if - it was filled with spaces. According to the GNU tar sources, - some tars (Sun and NeXT) calculate chksum with signed char, - which will be different if there are chars in the buffer with - the high bit set. So we calculate two checksums, unsigned and - signed. - """ - unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf)) - signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf)) - return unsigned_chksum, signed_chksum - -def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): - """Copy length bytes from fileobj src to fileobj dst. - If length is None, copy the entire content. - """ - bufsize = bufsize or 16 * 1024 - if length == 0: - return - if length is None: - shutil.copyfileobj(src, dst, bufsize) - return - - blocks, remainder = divmod(length, bufsize) - for b in range(blocks): - buf = src.read(bufsize) - if len(buf) < bufsize: - raise exception("unexpected end of data") - dst.write(buf) - - if remainder != 0: - buf = src.read(remainder) - if len(buf) < remainder: - raise exception("unexpected end of data") - dst.write(buf) - return - -def _safe_print(s): - encoding = getattr(sys.stdout, 'encoding', None) - if encoding is not None: - s = s.encode(encoding, 'backslashreplace').decode(encoding) - print(s, end=' ') - - -class TarError(Exception): - """Base exception.""" - pass -class ExtractError(TarError): - """General exception for extract errors.""" - pass -class ReadError(TarError): - """Exception for unreadable tar archives.""" - pass -class CompressionError(TarError): - """Exception for unavailable compression methods.""" - pass -class StreamError(TarError): - """Exception for unsupported operations on stream-like TarFiles.""" - pass -class HeaderError(TarError): - """Base exception for header errors.""" - pass -class EmptyHeaderError(HeaderError): - """Exception for empty headers.""" - pass -class TruncatedHeaderError(HeaderError): - """Exception for truncated headers.""" - pass -class EOFHeaderError(HeaderError): - """Exception for end of file headers.""" - pass -class InvalidHeaderError(HeaderError): - """Exception for invalid headers.""" - pass -class SubsequentHeaderError(HeaderError): - """Exception for missing and invalid extended headers.""" - pass - -#--------------------------- -# internal stream interface -#--------------------------- -class _LowLevelFile: - """Low-level file object. Supports reading and writing. - It is used instead of a regular file object for streaming - access. - """ - - def __init__(self, name, mode): - mode = { - "r": os.O_RDONLY, - "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - }[mode] - if hasattr(os, "O_BINARY"): - mode |= os.O_BINARY - self.fd = os.open(name, mode, 0o666) - - def close(self): - os.close(self.fd) - - def read(self, size): - return os.read(self.fd, size) - - def write(self, s): - os.write(self.fd, s) - -class _Stream: - """Class that serves as an adapter between TarFile and - a stream-like object. The stream-like object only - needs to have a read() or write() method that works with bytes, - and the method is accessed blockwise. - Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin.buffer, - sys.stdout.buffer, a socket, a tape device etc. - - _Stream is intended to be used only internally. - """ - - def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel): - """Construct a _Stream object. - """ - self._extfileobj = True - if fileobj is None: - fileobj = _LowLevelFile(name, mode) - self._extfileobj = False - - if comptype == '*': - # Enable transparent compression detection for the - # stream interface - fileobj = _StreamProxy(fileobj) - comptype = fileobj.getcomptype() - - self.name = name or "" - self.mode = mode - self.comptype = comptype - self.fileobj = fileobj - self.bufsize = bufsize - self.buf = b"" - self.pos = 0 - self.closed = False - - try: - if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") from None - self.zlib = zlib - self.crc = zlib.crc32(b"") - if mode == "r": - self.exception = zlib.error - self._init_read_gz() - else: - self._init_write_gz(compresslevel) - - elif comptype == "bz2": - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = bz2.BZ2Decompressor() - self.exception = OSError - else: - self.cmp = bz2.BZ2Compressor(compresslevel) - - elif comptype == "xz": - try: - import lzma - except ImportError: - raise CompressionError("lzma module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = lzma.LZMADecompressor() - self.exception = lzma.LZMAError - else: - self.cmp = lzma.LZMACompressor() - - elif comptype != "tar": - raise CompressionError("unknown compression type %r" % comptype) - - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - def __del__(self): - if hasattr(self, "closed") and not self.closed: - self.close() - - def _init_write_gz(self, compresslevel): - """Initialize for writing with gzip compression. - """ - self.cmp = self.zlib.compressobj(compresslevel, - self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) - timestamp = struct.pack("<L", int(time.time())) - self.__write(b"\037\213\010\010" + timestamp + b"\002\377") - if self.name.endswith(".gz"): - self.name = self.name[:-3] - # Honor "directory components removed" from RFC1952 - self.name = os.path.basename(self.name) - # RFC1952 says we must use ISO-8859-1 for the FNAME field. - self.__write(self.name.encode("iso-8859-1", "replace") + NUL) - - def write(self, s): - """Write string s to the stream. - """ - if self.comptype == "gz": - self.crc = self.zlib.crc32(s, self.crc) - self.pos += len(s) - if self.comptype != "tar": - s = self.cmp.compress(s) - self.__write(s) - - def __write(self, s): - """Write string s to the stream if a whole new block - is ready to be written. - """ - self.buf += s - while len(self.buf) > self.bufsize: - self.fileobj.write(self.buf[:self.bufsize]) - self.buf = self.buf[self.bufsize:] - - def close(self): - """Close the _Stream object. No operation should be - done on it afterwards. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode == "w" and self.comptype != "tar": - self.buf += self.cmp.flush() - - if self.mode == "w" and self.buf: - self.fileobj.write(self.buf) - self.buf = b"" - if self.comptype == "gz": - self.fileobj.write(struct.pack("<L", self.crc)) - self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def _init_read_gz(self): - """Initialize for reading a gzip compressed fileobj. - """ - self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" - - # taken from gzip.GzipFile with some alterations - if self.__read(2) != b"\037\213": - raise ReadError("not a gzip file") - if self.__read(1) != b"\010": - raise CompressionError("unsupported compression method") - - flag = ord(self.__read(1)) - self.__read(6) - - if flag & 4: - xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) - if flag & 8: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 16: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 2: - self.__read(2) - - def tell(self): - """Return the stream's file pointer position. - """ - return self.pos - - def seek(self, pos=0): - """Set the stream's file pointer to pos. Negative seeking - is forbidden. - """ - if pos - self.pos >= 0: - blocks, remainder = divmod(pos - self.pos, self.bufsize) - for i in range(blocks): - self.read(self.bufsize) - self.read(remainder) - else: - raise StreamError("seeking backwards is not allowed") - return self.pos - - def read(self, size): - """Return the next size number of bytes from the stream.""" - assert size is not None - buf = self._read(size) - self.pos += len(buf) - return buf - - def _read(self, size): - """Return size bytes from the stream. - """ - if self.comptype == "tar": - return self.__read(size) - - c = len(self.dbuf) - t = [self.dbuf] - while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" - else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] - - def __read(self, size): - """Return size bytes from stream. If internal buffer is empty, - read another block from the stream. - """ - c = len(self.buf) - t = [self.buf] - while c < size: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - t.append(buf) - c += len(buf) - t = b"".join(t) - self.buf = t[size:] - return t[:size] -# class _Stream - -class _StreamProxy(object): - """Small proxy class that enables transparent compression - detection for the Stream interface (mode 'r|*'). - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - self.buf = self.fileobj.read(BLOCKSIZE) - - def read(self, size): - self.read = self.fileobj.read - return self.buf - - def getcomptype(self): - if self.buf.startswith(b"\x1f\x8b\x08"): - return "gz" - elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY": - return "bz2" - elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): - return "xz" - else: - return "tar" - - def close(self): - self.fileobj.close() -# class StreamProxy - -#------------------------ -# Extraction file object -#------------------------ -class _FileInFile(object): - """A thin wrapper around an existing file object that - provides a part of its data as an individual file - object. - """ - - def __init__(self, fileobj, offset, size, name, blockinfo=None): - self.fileobj = fileobj - self.offset = offset - self.size = size - self.position = 0 - self.name = name - self.closed = False - - if blockinfo is None: - blockinfo = [(0, size)] - - # Construct a map with data and zero blocks. - self.map_index = 0 - self.map = [] - lastpos = 0 - realpos = self.offset - for offset, size in blockinfo: - if offset > lastpos: - self.map.append((False, lastpos, offset, None)) - self.map.append((True, offset, offset + size, realpos)) - realpos += size - lastpos = offset + size - if lastpos < self.size: - self.map.append((False, lastpos, self.size, None)) - - def flush(self): - pass - - def readable(self): - return True - - def writable(self): - return False - - def seekable(self): - return self.fileobj.seekable() - - def tell(self): - """Return the current file position. - """ - return self.position - - def seek(self, position, whence=io.SEEK_SET): - """Seek to a position in the file. - """ - if whence == io.SEEK_SET: - self.position = min(max(position, 0), self.size) - elif whence == io.SEEK_CUR: - if position < 0: - self.position = max(self.position + position, 0) - else: - self.position = min(self.position + position, self.size) - elif whence == io.SEEK_END: - self.position = max(min(self.size + position, self.size), 0) - else: - raise ValueError("Invalid argument") - return self.position - - def read(self, size=None): - """Read data from the file. - """ - if size is None: - size = self.size - self.position - else: - size = min(size, self.size - self.position) - - buf = b"" - while size > 0: - while True: - data, start, stop, offset = self.map[self.map_index] - if start <= self.position < stop: - break - else: - self.map_index += 1 - if self.map_index == len(self.map): - self.map_index = 0 - length = min(size, stop - self.position) - if data: - self.fileobj.seek(offset + (self.position - start)) - b = self.fileobj.read(length) - if len(b) != length: - raise ReadError("unexpected end of data") - buf += b - else: - buf += NUL * length - size -= length - self.position += length - return buf - - def readinto(self, b): - buf = self.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def close(self): - self.closed = True -#class _FileInFile - -class ExFileObject(io.BufferedReader): - - def __init__(self, tarfile, tarinfo): - fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data, - tarinfo.size, tarinfo.name, tarinfo.sparse) - super().__init__(fileobj) -#class ExFileObject - - -#----------------------------- -# extraction filters (PEP 706) -#----------------------------- - -class FilterError(TarError): - pass - -class AbsolutePathError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'member {tarinfo.name!r} has an absolute path') - -class OutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, ' - + 'which is outside the destination') - -class SpecialFileError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a special file') - -class AbsoluteLinkError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a link to an absolute path') - -class LinkOutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' - + 'which is outside the destination') - -def _get_filtered_attrs(member, dest_path, for_data=True): - new_attrs = {} - name = member.name - dest_path = os.path.realpath(dest_path) - # Strip leading / (tar's directory separator) from filenames. - # Include os.sep (target OS directory separator) as well. - if name.startswith(('/', os.sep)): - name = new_attrs['name'] = member.path.lstrip('/' + os.sep) - if os.path.isabs(name): - # Path is absolute even after stripping. - # For example, 'C:/foo' on Windows. - raise AbsolutePathError(member) - # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise OutsideDestinationError(member, target_path) - # Limit permissions (no high bits, and go-w) - mode = member.mode - if mode is not None: - # Strip high bits & group/other write bits - mode = mode & 0o755 - if for_data: - # For data, handle permissions & file types - if member.isreg() or member.islnk(): - if not mode & 0o100: - # Clear executable bits if not executable by user - mode &= ~0o111 - # Ensure owner can read & write - mode |= 0o600 - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Reject special files - raise SpecialFileError(member) - if mode != member.mode: - new_attrs['mode'] = mode - if for_data: - # Ignore ownership for 'data' - if member.uid is not None: - new_attrs['uid'] = None - if member.gid is not None: - new_attrs['gid'] = None - if member.uname is not None: - new_attrs['uname'] = None - if member.gname is not None: - new_attrs['gname'] = None - # Check link destination for 'data' - if member.islnk() or member.issym(): - if os.path.isabs(member.linkname): - raise AbsoluteLinkError(member) - if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) - else: - target_path = os.path.join(dest_path, - member.linkname) - target_path = os.path.realpath(target_path) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise LinkOutsideDestinationError(member, target_path) - return new_attrs - -def fully_trusted_filter(member, dest_path): - return member - -def tar_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, False) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -def data_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, True) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -_NAMED_FILTERS = { - "fully_trusted": fully_trusted_filter, - "tar": tar_filter, - "data": data_filter, -} - -#------------------ -# Exported Classes -#------------------ - -# Sentinel for replace() defaults, meaning "don't change the attribute" -_KEEP = object() - -class TarInfo(object): - """Informational class which holds the details about an - archive member given by a tar header block. - TarInfo objects are returned by TarFile.getmember(), - TarFile.getmembers() and TarFile.gettarinfo() and are - usually created internally. - """ - - __slots__ = dict( - name = 'Name of the archive member.', - mode = 'Permission bits.', - uid = 'User ID of the user who originally stored this member.', - gid = 'Group ID of the user who originally stored this member.', - size = 'Size in bytes.', - mtime = 'Time of last modification.', - chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), - linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), - uname = 'User name.', - gname = 'Group name.', - devmajor = 'Device major number.', - devminor = 'Device minor number.', - offset = 'The tar header starts here.', - offset_data = "The file's data starts here.", - pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), - sparse = 'Sparse member information.', - tarfile = None, - _sparse_structs = None, - _link_target = None, - ) - - def __init__(self, name=""): - """Construct a TarInfo object. name is the optional name - of the member. - """ - self.name = name # member name - self.mode = 0o644 # file permissions - self.uid = 0 # user id - self.gid = 0 # group id - self.size = 0 # file size - self.mtime = 0 # modification time - self.chksum = 0 # header checksum - self.type = REGTYPE # member type - self.linkname = "" # link name - self.uname = "" # user name - self.gname = "" # group name - self.devmajor = 0 # device major number - self.devminor = 0 # device minor number - - self.offset = 0 # the tar header starts here - self.offset_data = 0 # the file's data starts here - - self.sparse = None # sparse member information - self.pax_headers = {} # pax header information - - @property - def path(self): - 'In pax headers, "name" is called "path".' - return self.name - - @path.setter - def path(self, name): - self.name = name - - @property - def linkpath(self): - 'In pax headers, "linkname" is called "linkpath".' - return self.linkname - - @linkpath.setter - def linkpath(self, linkname): - self.linkname = linkname - - def __repr__(self): - return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) - - def replace(self, *, - name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, - uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, - deep=True, _KEEP=_KEEP): - """Return a deep copy of self with the given attributes replaced. - """ - if deep: - result = copy.deepcopy(self) - else: - result = copy.copy(self) - if name is not _KEEP: - result.name = name - if mtime is not _KEEP: - result.mtime = mtime - if mode is not _KEEP: - result.mode = mode - if linkname is not _KEEP: - result.linkname = linkname - if uid is not _KEEP: - result.uid = uid - if gid is not _KEEP: - result.gid = gid - if uname is not _KEEP: - result.uname = uname - if gname is not _KEEP: - result.gname = gname - return result - - def get_info(self): - """Return the TarInfo's attributes as a dictionary. - """ - if self.mode is None: - mode = None - else: - mode = self.mode & 0o7777 - info = { - "name": self.name, - "mode": mode, - "uid": self.uid, - "gid": self.gid, - "size": self.size, - "mtime": self.mtime, - "chksum": self.chksum, - "type": self.type, - "linkname": self.linkname, - "uname": self.uname, - "gname": self.gname, - "devmajor": self.devmajor, - "devminor": self.devminor - } - - if info["type"] == DIRTYPE and not info["name"].endswith("/"): - info["name"] += "/" - - return info - - def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): - """Return a tar header as a string of 512 byte blocks. - """ - info = self.get_info() - for name, value in info.items(): - if value is None: - raise ValueError("%s may not be None" % name) - - if format == USTAR_FORMAT: - return self.create_ustar_header(info, encoding, errors) - elif format == GNU_FORMAT: - return self.create_gnu_header(info, encoding, errors) - elif format == PAX_FORMAT: - return self.create_pax_header(info, encoding) - else: - raise ValueError("invalid format") - - def create_ustar_header(self, info, encoding, errors): - """Return the object as a ustar header block. - """ - info["magic"] = POSIX_MAGIC - - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - raise ValueError("linkname is too long") - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - info["prefix"], info["name"] = self._posix_split_name(info["name"], encoding, errors) - - return self._create_header(info, USTAR_FORMAT, encoding, errors) - - def create_gnu_header(self, info, encoding, errors): - """Return the object as a GNU header block sequence. - """ - info["magic"] = GNU_MAGIC - - buf = b"" - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) - - return buf + self._create_header(info, GNU_FORMAT, encoding, errors) - - def create_pax_header(self, info, encoding): - """Return the object as a ustar header block. If it cannot be - represented this way, prepend a pax extended header sequence - with supplement information. - """ - info["magic"] = POSIX_MAGIC - pax_headers = self.pax_headers.copy() - - # Test string fields for values that exceed the field length or cannot - # be represented in ASCII encoding. - for name, hname, length in ( - ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), - ("uname", "uname", 32), ("gname", "gname", 32)): - - if hname in pax_headers: - # The pax header has priority. - continue - - # Try to encode the string as ASCII. - try: - info[name].encode("ascii", "strict") - except UnicodeEncodeError: - pax_headers[hname] = info[name] - continue - - if len(info[name]) > length: - pax_headers[hname] = info[name] - - # Test number fields for values that exceed the field limit or values - # that like to be stored as float. - for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): - needs_pax = False - - val = info[name] - val_is_float = isinstance(val, float) - val_int = round(val) if val_is_float else val - if not 0 <= val_int < 8 ** (digits - 1): - # Avoid overflow. - info[name] = 0 - needs_pax = True - elif val_is_float: - # Put rounded value in ustar header, and full - # precision value in pax header. - info[name] = val_int - needs_pax = True - - # The existing pax header has priority. - if needs_pax and name not in pax_headers: - pax_headers[name] = str(val) - - # Create a pax extended header if necessary. - if pax_headers: - buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) - else: - buf = b"" - - return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") - - @classmethod - def create_pax_global_header(cls, pax_headers): - """Return the object as a pax global header block sequence. - """ - return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") - - def _posix_split_name(self, name, encoding, errors): - """Split a name longer than 100 chars into a prefix - and a name part. - """ - components = name.split("/") - for i in range(1, len(components)): - prefix = "/".join(components[:i]) - name = "/".join(components[i:]) - if len(prefix.encode(encoding, errors)) <= LENGTH_PREFIX and \ - len(name.encode(encoding, errors)) <= LENGTH_NAME: - break - else: - raise ValueError("name is too long") - - return prefix, name - - @staticmethod - def _create_header(info, format, encoding, errors): - """Return a header block. info is a dictionary with file - information, format must be one of the *_FORMAT constants. - """ - has_device_fields = info.get("type") in (CHRTYPE, BLKTYPE) - if has_device_fields: - devmajor = itn(info.get("devmajor", 0), 8, format) - devminor = itn(info.get("devminor", 0), 8, format) - else: - devmajor = stn("", 8, encoding, errors) - devminor = stn("", 8, encoding, errors) - - # None values in metadata should cause ValueError. - # itn()/stn() do this for all fields except type. - filetype = info.get("type", REGTYPE) - if filetype is None: - raise ValueError("TarInfo.type must not be None") - - parts = [ - stn(info.get("name", ""), 100, encoding, errors), - itn(info.get("mode", 0) & 0o7777, 8, format), - itn(info.get("uid", 0), 8, format), - itn(info.get("gid", 0), 8, format), - itn(info.get("size", 0), 12, format), - itn(info.get("mtime", 0), 12, format), - b" ", # checksum field - filetype, - stn(info.get("linkname", ""), 100, encoding, errors), - info.get("magic", POSIX_MAGIC), - stn(info.get("uname", ""), 32, encoding, errors), - stn(info.get("gname", ""), 32, encoding, errors), - devmajor, - devminor, - stn(info.get("prefix", ""), 155, encoding, errors) - ] - - buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) - chksum = calc_chksums(buf[-BLOCKSIZE:])[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - - @staticmethod - def _create_payload(payload): - """Return the string payload filled with zero bytes - up to the next 512 byte border. - """ - blocks, remainder = divmod(len(payload), BLOCKSIZE) - if remainder > 0: - payload += (BLOCKSIZE - remainder) * NUL - return payload - - @classmethod - def _create_gnu_long_header(cls, name, type, encoding, errors): - """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence - for name. - """ - name = name.encode(encoding, errors) + NUL - - info = {} - info["name"] = "././@LongLink" - info["type"] = type - info["size"] = len(name) - info["magic"] = GNU_MAGIC - - # create extended header + name blocks. - return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ - cls._create_payload(name) - - @classmethod - def _create_pax_generic_header(cls, pax_headers, type, encoding): - """Return a POSIX.1-2008 extended or global header sequence - that contains a list of keyword, value pairs. The values - must be strings. - """ - # Check if one of the fields contains surrogate characters and thereby - # forces hdrcharset=BINARY, see _proc_pax() for more information. - binary = False - for keyword, value in pax_headers.items(): - try: - value.encode("utf-8", "strict") - except UnicodeEncodeError: - binary = True - break - - records = b"" - if binary: - # Put the hdrcharset field at the beginning of the header. - records += b"21 hdrcharset=BINARY\n" - - for keyword, value in pax_headers.items(): - keyword = keyword.encode("utf-8") - if binary: - # Try to restore the original byte representation of `value'. - # Needless to say, that the encoding must match the string. - value = value.encode(encoding, "surrogateescape") - else: - value = value.encode("utf-8") - - l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' - n = p = 0 - while True: - n = l + len(str(p)) - if n == p: - break - p = n - records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" - - # We use a hardcoded "././@PaxHeader" name like star does - # instead of the one that POSIX recommends. - info = {} - info["name"] = "././@PaxHeader" - info["type"] = type - info["size"] = len(records) - info["magic"] = POSIX_MAGIC - - # Create pax header + record blocks. - return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ - cls._create_payload(records) - - @classmethod - def frombuf(cls, buf, encoding, errors): - """Construct a TarInfo object from a 512 byte bytes object. - """ - if len(buf) == 0: - raise EmptyHeaderError("empty header") - if len(buf) != BLOCKSIZE: - raise TruncatedHeaderError("truncated header") - if buf.count(NUL) == BLOCKSIZE: - raise EOFHeaderError("end of file header") - - chksum = nti(buf[148:156]) - if chksum not in calc_chksums(buf): - raise InvalidHeaderError("bad checksum") - - obj = cls() - obj.name = nts(buf[0:100], encoding, errors) - obj.mode = nti(buf[100:108]) - obj.uid = nti(buf[108:116]) - obj.gid = nti(buf[116:124]) - obj.size = nti(buf[124:136]) - obj.mtime = nti(buf[136:148]) - obj.chksum = chksum - obj.type = buf[156:157] - obj.linkname = nts(buf[157:257], encoding, errors) - obj.uname = nts(buf[265:297], encoding, errors) - obj.gname = nts(buf[297:329], encoding, errors) - obj.devmajor = nti(buf[329:337]) - obj.devminor = nti(buf[337:345]) - prefix = nts(buf[345:500], encoding, errors) - - # Old V7 tar format represents a directory as a regular - # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): - obj.type = DIRTYPE - - # The old GNU sparse format occupies some of the unused - # space in the buffer for up to 4 sparse structures. - # Save them for later processing in _proc_sparse(). - if obj.type == GNUTYPE_SPARSE: - pos = 386 - structs = [] - for i in range(4): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[482]) - origsize = nti(buf[483:495]) - obj._sparse_structs = (structs, isextended, origsize) - - # Remove redundant slashes from directories. - if obj.isdir(): - obj.name = obj.name.rstrip("/") - - # Reconstruct a ustar longname. - if prefix and obj.type not in GNU_TYPES: - obj.name = prefix + "/" + obj.name - return obj - - @classmethod - def fromtarfile(cls, tarfile): - """Return the next TarInfo object from TarFile object - tarfile. - """ - buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) - obj.offset = tarfile.fileobj.tell() - BLOCKSIZE - return obj._proc_member(tarfile) - - #-------------------------------------------------------------------------- - # The following are methods that are called depending on the type of a - # member. The entry point is _proc_member() which can be overridden in a - # subclass to add custom _proc_*() methods. A _proc_*() method MUST - # implement the following - # operations: - # 1. Set self.offset_data to the position where the data blocks begin, - # if there is data that follows. - # 2. Set tarfile.offset to the position where the next member's header will - # begin. - # 3. Return self or another valid TarInfo object. - def _proc_member(self, tarfile): - """Choose the right processing method depending on - the type and call it. - """ - if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): - return self._proc_gnulong(tarfile) - elif self.type == GNUTYPE_SPARSE: - return self._proc_sparse(tarfile) - elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): - return self._proc_pax(tarfile) - else: - return self._proc_builtin(tarfile) - - def _proc_builtin(self, tarfile): - """Process a builtin type or an unknown type which - will be treated as a regular file. - """ - self.offset_data = tarfile.fileobj.tell() - offset = self.offset_data - if self.isreg() or self.type not in SUPPORTED_TYPES: - # Skip the following data blocks. - offset += self._block(self.size) - tarfile.offset = offset - - # Patch the TarInfo object with saved global - # header information. - self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if self.isdir(): - self.name = self.name.rstrip("/") - - return self - - def _proc_gnulong(self, tarfile): - """Process the blocks that hold a GNU longname - or longlink member. - """ - buf = tarfile.fileobj.read(self._block(self.size)) - - # Fetch the next header and process it. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Patch the TarInfo object from the next header with - # the longname information. - next.offset = self.offset - if self.type == GNUTYPE_LONGNAME: - next.name = nts(buf, tarfile.encoding, tarfile.errors) - elif self.type == GNUTYPE_LONGLINK: - next.linkname = nts(buf, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if next.isdir(): - next.name = next.name.removesuffix("/") - - return next - - def _proc_sparse(self, tarfile): - """Process a GNU sparse header plus extra headers. - """ - # We already collected some sparse structures in frombuf(). - structs, isextended, origsize = self._sparse_structs - del self._sparse_structs - - # Collect sparse structures from extended header blocks. - while isextended: - buf = tarfile.fileobj.read(BLOCKSIZE) - pos = 0 - for i in range(21): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset and numbytes: - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[504]) - self.sparse = structs - - self.offset_data = tarfile.fileobj.tell() - tarfile.offset = self.offset_data + self._block(self.size) - self.size = origsize - return self - - def _proc_pax(self, tarfile): - """Process an extended or global header as described in - POSIX.1-2008. - """ - # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) - - # A pax header stores supplemental information for either - # the following file (extended) or all following files - # (global). - if self.type == XGLTYPE: - pax_headers = tarfile.pax_headers - else: - pax_headers = tarfile.pax_headers.copy() - - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - - # Parse pax header information. A record looks like that: - # "%d %s=%s\n" % (length, keyword, value). length is the size - # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") - pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: - raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] - - # Normally, we could just use "utf-8" as the encoding and "strict" - # as the error handler, but we better not take the risk. For - # example, GNU tar <= 1.23 is known to store filenames it cannot - # translate to UTF-8 as raw strings (unfortunately without a - # hdrcharset=BINARY header). - # We first try the strict standard encoding, and if that fails we - # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", - tarfile.errors) - if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, - tarfile.errors) - else: - value = self._decode_pax_field(value, "utf-8", "utf-8", - tarfile.errors) - - pax_headers[keyword] = value - pos += length - - # Fetch the next header. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Process GNU sparse information. - if "GNU.sparse.map" in pax_headers: - # GNU extended sparse format version 0.1. - self._proc_gnusparse_01(next, pax_headers) - - elif "GNU.sparse.size" in pax_headers: - # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) - - elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": - # GNU extended sparse format version 1.0. - self._proc_gnusparse_10(next, pax_headers, tarfile) - - if self.type in (XHDTYPE, SOLARIS_XHDTYPE): - # Patch the TarInfo object with the extended header info. - next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset - - if "size" in pax_headers: - # If the extended header replaces the size field, - # we need to recalculate the offset where the next - # header starts. - offset = next.offset_data - if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) - tarfile.offset = offset - - return next - - def _proc_gnusparse_00(self, next, pax_headers, buf): - """Process a GNU tar extended sparse header, version 0.0. - """ - offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) - numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) - next.sparse = list(zip(offsets, numbytes)) - - def _proc_gnusparse_01(self, next, pax_headers): - """Process a GNU tar extended sparse header, version 0.1. - """ - sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _proc_gnusparse_10(self, next, pax_headers, tarfile): - """Process a GNU tar extended sparse header, version 1.0. - """ - fields = None - sparse = [] - buf = tarfile.fileobj.read(BLOCKSIZE) - fields, buf = buf.split(b"\n", 1) - fields = int(fields) - while len(sparse) < fields * 2: - if b"\n" not in buf: - buf += tarfile.fileobj.read(BLOCKSIZE) - number, buf = buf.split(b"\n", 1) - sparse.append(int(number)) - next.offset_data = tarfile.fileobj.tell() - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _apply_pax_info(self, pax_headers, encoding, errors): - """Replace fields with supplemental information from a previous - pax extended or global header. - """ - for keyword, value in pax_headers.items(): - if keyword == "GNU.sparse.name": - setattr(self, "path", value) - elif keyword == "GNU.sparse.size": - setattr(self, "size", int(value)) - elif keyword == "GNU.sparse.realsize": - setattr(self, "size", int(value)) - elif keyword in PAX_FIELDS: - if keyword in PAX_NUMBER_FIELDS: - try: - value = PAX_NUMBER_FIELDS[keyword](value) - except ValueError: - value = 0 - if keyword == "path": - value = value.rstrip("/") - setattr(self, keyword, value) - - self.pax_headers = pax_headers.copy() - - def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): - """Decode a single field from a pax record. - """ - try: - return value.decode(encoding, "strict") - except UnicodeDecodeError: - return value.decode(fallback_encoding, fallback_errors) - - def _block(self, count): - """Round up a byte count by BLOCKSIZE and return it, - e.g. _block(834) => 1024. - """ - blocks, remainder = divmod(count, BLOCKSIZE) - if remainder: - blocks += 1 - return blocks * BLOCKSIZE - - def isreg(self): - 'Return True if the Tarinfo object is a regular file.' - return self.type in REGULAR_TYPES - - def isfile(self): - 'Return True if the Tarinfo object is a regular file.' - return self.isreg() - - def isdir(self): - 'Return True if it is a directory.' - return self.type == DIRTYPE - - def issym(self): - 'Return True if it is a symbolic link.' - return self.type == SYMTYPE - - def islnk(self): - 'Return True if it is a hard link.' - return self.type == LNKTYPE - - def ischr(self): - 'Return True if it is a character device.' - return self.type == CHRTYPE - - def isblk(self): - 'Return True if it is a block device.' - return self.type == BLKTYPE - - def isfifo(self): - 'Return True if it is a FIFO.' - return self.type == FIFOTYPE - - def issparse(self): - return self.sparse is not None - - def isdev(self): - 'Return True if it is one of character device, block device or FIFO.' - return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) -# class TarInfo - -class TarFile(object): - """The TarFile Class provides an interface to tar archives. - """ - - debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) - - dereference = False # If true, add content of linked file to the - # tar file, else the link. - - ignore_zeros = False # If true, skips empty or invalid blocks and - # continues processing. - - errorlevel = 1 # If 0, fatal errors only appear in debug - # messages (if debug >= 0). If > 0, errors - # are passed to the caller as exceptions. - - format = DEFAULT_FORMAT # The format to use when creating an archive. - - encoding = ENCODING # Encoding for 8-bit character strings. - - errors = None # Error handler for unicode conversion. - - tarinfo = TarInfo # The default TarInfo class to use. - - fileobject = ExFileObject # The file-object for extractfile(). - - extraction_filter = None # The default filter for extraction. - - def __init__(self, name=None, mode="r", fileobj=None, format=None, - tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, - errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to - read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' - defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. - """ - modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} - if mode not in modes: - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - self.mode = mode - self._mode = modes[mode] - - if not fileobj: - if self.mode == "a" and not os.path.exists(name): - # Create nonexistent files in append mode. - self.mode = "w" - self._mode = "wb" - fileobj = bltn_open(name, self._mode) - self._extfileobj = False - else: - if (name is None and hasattr(fileobj, "name") and - isinstance(fileobj.name, (str, bytes))): - name = fileobj.name - if hasattr(fileobj, "mode"): - self._mode = fileobj.mode - self._extfileobj = True - self.name = os.path.abspath(name) if name else None - self.fileobj = fileobj - - # Init attributes. - if format is not None: - self.format = format - if tarinfo is not None: - self.tarinfo = tarinfo - if dereference is not None: - self.dereference = dereference - if ignore_zeros is not None: - self.ignore_zeros = ignore_zeros - if encoding is not None: - self.encoding = encoding - self.errors = errors - - if pax_headers is not None and self.format == PAX_FORMAT: - self.pax_headers = pax_headers - else: - self.pax_headers = {} - - if debug is not None: - self.debug = debug - if errorlevel is not None: - self.errorlevel = errorlevel - - # Init datastructures. - self.copybufsize = copybufsize - self.closed = False - self.members = [] # list of members as TarInfo objects - self._loaded = False # flag if all members have been read - self.offset = self.fileobj.tell() - # current position in the archive file - self.inodes = {} # dictionary caching the inodes of - # archive members already added - - try: - if self.mode == "r": - self.firstmember = None - self.firstmember = self.next() - - if self.mode == "a": - # Move to the end of the archive, - # before the first empty block. - while True: - self.fileobj.seek(self.offset) - try: - tarinfo = self.tarinfo.fromtarfile(self) - self.members.append(tarinfo) - except EOFHeaderError: - self.fileobj.seek(self.offset) - break - except HeaderError as e: - raise ReadError(str(e)) from None - - if self.mode in ("a", "w", "x"): - self._loaded = True - - if self.pax_headers: - buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) - self.fileobj.write(buf) - self.offset += len(buf) - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - #-------------------------------------------------------------------------- - # Below are the classmethods which act as alternate constructors to the - # TarFile class. The open() method is the only one that is needed for - # public use; it is the "super"-constructor and is able to select an - # adequate "sub"-constructor for a particular compression using the mapping - # from OPEN_METH. - # - # This concept allows one to subclass TarFile without losing the comfort of - # the super-constructor. A sub-constructor is registered and made available - # by adding it to the mapping in OPEN_METH. - - @classmethod - def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): - r"""Open a tar archive for reading, writing or appending. Return - an appropriate TarFile class. - - mode: - 'r' or 'r:\*' open for reading with transparent compression - 'r:' open for reading exclusively uncompressed - 'r:gz' open for reading with gzip compression - 'r:bz2' open for reading with bzip2 compression - 'r:xz' open for reading with lzma compression - 'a' or 'a:' open for appending, creating the file if necessary - 'w' or 'w:' open for writing without compression - 'w:gz' open for writing with gzip compression - 'w:bz2' open for writing with bzip2 compression - 'w:xz' open for writing with lzma compression - - 'x' or 'x:' create a tarfile exclusively without compression, raise - an exception if the file is already created - 'x:gz' create a gzip compressed tarfile, raise an exception - if the file is already created - 'x:bz2' create a bzip2 compressed tarfile, raise an exception - if the file is already created - 'x:xz' create an lzma compressed tarfile, raise an exception - if the file is already created - - 'r|\*' open a stream of tar blocks with transparent compression - 'r|' open an uncompressed stream of tar blocks for reading - 'r|gz' open a gzip compressed stream of tar blocks - 'r|bz2' open a bzip2 compressed stream of tar blocks - 'r|xz' open an lzma compressed stream of tar blocks - 'w|' open an uncompressed stream for writing - 'w|gz' open a gzip compressed stream for writing - 'w|bz2' open a bzip2 compressed stream for writing - 'w|xz' open an lzma compressed stream for writing - """ - - if not name and not fileobj: - raise ValueError("nothing to open") - - if mode in ("r", "r:*"): - # Find out which *open() is appropriate for opening the file. - def not_compressed(comptype): - return cls.OPEN_METH[comptype] == 'taropen' - error_msgs = [] - for comptype in sorted(cls.OPEN_METH, key=not_compressed): - func = getattr(cls, cls.OPEN_METH[comptype]) - if fileobj is not None: - saved_pos = fileobj.tell() - try: - return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError) as e: - error_msgs.append(f'- method {comptype}: {e!r}') - if fileobj is not None: - fileobj.seek(saved_pos) - continue - error_msgs_summary = '\n'.join(error_msgs) - raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}") - - elif ":" in mode: - filemode, comptype = mode.split(":", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - # Select the *open() function according to - # given compression. - if comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - else: - raise CompressionError("unknown compression type %r" % comptype) - return func(name, filemode, fileobj, **kwargs) - - elif "|" in mode: - filemode, comptype = mode.split("|", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - if filemode not in ("r", "w"): - raise ValueError("mode must be 'r' or 'w'") - - compresslevel = kwargs.pop("compresslevel", 9) - stream = _Stream(name, filemode, comptype, fileobj, bufsize, - compresslevel) - try: - t = cls(name, filemode, stream, **kwargs) - except: - stream.close() - raise - t._extfileobj = False - return t - - elif mode in ("a", "w", "x"): - return cls.taropen(name, mode, fileobj, **kwargs) - - raise ValueError("undiscernible mode") - - @classmethod - def taropen(cls, name, mode="r", fileobj=None, **kwargs): - """Open uncompressed tar archive name for reading or writing. - """ - if mode not in ("r", "a", "w", "x"): - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - return cls(name, mode, fileobj, **kwargs) - - @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open gzip compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from gzip import GzipFile - except ImportError: - raise CompressionError("gzip module is not available") from None - - try: - fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) - except OSError as e: - if fileobj is not None and mode == 'r': - raise ReadError("not a gzip file") from e - raise - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except OSError as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a gzip file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open bzip2 compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from bz2 import BZ2File - except ImportError: - raise CompressionError("bz2 module is not available") from None - - fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (OSError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a bzip2 file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): - """Open lzma compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from lzma import LZMAFile, LZMAError - except ImportError: - raise CompressionError("lzma module is not available") from None - - fileobj = LZMAFile(fileobj or name, mode, preset=preset) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (LZMAError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not an lzma file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - # All *open() methods are registered here. - OPEN_METH = { - "tar": "taropen", # uncompressed tar - "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar - } - - #-------------------------------------------------------------------------- - # The public methods which TarFile provides: - - def close(self): - """Close the TarFile. In write-mode, two finishing zero blocks are - appended to the archive. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode in ("a", "w", "x"): - self.fileobj.write(NUL * (BLOCKSIZE * 2)) - self.offset += (BLOCKSIZE * 2) - # fill up the end with zero-blocks - # (like option -b20 for tar does) - blocks, remainder = divmod(self.offset, RECORDSIZE) - if remainder > 0: - self.fileobj.write(NUL * (RECORDSIZE - remainder)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def getmember(self, name): - """Return a TarInfo object for member ``name``. If ``name`` can not be - found in the archive, KeyError is raised. If a member occurs more - than once in the archive, its last occurrence is assumed to be the - most up-to-date version. - """ - tarinfo = self._getmember(name.rstrip('/')) - if tarinfo is None: - raise KeyError("filename %r not found" % name) - return tarinfo - - def getmembers(self): - """Return the members of the archive as a list of TarInfo objects. The - list has the same order as the members in the archive. - """ - self._check() - if not self._loaded: # if we want to obtain a list of - self._load() # all members, we first have to - # scan the whole archive. - return self.members - - def getnames(self): - """Return the members of the archive as a list of their names. It has - the same order as the list returned by getmembers(). - """ - return [tarinfo.name for tarinfo in self.getmembers()] - - def gettarinfo(self, name=None, arcname=None, fileobj=None): - """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by ``name``, or - specified as a file object ``fileobj`` with a file descriptor. If - given, ``arcname`` specifies an alternative name for the file in the - archive, otherwise, the name is taken from the 'name' attribute of - 'fileobj', or the 'name' argument. The name should be a text - string. - """ - self._check("awx") - - # When fileobj is given, replace name by - # fileobj's real name. - if fileobj is not None: - name = fileobj.name - - # Building the name of the member in the archive. - # Backward slashes are converted to forward slashes, - # Absolute paths are turned to relative paths. - if arcname is None: - arcname = name - drv, arcname = os.path.splitdrive(arcname) - arcname = arcname.replace(os.sep, "/") - arcname = arcname.lstrip("/") - - # Now, fill the TarInfo object with - # information specific for the file. - tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed - - # Use os.stat or os.lstat, depending on if symlinks shall be resolved. - if fileobj is None: - if not self.dereference: - statres = os.lstat(name) - else: - statres = os.stat(name) - else: - statres = os.fstat(fileobj.fileno()) - linkname = "" - - stmd = statres.st_mode - if stat.S_ISREG(stmd): - inode = (statres.st_ino, statres.st_dev) - if not self.dereference and statres.st_nlink > 1 and \ - inode in self.inodes and arcname != self.inodes[inode]: - # Is it a hardlink to an already - # archived file? - type = LNKTYPE - linkname = self.inodes[inode] - else: - # The inode is added only if its valid. - # For win32 it is always 0. - type = REGTYPE - if inode[0]: - self.inodes[inode] = arcname - elif stat.S_ISDIR(stmd): - type = DIRTYPE - elif stat.S_ISFIFO(stmd): - type = FIFOTYPE - elif stat.S_ISLNK(stmd): - type = SYMTYPE - linkname = os.readlink(name) - elif stat.S_ISCHR(stmd): - type = CHRTYPE - elif stat.S_ISBLK(stmd): - type = BLKTYPE - else: - return None - - # Fill the TarInfo object with all - # information we can get. - tarinfo.name = arcname - tarinfo.mode = stmd - tarinfo.uid = statres.st_uid - tarinfo.gid = statres.st_gid - if type == REGTYPE: - tarinfo.size = statres.st_size - else: - tarinfo.size = 0 - tarinfo.mtime = statres.st_mtime - tarinfo.type = type - tarinfo.linkname = linkname - if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if type in (CHRTYPE, BLKTYPE): - if hasattr(os, "major") and hasattr(os, "minor"): - tarinfo.devmajor = os.major(statres.st_rdev) - tarinfo.devminor = os.minor(statres.st_rdev) - return tarinfo - - def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If ``verbose`` is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. ``members`` is optional and must be a subset of the - list returned by getmembers(). - """ - self._check() - - if members is None: - members = self - for tarinfo in members: - if verbose: - if tarinfo.mode is None: - _safe_print("??????????") - else: - _safe_print(stat.filemode(tarinfo.mode)) - _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid)) - if tarinfo.ischr() or tarinfo.isblk(): - _safe_print("%10s" % - ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) - else: - _safe_print("%10d" % tarinfo.size) - if tarinfo.mtime is None: - _safe_print("????-??-?? ??:??:??") - else: - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) - - _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) - - if verbose: - if tarinfo.issym(): - _safe_print("-> " + tarinfo.linkname) - if tarinfo.islnk(): - _safe_print("link to " + tarinfo.linkname) - print() - - def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file ``name`` to the archive. ``name`` may be any type of file - (directory, fifo, symbolic link, etc.). If given, ``arcname`` - specifies an alternative name for the file in the archive. - Directories are added recursively by default. This can be avoided by - setting ``recursive`` to False. ``filter`` is a function - that expects a TarInfo object argument and returns the changed - TarInfo object, if it returns None the TarInfo object will be - excluded from the archive. - """ - self._check("awx") - - if arcname is None: - arcname = name - - # Skip if somebody tries to archive the archive... - if self.name is not None and os.path.abspath(name) == self.name: - self._dbg(2, "tarfile: Skipped %r" % name) - return - - self._dbg(1, name) - - # Create a TarInfo object from the file. - tarinfo = self.gettarinfo(name, arcname) - - if tarinfo is None: - self._dbg(1, "tarfile: Unsupported type %r" % name) - return - - # Change or exclude the TarInfo object. - if filter is not None: - tarinfo = filter(tarinfo) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Append the tar header and data to the archive. - if tarinfo.isreg(): - with bltn_open(name, "rb") as f: - self.addfile(tarinfo, f) - - elif tarinfo.isdir(): - self.addfile(tarinfo) - if recursive: - for f in sorted(os.listdir(name)): - self.add(os.path.join(name, f), os.path.join(arcname, f), - recursive, filter=filter) - - else: - self.addfile(tarinfo) - - def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object ``tarinfo`` to the archive. If ``fileobj`` is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). - """ - self._check("awx") - - tarinfo = copy.copy(tarinfo) - - buf = tarinfo.tobuf(self.format, self.encoding, self.errors) - self.fileobj.write(buf) - self.offset += len(buf) - bufsize=self.copybufsize - # If there's data to follow, append it. - if fileobj is not None: - copyfileobj(fileobj, self.fileobj, tarinfo.size, bufsize=bufsize) - blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) - if remainder > 0: - self.fileobj.write(NUL * (BLOCKSIZE - remainder)) - blocks += 1 - self.offset += blocks * BLOCKSIZE - - self.members.append(tarinfo) - - def _get_filter_function(self, filter): - if filter is None: - filter = self.extraction_filter - if filter is None: - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter - if isinstance(filter, str): - raise TypeError( - 'String names are not supported for ' - + 'TarFile.extraction_filter. Use a function such as ' - + 'tarfile.data_filter directly.') - return filter - if callable(filter): - return filter - try: - return _NAMED_FILTERS[filter] - except KeyError: - raise ValueError(f"filter {filter!r} not found") from None - - def extractall(self, path=".", members=None, *, numeric_owner=False, - filter=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only - the numbers for user/group names are used and not the names. - - The `filter` function will be called on each member just - before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - directories = [] - - filter_function = self._get_filter_function(filter) - if members is None: - members = self - - for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is None: - continue - if tarinfo.isdir(): - # For directories, delay setting attributes until later, - # since permissions can interfere with extraction and - # extracting contents can reset mtime. - directories.append(tarinfo) - self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) - - # Reverse sort directories. - directories.sort(key=lambda a: a.name, reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, - filter=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` - is True, only the numbers for user/group names are used and not - the names. - - The `filter` function will be called before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) - - def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - unfiltered = tarinfo - try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo - - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" - self._check("r") - - try: - self._extract_member(tarinfo, os.path.join(path, tarinfo.name), - set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def _handle_nonfatal_error(self, e): - """Handle non-fatal error (ExtractError) according to errorlevel""" - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def _handle_fatal_error(self, e): - """Handle "fatal" error according to self.errorlevel""" - if self.errorlevel > 0: - raise - elif isinstance(e, OSError): - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) - else: - self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) - - def extractfile(self, member): - """Extract a member from the archive as a file object. ``member`` may be - a filename or a TarInfo object. If ``member`` is a regular file or - a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If ``member`` does not appear - in the archive, KeyError is raised. - """ - self._check("r") - - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES: - # Members with unknown types are treated as regular files. - return self.fileobject(self, tarinfo) - - elif tarinfo.islnk() or tarinfo.issym(): - if isinstance(self.fileobj, _Stream): - # A small but ugly workaround for the case that someone tries - # to extract a (sym)link as a file-object from a non-seekable - # stream of tar blocks. - raise StreamError("cannot extract (sym)link as file object") - else: - # A (sym)link's file object is its target's file object. - return self.extractfile(self._find_link_target(tarinfo)) - else: - # If there's no data associated with the member (directory, chrdev, - # blkdev, etc.), return None instead of a file object. - return None - - def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical - file called targetpath. - """ - # Fetch the TarInfo object for the given name - # and build the destination pathname, replacing - # forward slashes to platform specific separators. - targetpath = targetpath.rstrip("/") - targetpath = targetpath.replace("/", os.sep) - - # Create all upper directories. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - # Create directories that are not part of the archive with - # default permissions. - os.makedirs(upperdirs) - - if tarinfo.islnk() or tarinfo.issym(): - self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) - else: - self._dbg(1, tarinfo.name) - - if tarinfo.isreg(): - self.makefile(tarinfo, targetpath) - elif tarinfo.isdir(): - self.makedir(tarinfo, targetpath) - elif tarinfo.isfifo(): - self.makefifo(tarinfo, targetpath) - elif tarinfo.ischr() or tarinfo.isblk(): - self.makedev(tarinfo, targetpath) - elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) - elif tarinfo.type not in SUPPORTED_TYPES: - self.makeunknown(tarinfo, targetpath) - else: - self.makefile(tarinfo, targetpath) - - if set_attrs: - self.chown(tarinfo, targetpath, numeric_owner) - if not tarinfo.issym(): - self.chmod(tarinfo, targetpath) - self.utime(tarinfo, targetpath) - - #-------------------------------------------------------------------------- - # Below are the different file methods. They are called via - # _extract_member() when extract() is called. They can be replaced in a - # subclass to implement other functionality. - - def makedir(self, tarinfo, targetpath): - """Make a directory called targetpath. - """ - try: - if tarinfo.mode is None: - # Use the system's default mode - os.mkdir(targetpath) - else: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) - except FileExistsError: - if not os.path.isdir(targetpath): - raise - - def makefile(self, tarinfo, targetpath): - """Make a file called targetpath. - """ - source = self.fileobj - source.seek(tarinfo.offset_data) - bufsize = self.copybufsize - with bltn_open(targetpath, "wb") as target: - if tarinfo.sparse is not None: - for offset, size in tarinfo.sparse: - target.seek(offset) - copyfileobj(source, target, size, ReadError, bufsize) - target.seek(tarinfo.size) - target.truncate() - else: - copyfileobj(source, target, tarinfo.size, ReadError, bufsize) - - def makeunknown(self, tarinfo, targetpath): - """Make a file from a TarInfo object with an unknown type - at targetpath. - """ - self.makefile(tarinfo, targetpath) - self._dbg(1, "tarfile: Unknown file type %r, " \ - "extracted as regular file." % tarinfo.type) - - def makefifo(self, tarinfo, targetpath): - """Make a fifo called targetpath. - """ - if hasattr(os, "mkfifo"): - os.mkfifo(targetpath) - else: - raise ExtractError("fifo not supported by system") - - def makedev(self, tarinfo, targetpath): - """Make a character or block device called targetpath. - """ - if not hasattr(os, "mknod") or not hasattr(os, "makedev"): - raise ExtractError("special devices not supported by system") - - mode = tarinfo.mode - if mode is None: - # Use mknod's default - mode = 0o600 - if tarinfo.isblk(): - mode |= stat.S_IFBLK - else: - mode |= stat.S_IFCHR - - os.mknod(targetpath, mode, - os.makedev(tarinfo.devmajor, tarinfo.devminor)) - - def makelink(self, tarinfo, targetpath): - """Make a (symbolic) link called targetpath. If it cannot be created - (platform limitation), we try to make a copy of the referenced file - instead of a link. - """ - try: - # For systems that support symbolic and hard links. - if tarinfo.issym(): - if os.path.lexists(targetpath): - # Avoid FileExistsError on following os.symlink. - os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) - else: - if os.path.exists(tarinfo._link_target): - os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except symlink_exception: - try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None - - def chown(self, tarinfo, targetpath, numeric_owner): - """Set owner of targetpath according to tarinfo. If numeric_owner - is True, use .gid/.uid instead of .gname/.uname. If numeric_owner - is False, fall back to .gid/.uid when the search based on name - fails. - """ - if hasattr(os, "geteuid") and os.geteuid() == 0: - # We have to be root to do so. - g = tarinfo.gid - u = tarinfo.uid - if not numeric_owner: - try: - if grp and tarinfo.gname: - g = grp.getgrnam(tarinfo.gname)[2] - except KeyError: - pass - try: - if pwd and tarinfo.uname: - u = pwd.getpwnam(tarinfo.uname)[2] - except KeyError: - pass - if g is None: - g = -1 - if u is None: - u = -1 - try: - if tarinfo.issym() and hasattr(os, "lchown"): - os.lchown(targetpath, u, g) - else: - os.chown(targetpath, u, g) - except OSError as e: - raise ExtractError("could not change owner") from e - - def chmod(self, tarinfo, targetpath): - """Set file permissions of targetpath according to tarinfo. - """ - if tarinfo.mode is None: - return - try: - os.chmod(targetpath, tarinfo.mode) - except OSError as e: - raise ExtractError("could not change mode") from e - - def utime(self, tarinfo, targetpath): - """Set modification time of targetpath according to tarinfo. - """ - mtime = tarinfo.mtime - if mtime is None: - return - if not hasattr(os, 'utime'): - return - try: - os.utime(targetpath, (mtime, mtime)) - except OSError as e: - raise ExtractError("could not change modification time") from e - - #-------------------------------------------------------------------------- - def next(self): - """Return the next member of the archive as a TarInfo object, when - TarFile is opened for reading. Return None if there is no more - available. - """ - self._check("ra") - if self.firstmember is not None: - m = self.firstmember - self.firstmember = None - return m - - # Advance the file pointer. - if self.offset != self.fileobj.tell(): - if self.offset == 0: - return None - self.fileobj.seek(self.offset - 1) - if not self.fileobj.read(1): - raise ReadError("unexpected end of data") - - # Read the next block. - tarinfo = None - while True: - try: - tarinfo = self.tarinfo.fromtarfile(self) - except EOFHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - except InvalidHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - elif self.offset == 0: - raise ReadError(str(e)) from None - except EmptyHeaderError: - if self.offset == 0: - raise ReadError("empty file") from None - except TruncatedHeaderError as e: - if self.offset == 0: - raise ReadError(str(e)) from None - except SubsequentHeaderError as e: - raise ReadError(str(e)) from None - except Exception as e: - try: - import zlib - if isinstance(e, zlib.error): - raise ReadError(f'zlib error: {e}') from None - else: - raise e - except ImportError: - raise e - break - - if tarinfo is not None: - self.members.append(tarinfo) - else: - self._loaded = True - - return tarinfo - - #-------------------------------------------------------------------------- - # Little helper methods: - - def _getmember(self, name, tarinfo=None, normalize=False): - """Find an archive member by name from bottom to top. - If tarinfo is given, it is used as the starting point. - """ - # Ensure that all members have been loaded. - members = self.getmembers() - - # Limit the member search list up to tarinfo. - skipping = False - if tarinfo is not None: - try: - index = members.index(tarinfo) - except ValueError: - # The given starting point might be a (modified) copy. - # We'll later skip members until we find an equivalent. - skipping = True - else: - # Happy fast path - members = members[:index] - - if normalize: - name = os.path.normpath(name) - - for member in reversed(members): - if skipping: - if tarinfo.offset == member.offset: - skipping = False - continue - if normalize: - member_name = os.path.normpath(member.name) - else: - member_name = member.name - - if name == member_name: - return member - - if skipping: - # Starting point was not found - raise ValueError(tarinfo) - - def _load(self): - """Read through the entire archive file and look for readable - members. - """ - while self.next() is not None: - pass - self._loaded = True - - def _check(self, mode=None): - """Check if TarFile is still open, and if the operation's mode - corresponds to TarFile's mode. - """ - if self.closed: - raise OSError("%s is closed" % self.__class__.__name__) - if mode is not None and self.mode not in mode: - raise OSError("bad operation for mode %r" % self.mode) - - def _find_link_target(self, tarinfo): - """Find the target member of a symlink or hardlink member in the - archive. - """ - if tarinfo.issym(): - # Always search the entire archive. - linkname = "/".join(filter(None, (os.path.dirname(tarinfo.name), tarinfo.linkname))) - limit = None - else: - # Search the archive before the link, because a hard link is - # just a reference to an already archived file. - linkname = tarinfo.linkname - limit = tarinfo - - member = self._getmember(linkname, tarinfo=limit, normalize=True) - if member is None: - raise KeyError("linkname %r not found" % linkname) - return member - - def __iter__(self): - """Provide an iterator object. - """ - if self._loaded: - yield from self.members - return - - # Yield items using TarFile's next() method. - # When all members have been read, set TarFile as _loaded. - index = 0 - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will have already exhausted the next() method. - if self.firstmember is not None: - tarinfo = self.next() - index += 1 - yield tarinfo - - while True: - if index < len(self.members): - tarinfo = self.members[index] - elif not self._loaded: - tarinfo = self.next() - if not tarinfo: - self._loaded = True - return - else: - return - index += 1 - yield tarinfo - - def _dbg(self, level, msg): - """Write debugging output to sys.stderr. - """ - if level <= self.debug: - print(msg, file=sys.stderr) - - def __enter__(self): - self._check() - return self - - def __exit__(self, type, value, traceback): - if type is None: - self.close() - else: - # An exception occurred. We must not call close() because - # it would try to write end-of-archive blocks and padding. - if not self._extfileobj: - self.fileobj.close() - self.closed = True - -#-------------------- -# exported functions -#-------------------- - -def is_tarfile(name): - """Return True if name points to a tar archive that we - are able to handle, else return False. - - 'name' should be a string, file, or file-like object. - """ - try: - if hasattr(name, "read"): - pos = name.tell() - t = open(fileobj=name) - name.seek(pos) - else: - t = open(name) - t.close() - return True - except TarError: - return False - -open = TarFile.open - - -def main(): - import argparse - - description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='Verbose output') - parser.add_argument('--filter', metavar='<filtername>', - choices=_NAMED_FILTERS, - help='Filter for extraction') - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='<tarfile>', - help='Show listing of a tarfile') - group.add_argument('-e', '--extract', nargs='+', - metavar=('<tarfile>', '<output_dir>'), - help='Extract tarfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('<name>', '<file>'), - help='Create tarfile from sources') - group.add_argument('-t', '--test', metavar='<tarfile>', - help='Test if a tarfile is valid') - - args = parser.parse_args() - - if args.filter and args.extract is None: - parser.exit(1, '--filter is only valid for extraction\n') - - if args.test is not None: - src = args.test - if is_tarfile(src): - with open(src, 'r') as tar: - tar.getmembers() - print(tar.getmembers(), file=sys.stderr) - if args.verbose: - print('{!r} is a tar archive.'.format(src)) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.list is not None: - src = args.list - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.list(verbose=args.verbose) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.extract is not None: - if len(args.extract) == 1: - src = args.extract[0] - curdir = os.curdir - elif len(args.extract) == 2: - src, curdir = args.extract - else: - parser.exit(1, parser.format_help()) - - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir, filter=args.filter) - if args.verbose: - if curdir == '.': - msg = '{!r} file is extracted.'.format(src) - else: - msg = ('{!r} file is extracted ' - 'into {!r} directory.').format(src, curdir) - print(msg) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.create is not None: - tar_name = args.create.pop(0) - _, ext = os.path.splitext(tar_name) - compressions = { - # gz - '.gz': 'gz', - '.tgz': 'gz', - # xz - '.xz': 'xz', - '.txz': 'xz', - # bz2 - '.bz2': 'bz2', - '.tbz': 'bz2', - '.tbz2': 'bz2', - '.tb2': 'bz2', - } - tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' - tar_files = args.create - - with TarFile.open(tar_name, tar_mode) as tf: - for file_name in tar_files: - tf.add(file_name) - - if args.verbose: - print('{!r} file created.'.format(tar_name)) - -if __name__ == '__main__': - main() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py deleted file mode 100644 index 34e3a9950cc..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Read resources contained within a package.""" - -from ._common import ( - as_file, - files, - Package, -) - -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - -from .abc import ResourceReader - - -__all__ = [ - 'Package', - 'Resource', - 'ResourceReader', - 'as_file', - 'contents', - 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py deleted file mode 100644 index ea363d86a56..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_adapters.py +++ /dev/null @@ -1,170 +0,0 @@ -from contextlib import suppress -from io import TextIOWrapper - -from . import abc - - -class SpecLoaderAdapter: - """ - Adapt a package spec to adapt the underlying loader. - """ - - def __init__(self, spec, adapter=lambda spec: spec.loader): - self.spec = spec - self.loader = adapter(spec) - - def __getattr__(self, name): - return getattr(self.spec, name) - - -class TraversableResourcesLoader: - """ - Adapt a loader to provide TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - def get_resource_reader(self, name): - return CompatibilityFiles(self.spec)._native() - - -def _io_wrapper(file, mode='r', *args, **kwargs): - if mode == 'r': - return TextIOWrapper(file, *args, **kwargs) - elif mode == 'rb': - return file - raise ValueError( - "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode) - ) - - -class CompatibilityFiles: - """ - Adapter for an existing or non-existent resource reader - to provide a compatibility .files(). - """ - - class SpecPath(abc.Traversable): - """ - Path tied to a module spec. - Can be read and exposes the resource reader children. - """ - - def __init__(self, spec, reader): - self._spec = spec - self._reader = reader - - def iterdir(self): - if not self._reader: - return iter(()) - return iter( - CompatibilityFiles.ChildPath(self._reader, path) - for path in self._reader.contents() - ) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - if not self._reader: - return CompatibilityFiles.OrphanPath(other) - return CompatibilityFiles.ChildPath(self._reader, other) - - @property - def name(self): - return self._spec.name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) - - class ChildPath(abc.Traversable): - """ - Path tied to a resource reader child. - Can be read but doesn't expose any meaningful children. - """ - - def __init__(self, reader, name): - self._reader = reader - self._name = name - - def iterdir(self): - return iter(()) - - def is_file(self): - return self._reader.is_resource(self.name) - - def is_dir(self): - return not self.is_file() - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(self.name, other) - - @property - def name(self): - return self._name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper( - self._reader.open_resource(self.name), mode, *args, **kwargs - ) - - class OrphanPath(abc.Traversable): - """ - Orphan path, not tied to a module spec or resource reader. - Can't be read and doesn't expose any meaningful children. - """ - - def __init__(self, *path_parts): - if len(path_parts) < 1: - raise ValueError('Need at least one path part to construct a path') - self._path = path_parts - - def iterdir(self): - return iter(()) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(*self._path, other) - - @property - def name(self): - return self._path[-1] - - def open(self, mode='r', *args, **kwargs): - raise FileNotFoundError("Can't open orphan path") - - def __init__(self, spec): - self.spec = spec - - @property - def _reader(self): - with suppress(AttributeError): - return self.spec.loader.get_resource_reader(self.spec.name) - - def _native(self): - """ - Return the native reader if it supports files(). - """ - reader = self._reader - return reader if hasattr(reader, 'files') else self - - def __getattr__(self, attr): - return getattr(self._reader, attr) - - def files(self): - return CompatibilityFiles.SpecPath(self.spec, self._reader) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - """ - return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py deleted file mode 100644 index 3c6de1cfb2e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_common.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -import pathlib -import tempfile -import functools -import contextlib -import types -import importlib -import inspect -import warnings -import itertools - -from typing import Union, Optional, cast -from .abc import ResourceReader, Traversable - -from ._compat import wrap_spec - -Package = Union[types.ModuleType, str] -Anchor = Package - - -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor -def files(anchor: Optional[Anchor] = None) -> Traversable: - """ - Get a Traversable resource for an anchor. - """ - return from_package(resolve(anchor)) - - -def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: - """ - Return the package's loader if it's a ResourceReader. - """ - # We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore - if reader is None: - return None - return reader(spec.name) # type: ignore - - -@functools.singledispatch -def resolve(cand: Optional[Anchor]) -> types.ModuleType: - return cast(types.ModuleType, cand) - - -@resolve.register -def _(cand: str) -> types.ModuleType: - return importlib.import_module(cand) - - -@resolve.register -def _(cand: None) -> types.ModuleType: - return resolve(_infer_caller().f_globals['__name__']) - - -def _infer_caller(): - """ - Walk the stack and find the frame of the first caller not in this module. - """ - - def is_this_file(frame_info): - return frame_info.filename == __file__ - - def is_wrapper(frame_info): - return frame_info.function == 'wrapper' - - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) - # also exclude 'wrapper' due to singledispatch in the call stack - callers = itertools.filterfalse(is_wrapper, not_this_file) - return next(callers).frame - - -def from_package(package: types.ModuleType): - """ - Return a Traversable object for the given package. - - """ - spec = wrap_spec(package) - reader = spec.loader.get_resource_reader(spec.name) - return reader.files() - - -@contextlib.contextmanager -def _tempfile( - reader, - suffix='', - # gh-93353: Keep a reference to call os.remove() in late Python - # finalization. - *, - _os_remove=os.remove, -): - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on Windows - # properly. - fd, raw_path = tempfile.mkstemp(suffix=suffix) - try: - try: - os.write(fd, reader()) - finally: - os.close(fd) - del reader - yield pathlib.Path(raw_path) - finally: - try: - _os_remove(raw_path) - except FileNotFoundError: - pass - - -def _temp_file(path): - return _tempfile(path.read_bytes, suffix=path.name) - - -def _is_present_dir(path: Traversable) -> bool: - """ - Some Traversables implement ``is_dir()`` to raise an - exception (i.e. ``FileNotFoundError``) when the - directory doesn't exist. This function wraps that call - to always return a boolean and only return True - if there's a dir and it exists. - """ - with contextlib.suppress(FileNotFoundError): - return path.is_dir() - return False - - -@functools.singledispatch -def as_file(path): - """ - Given a Traversable object, return that object as a - path on the local file system in a context manager. - """ - return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) - - -@as_file.register(pathlib.Path) -@contextlib.contextmanager -def _(path): - """ - Degenerate behavior for pathlib.Path objects. - """ - yield path - - -@contextlib.contextmanager -def _temp_path(dir: tempfile.TemporaryDirectory): - """ - Wrap tempfile.TemporyDirectory to return a pathlib object. - """ - with dir as result: - yield pathlib.Path(result) - - -@contextlib.contextmanager -def _temp_dir(path): - """ - Given a traversable dir, recursively replicate the whole tree - to the file system in a context manager. - """ - assert path.is_dir() - with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: - yield _write_contents(temp_dir, path) - - -def _write_contents(target, source): - child = target.joinpath(source.name) - if source.is_dir(): - child.mkdir() - for item in source.iterdir(): - _write_contents(child, item) - else: - child.write_bytes(source.read_bytes()) - return child diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py deleted file mode 100644 index 8b5b1d280f3..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_compat.py +++ /dev/null @@ -1,108 +0,0 @@ -# flake8: noqa - -import abc -import os -import sys -import pathlib -from contextlib import suppress -from typing import Union - - -if sys.version_info >= (3, 10): - from zipfile import Path as ZipPath # type: ignore -else: - from ..zipp import Path as ZipPath # type: ignore - - -try: - from typing import runtime_checkable # type: ignore -except ImportError: - - def runtime_checkable(cls): # type: ignore - return cls - - -try: - from typing import Protocol # type: ignore -except ImportError: - Protocol = abc.ABC # type: ignore - - -class TraversableResourcesLoader: - """ - Adapt loaders to provide TraversableResources and other - compatibility. - - Used primarily for Python 3.9 and earlier where the native - loaders do not yet implement TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - @property - def path(self): - return self.spec.origin - - def get_resource_reader(self, name): - from . import readers, _adapters - - def _zip_reader(spec): - with suppress(AttributeError): - return readers.ZipReader(spec.loader, spec.name) - - def _namespace_reader(spec): - with suppress(AttributeError, ValueError): - return readers.NamespaceReader(spec.submodule_search_locations) - - def _available_reader(spec): - with suppress(AttributeError): - return spec.loader.get_resource_reader(spec.name) - - def _native_reader(spec): - reader = _available_reader(spec) - return reader if hasattr(reader, 'files') else None - - def _file_reader(spec): - try: - path = pathlib.Path(self.path) - except TypeError: - return None - if path.exists(): - return readers.FileReader(self) - - return ( - # native reader if it supplies 'files' - _native_reader(self.spec) - or - # local ZipReader if a zip module - _zip_reader(self.spec) - or - # local NamespaceReader if a namespace module - _namespace_reader(self.spec) - or - # local FileReader - _file_reader(self.spec) - # fallback - adapt the spec ResourceReader to TraversableReader - or _adapters.CompatibilityFiles(self.spec) - ) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - - Supersedes _adapters.wrap_spec to use TraversableResourcesLoader - from above for older Python compatibility (<3.10). - """ - from . import _adapters - - return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py deleted file mode 100644 index cce05582ffc..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_itertools.py +++ /dev/null @@ -1,35 +0,0 @@ -from itertools import filterfalse - -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py deleted file mode 100644 index 23b6aeafe4f..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/abc.py +++ /dev/null @@ -1,170 +0,0 @@ -import abc -import io -import itertools -import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional - -from ._compat import runtime_checkable, Protocol, StrPath - - -__all__ = ["ResourceReader", "Traversable", "TraversableResources"] - - -class ResourceReader(metaclass=abc.ABCMeta): - """Abstract base class for loaders to provide resource reading support.""" - - @abc.abstractmethod - def open_resource(self, resource: Text) -> BinaryIO: - """Return an opened, file-like object for binary reading. - - The 'resource' argument is expected to represent only a file name. - If the resource cannot be found, FileNotFoundError is raised. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def resource_path(self, resource: Text) -> Text: - """Return the file system path to the specified resource. - - The 'resource' argument is expected to represent only a file name. - If the resource does not exist on the file system, raise - FileNotFoundError. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def is_resource(self, path: Text) -> bool: - """Return True if the named 'path' is a resource. - - Files are resources, directories are not. - """ - raise FileNotFoundError - - @abc.abstractmethod - def contents(self) -> Iterable[str]: - """Return an iterable of entries in `package`.""" - raise FileNotFoundError - - -class TraversalError(Exception): - pass - - -@runtime_checkable -class Traversable(Protocol): - """ - An object with a subset of pathlib.Path methods suitable for - traversing directories and opening files. - - Any exceptions that occur when accessing the backing resource - may propagate unaltered. - """ - - @abc.abstractmethod - def iterdir(self) -> Iterator["Traversable"]: - """ - Yield Traversable objects in self - """ - - def read_bytes(self) -> bytes: - """ - Read contents of self as bytes - """ - with self.open('rb') as strm: - return strm.read() - - def read_text(self, encoding: Optional[str] = None) -> str: - """ - Read contents of self as text - """ - with self.open(encoding=encoding) as strm: - return strm.read() - - @abc.abstractmethod - def is_dir(self) -> bool: - """ - Return True if self is a directory - """ - - @abc.abstractmethod - def is_file(self) -> bool: - """ - Return True if self is a file - """ - - def joinpath(self, *descendants: StrPath) -> "Traversable": - """ - Return Traversable resolved with any descendants applied. - - Each descendant should be a path segment relative to self - and each may contain multiple levels separated by - ``posixpath.sep`` (``/``). - """ - if not descendants: - return self - names = itertools.chain.from_iterable( - path.parts for path in map(pathlib.PurePosixPath, descendants) - ) - target = next(names) - matches = ( - traversable for traversable in self.iterdir() if traversable.name == target - ) - try: - match = next(matches) - except StopIteration: - raise TraversalError( - "Target not found during traversal.", target, list(names) - ) - return match.joinpath(*names) - - def __truediv__(self, child: StrPath) -> "Traversable": - """ - Return Traversable child in self - """ - return self.joinpath(child) - - @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): - """ - mode may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as pathlib.Path.open). - - When opening as text, accepts encoding parameters such as those - accepted by io.TextIOWrapper. - """ - - @property - @abc.abstractmethod - def name(self) -> str: - """ - The base name of this object without any parent references. - """ - - -class TraversableResources(ResourceReader): - """ - The required interface for providing traversable - resources. - """ - - @abc.abstractmethod - def files(self) -> "Traversable": - """Return a Traversable object for the loaded package.""" - - def open_resource(self, resource: StrPath) -> io.BufferedReader: - return self.files().joinpath(resource).open('rb') - - def resource_path(self, resource: Any) -> NoReturn: - raise FileNotFoundError(resource) - - def is_resource(self, path: StrPath) -> bool: - return self.files().joinpath(path).is_file() - - def contents(self) -> Iterator[str]: - return (item.name for item in self.files().iterdir()) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py deleted file mode 100644 index ab34db74091..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/readers.py +++ /dev/null @@ -1,120 +0,0 @@ -import collections -import pathlib -import operator - -from . import abc - -from ._itertools import unique_everseen -from ._compat import ZipPath - - -def remove_duplicates(items): - return iter(collections.OrderedDict.fromkeys(items)) - - -class FileReader(abc.TraversableResources): - def __init__(self, loader): - self.path = pathlib.Path(loader.path).parent - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path - - -class ZipReader(abc.TraversableResources): - def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' - self.archive = loader.archive - - def open_resource(self, resource): - try: - return super().open_resource(resource) - except KeyError as exc: - raise FileNotFoundError(exc.args[0]) - - def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. - target = self.files().joinpath(path) - return target.is_file() and target.exists() - - def files(self): - return ZipPath(self.archive, self.prefix) - - -class MultiplexedPath(abc.Traversable): - """ - Given a series of Traversable objects, implement a merged - version of the interface across all objects. Useful for - namespace packages which may be multihomed at a single - name. - """ - - def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) - if not self._paths: - message = 'MultiplexedPath must contain at least one path' - raise FileNotFoundError(message) - if not all(path.is_dir() for path in self._paths): - raise NotADirectoryError('MultiplexedPath only supports directories') - - def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) - - def read_bytes(self): - raise FileNotFoundError(f'{self} is not a file') - - def read_text(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - def is_dir(self): - return True - - def is_file(self): - return False - - def joinpath(self, *descendants): - try: - return super().joinpath(*descendants) - except abc.TraversalError: - # One of the paths did not resolve (a directory does not exist). - # Just return something that will not exist. - return self._paths[0].joinpath(*descendants) - - def open(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - @property - def name(self): - return self._paths[0].name - - def __repr__(self): - paths = ', '.join(f"'{path}'" for path in self._paths) - return f'MultiplexedPath({paths})' - - -class NamespaceReader(abc.TraversableResources): - def __init__(self, namespace_path): - if 'NamespacePath' not in str(namespace_path): - raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py deleted file mode 100644 index 7770c922c84..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/importlib_resources/simple.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Interface adapters for low-level readers. -""" - -import abc -import io -import itertools -from typing import BinaryIO, List - -from .abc import Traversable, TraversableResources - - -class SimpleReader(abc.ABC): - """ - The minimum, low-level interface required from a resource - provider. - """ - - @property - @abc.abstractmethod - def package(self) -> str: - """ - The name of the package for which this reader loads resources. - """ - - @abc.abstractmethod - def children(self) -> List['SimpleReader']: - """ - Obtain an iterable of SimpleReader for available - child containers (e.g. directories). - """ - - @abc.abstractmethod - def resources(self) -> List[str]: - """ - Obtain available named resources for this virtual package. - """ - - @abc.abstractmethod - def open_binary(self, resource: str) -> BinaryIO: - """ - Obtain a File-like for a named resource. - """ - - @property - def name(self): - return self.package.split('.')[-1] - - -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader: SimpleReader): - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - -class ResourceHandle(Traversable): - """ - Handle to a named resource in a ResourceReader. - """ - - def __init__(self, parent: ResourceContainer, name: str): - self.parent = parent - self.name = name # type: ignore - - def is_file(self): - return True - - def is_dir(self): - return False - - def open(self, mode='r', *args, **kwargs): - stream = self.parent.reader.open_binary(self.name) - if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) - return stream - - def joinpath(self, name): - raise RuntimeError("Cannot traverse into a resource") - - -class TraversableReader(TraversableResources, SimpleReader): - """ - A TraversableResources based on SimpleReader. Resource providers - may derive from this class to provide the TraversableResources - interface by supplying the SimpleReader interface. - """ - - def files(self): - return ResourceContainer(self) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py deleted file mode 100644 index c42f6135d55..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/context.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import annotations - -import contextlib -import functools -import operator -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.request -import warnings -from typing import Iterator - - -if sys.version_info < (3, 12): - from pkg_resources.extern.backports import tarfile -else: - import tarfile - - -@contextlib.contextmanager -def pushd(dir: str | os.PathLike) -> Iterator[str | os.PathLike]: - """ - >>> tmp_path = getfixture('tmp_path') - >>> with pushd(tmp_path): - ... assert os.getcwd() == os.fspath(tmp_path) - >>> assert os.getcwd() != os.fspath(tmp_path) - """ - - orig = os.getcwd() - os.chdir(dir) - try: - yield dir - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tarball( - url, target_dir: str | os.PathLike | None = None -) -> Iterator[str | os.PathLike]: - """ - Get a tarball, extract it, yield, then clean up. - - >>> import urllib.request - >>> url = getfixture('tarfile_served') - >>> target = getfixture('tmp_path') / 'out' - >>> tb = tarball(url, target_dir=target) - >>> import pathlib - >>> with tb as extracted: - ... contents = pathlib.Path(extracted, 'contents.txt').read_text(encoding='utf-8') - >>> assert not os.path.exists(extracted) - """ - if target_dir is None: - target_dir = os.path.basename(url).replace('.tar.gz', '').replace('.tgz', '') - # In the tar command, use --strip-components=1 to strip the first path and - # then - # use -C to cause the files to be extracted to {target_dir}. This ensures - # that we always know where the files were extracted. - os.mkdir(target_dir) - try: - req = urllib.request.urlopen(url) - with tarfile.open(fileobj=req, mode='r|*') as tf: - tf.extractall(path=target_dir, filter=strip_first_component) - yield target_dir - finally: - shutil.rmtree(target_dir) - - -def strip_first_component( - member: tarfile.TarInfo, - path, -) -> tarfile.TarInfo: - _, member.name = member.name.split('/', 1) - return member - - -def _compose(*cmgrs): - """ - Compose any number of dependent context managers into a single one. - - The last, innermost context manager may take arbitrary arguments, but - each successive context manager should accept the result from the - previous as a single parameter. - - Like :func:`jaraco.functools.compose`, behavior works from right to - left, so the context manager should be indicated from outermost to - innermost. - - Example, to create a context manager to change to a temporary - directory: - - >>> temp_dir_as_cwd = _compose(pushd, temp_dir) - >>> with temp_dir_as_cwd() as dir: - ... assert os.path.samefile(os.getcwd(), dir) - """ - - def compose_two(inner, outer): - def composed(*args, **kwargs): - with inner(*args, **kwargs) as saved, outer(saved) as res: - yield res - - return contextlib.contextmanager(composed) - - return functools.reduce(compose_two, reversed(cmgrs)) - - -tarball_cwd = _compose(pushd, tarball) - - -@contextlib.contextmanager -def tarball_context(*args, **kwargs): - warnings.warn( - "tarball_context is deprecated. Use tarball or tarball_cwd instead.", - DeprecationWarning, - stacklevel=2, - ) - pushd_ctx = kwargs.pop('pushd', pushd) - with tarball(*args, **kwargs) as tball, pushd_ctx(tball) as dir: - yield dir - - -def infer_compression(url): - """ - Given a URL or filename, infer the compression code for tar. - - >>> infer_compression('http://foo/bar.tar.gz') - 'z' - >>> infer_compression('http://foo/bar.tgz') - 'z' - >>> infer_compression('file.bz') - 'j' - >>> infer_compression('file.xz') - 'J' - """ - warnings.warn( - "infer_compression is deprecated with no replacement", - DeprecationWarning, - stacklevel=2, - ) - # cheat and just assume it's the last two characters - compression_indicator = url[-2:] - mapping = dict(gz='z', bz='j', xz='J') - # Assume 'z' (gzip) if no match - return mapping.get(compression_indicator, 'z') - - -@contextlib.contextmanager -def temp_dir(remover=shutil.rmtree): - """ - Create a temporary directory context. Pass a custom remover - to override the removal behavior. - - >>> import pathlib - >>> with temp_dir() as the_dir: - ... assert os.path.isdir(the_dir) - ... _ = pathlib.Path(the_dir).joinpath('somefile').write_text('contents', encoding='utf-8') - >>> assert not os.path.exists(the_dir) - """ - temp_dir = tempfile.mkdtemp() - try: - yield temp_dir - finally: - remover(temp_dir) - - -@contextlib.contextmanager -def repo_context(url, branch=None, quiet=True, dest_ctx=temp_dir): - """ - Check out the repo indicated by url. - - If dest_ctx is supplied, it should be a context manager - to yield the target directory for the check out. - """ - exe = 'git' if 'git' in url else 'hg' - with dest_ctx() as repo_dir: - cmd = [exe, 'clone', url, repo_dir] - if branch: - cmd.extend(['--branch', branch]) - devnull = open(os.path.devnull, 'w') - stdout = devnull if quiet else None - subprocess.check_call(cmd, stdout=stdout) - yield repo_dir - - -def null(): - """ - A null context suitable to stand in for a meaningful context. - - >>> with null() as value: - ... assert value is None - - This context is most useful when dealing with two or more code - branches but only some need a context. Wrap the others in a null - context to provide symmetry across all options. - """ - warnings.warn( - "null is deprecated. Use contextlib.nullcontext", - DeprecationWarning, - stacklevel=2, - ) - return contextlib.nullcontext() - - -class ExceptionTrap: - """ - A context manager that will catch certain exceptions and provide an - indication they occurred. - - >>> with ExceptionTrap() as trap: - ... raise Exception() - >>> bool(trap) - True - - >>> with ExceptionTrap() as trap: - ... pass - >>> bool(trap) - False - - >>> with ExceptionTrap(ValueError) as trap: - ... raise ValueError("1 + 1 is not 3") - >>> bool(trap) - True - >>> trap.value - ValueError('1 + 1 is not 3') - >>> trap.tb - <traceback object at ...> - - >>> with ExceptionTrap(ValueError) as trap: - ... raise Exception() - Traceback (most recent call last): - ... - Exception - - >>> bool(trap) - False - """ - - exc_info = None, None, None - - def __init__(self, exceptions=(Exception,)): - self.exceptions = exceptions - - def __enter__(self): - return self - - @property - def type(self): - return self.exc_info[0] - - @property - def value(self): - return self.exc_info[1] - - @property - def tb(self): - return self.exc_info[2] - - def __exit__(self, *exc_info): - type = exc_info[0] - matches = type and issubclass(type, self.exceptions) - if matches: - self.exc_info = exc_info - return matches - - def __bool__(self): - return bool(self.type) - - def raises(self, func, *, _test=bool): - """ - Wrap func and replace the result with the truth - value of the trap (True if an exception occurred). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> raises = ExceptionTrap(ValueError).raises - - Now decorate a function that always fails. - - >>> @raises - ... def fail(): - ... raise ValueError('failed') - >>> fail() - True - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - with ExceptionTrap(self.exceptions) as trap: - func(*args, **kwargs) - return _test(trap) - - return wrapper - - def passes(self, func): - """ - Wrap func and replace the result with the truth - value of the trap (True if no exception). - - First, give the decorator an alias to support Python 3.8 - Syntax. - - >>> passes = ExceptionTrap(ValueError).passes - - Now decorate a function that always fails. - - >>> @passes - ... def fail(): - ... raise ValueError('failed') - - >>> fail() - False - """ - return self.raises(func, _test=operator.not_) - - -class suppress(contextlib.suppress, contextlib.ContextDecorator): - """ - A version of contextlib.suppress with decorator support. - - >>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ - - -class on_interrupt(contextlib.ContextDecorator): - """ - Replace a KeyboardInterrupt with SystemExit(1) - - >>> def do_interrupt(): - ... raise KeyboardInterrupt() - >>> on_interrupt('error')(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 1 - >>> on_interrupt('error', code=255)(do_interrupt)() - Traceback (most recent call last): - ... - SystemExit: 255 - >>> on_interrupt('suppress')(do_interrupt)() - >>> with __import__('pytest').raises(KeyboardInterrupt): - ... on_interrupt('ignore')(do_interrupt)() - """ - - def __init__(self, action='error', /, code=1): - self.action = action - self.code = code - - def __enter__(self): - return self - - def __exit__(self, exctype, excinst, exctb): - if exctype is not KeyboardInterrupt or self.action == 'ignore': - return - elif self.action == 'error': - raise SystemExit(self.code) from excinst - return self.action == 'suppress' diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py deleted file mode 100644 index f523099c723..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.py +++ /dev/null @@ -1,633 +0,0 @@ -import collections.abc -import functools -import inspect -import itertools -import operator -import time -import types -import warnings - -import pkg_resources.extern.more_itertools - - -def compose(*funcs): - """ - Compose any number of unary functions into a single unary function. - - >>> import textwrap - >>> expected = str.strip(textwrap.dedent(compose.__doc__)) - >>> strip_and_dedent = compose(str.strip, textwrap.dedent) - >>> strip_and_dedent(compose.__doc__) == expected - True - - Compose also allows the innermost function to take arbitrary arguments. - - >>> round_three = lambda x: round(x, ndigits=3) - >>> f = compose(round_three, int.__truediv__) - >>> [f(3*x, x+1) for x in range(1,10)] - [1.5, 2.0, 2.25, 2.4, 2.5, 2.571, 2.625, 2.667, 2.7] - """ - - def compose_two(f1, f2): - return lambda *args, **kwargs: f1(f2(*args, **kwargs)) - - return functools.reduce(compose_two, funcs) - - -def once(func): - """ - Decorate func so it's only ever called the first time. - - This decorator can ensure that an expensive or non-idempotent function - will not be expensive on subsequent calls and is idempotent. - - >>> add_three = once(lambda a: a+3) - >>> add_three(3) - 6 - >>> add_three(9) - 6 - >>> add_three('12') - 6 - - To reset the stored value, simply clear the property ``saved_result``. - - >>> del add_three.saved_result - >>> add_three(9) - 12 - >>> add_three(8) - 12 - - Or invoke 'reset()' on it. - - >>> add_three.reset() - >>> add_three(-3) - 0 - >>> add_three(0) - 0 - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if not hasattr(wrapper, 'saved_result'): - wrapper.saved_result = func(*args, **kwargs) - return wrapper.saved_result - - wrapper.reset = lambda: vars(wrapper).__delitem__('saved_result') - return wrapper - - -def method_cache(method, cache_wrapper=functools.lru_cache()): - """ - Wrap lru_cache to support storing the cache data in the object instances. - - Abstracts the common paradigm where the method explicitly saves an - underscore-prefixed protected property on first call and returns that - subsequently. - - >>> class MyClass: - ... calls = 0 - ... - ... @method_cache - ... def method(self, value): - ... self.calls += 1 - ... return value - - >>> a = MyClass() - >>> a.method(3) - 3 - >>> for x in range(75): - ... res = a.method(x) - >>> a.calls - 75 - - Note that the apparent behavior will be exactly like that of lru_cache - except that the cache is stored on each instance, so values in one - instance will not flush values from another, and when an instance is - deleted, so are the cached values for that instance. - - >>> b = MyClass() - >>> for x in range(35): - ... res = b.method(x) - >>> b.calls - 35 - >>> a.method(0) - 0 - >>> a.calls - 75 - - Note that if method had been decorated with ``functools.lru_cache()``, - a.calls would have been 76 (due to the cached value of 0 having been - flushed by the 'b' instance). - - Clear the cache with ``.cache_clear()`` - - >>> a.method.cache_clear() - - Same for a method that hasn't yet been called. - - >>> c = MyClass() - >>> c.method.cache_clear() - - Another cache wrapper may be supplied: - - >>> cache = functools.lru_cache(maxsize=2) - >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) - >>> a = MyClass() - >>> a.method2() - 3 - - Caution - do not subsequently wrap the method with another decorator, such - as ``@property``, which changes the semantics of the function. - - See also - http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ - for another implementation and additional justification. - """ - - def wrapper(self, *args, **kwargs): - # it's the first call, replace the method with a cached, bound method - bound_method = types.MethodType(method, self) - cached_method = cache_wrapper(bound_method) - setattr(self, method.__name__, cached_method) - return cached_method(*args, **kwargs) - - # Support cache clear even before cache has been created. - wrapper.cache_clear = lambda: None - - return _special_method_cache(method, cache_wrapper) or wrapper - - -def _special_method_cache(method, cache_wrapper): - """ - Because Python treats special methods differently, it's not - possible to use instance attributes to implement the cached - methods. - - Instead, install the wrapper method under a different name - and return a simple proxy to that wrapper. - - https://github.com/jaraco/jaraco.functools/issues/5 - """ - name = method.__name__ - special_names = '__getattr__', '__getitem__' - - if name not in special_names: - return None - - wrapper_name = '__cached' + name - - def proxy(self, /, *args, **kwargs): - if wrapper_name not in vars(self): - bound = types.MethodType(method, self) - cache = cache_wrapper(bound) - setattr(self, wrapper_name, cache) - else: - cache = getattr(self, wrapper_name) - return cache(*args, **kwargs) - - return proxy - - -def apply(transform): - """ - Decorate a function with a transform function that is - invoked on results returned from the decorated function. - - >>> @apply(reversed) - ... def get_numbers(start): - ... "doc for get_numbers" - ... return range(start, start+3) - >>> list(get_numbers(4)) - [6, 5, 4] - >>> get_numbers.__doc__ - 'doc for get_numbers' - """ - - def wrap(func): - return functools.wraps(func)(compose(transform, func)) - - return wrap - - -def result_invoke(action): - r""" - Decorate a function with an action function that is - invoked on the results returned from the decorated - function (for its side effect), then return the original - result. - - >>> @result_invoke(print) - ... def add_two(a, b): - ... return a + b - >>> x = add_two(2, 3) - 5 - >>> x - 5 - """ - - def wrap(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - action(result) - return result - - return wrapper - - return wrap - - -def invoke(f, /, *args, **kwargs): - """ - Call a function for its side effect after initialization. - - The benefit of using the decorator instead of simply invoking a function - after defining it is that it makes explicit the author's intent for the - function to be called immediately. Whereas if one simply calls the - function immediately, it's less obvious if that was intentional or - incidental. It also avoids repeating the name - the two actions, defining - the function and calling it immediately are modeled separately, but linked - by the decorator construct. - - The benefit of having a function construct (opposed to just invoking some - behavior inline) is to serve as a scope in which the behavior occurs. It - avoids polluting the global namespace with local variables, provides an - anchor on which to attach documentation (docstring), keeps the behavior - logically separated (instead of conceptually separated or not separated at - all), and provides potential to re-use the behavior for testing or other - purposes. - - This function is named as a pithy way to communicate, "call this function - primarily for its side effect", or "while defining this function, also - take it aside and call it". It exists because there's no Python construct - for "define and call" (nor should there be, as decorators serve this need - just fine). The behavior happens immediately and synchronously. - - >>> @invoke - ... def func(): print("called") - called - >>> func() - called - - Use functools.partial to pass parameters to the initial call - - >>> @functools.partial(invoke, name='bingo') - ... def func(name): print('called with', name) - called with bingo - """ - f(*args, **kwargs) - return f - - -class Throttler: - """Rate-limit a function (or other callable).""" - - def __init__(self, func, max_rate=float('Inf')): - if isinstance(func, Throttler): - func = func.func - self.func = func - self.max_rate = max_rate - self.reset() - - def reset(self): - self.last_called = 0 - - def __call__(self, *args, **kwargs): - self._wait() - return self.func(*args, **kwargs) - - def _wait(self): - """Ensure at least 1/max_rate seconds from last call.""" - elapsed = time.time() - self.last_called - must_wait = 1 / self.max_rate - elapsed - time.sleep(max(0, must_wait)) - self.last_called = time.time() - - def __get__(self, obj, owner=None): - return first_invoke(self._wait, functools.partial(self.func, obj)) - - -def first_invoke(func1, func2): - """ - Return a function that when invoked will invoke func1 without - any parameters (for its side effect) and then invoke func2 - with whatever parameters were passed, returning its result. - """ - - def wrapper(*args, **kwargs): - func1() - return func2(*args, **kwargs) - - return wrapper - - -method_caller = first_invoke( - lambda: warnings.warn( - '`jaraco.functools.method_caller` is deprecated, ' - 'use `operator.methodcaller` instead', - DeprecationWarning, - stacklevel=3, - ), - operator.methodcaller, -) - - -def retry_call(func, cleanup=lambda: None, retries=0, trap=()): - """ - Given a callable func, trap the indicated exceptions - for up to 'retries' times, invoking cleanup on the - exception. On the final attempt, allow any exceptions - to propagate. - """ - attempts = itertools.count() if retries == float('inf') else range(retries) - for _ in attempts: - try: - return func() - except trap: - cleanup() - - return func() - - -def retry(*r_args, **r_kwargs): - """ - Decorator wrapper for retry_call. Accepts arguments to retry_call - except func and then returns a decorator for the decorated function. - - Ex: - - >>> @retry(retries=3) - ... def my_func(a, b): - ... "this is my funk" - ... print(a, b) - >>> my_func.__doc__ - 'this is my funk' - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*f_args, **f_kwargs): - bound = functools.partial(func, *f_args, **f_kwargs) - return retry_call(bound, *r_args, **r_kwargs) - - return wrapper - - return decorate - - -def print_yielded(func): - """ - Convert a generator into a function that prints all yielded elements. - - >>> @print_yielded - ... def x(): - ... yield 3; yield None - >>> x() - 3 - None - """ - print_all = functools.partial(map, print) - print_results = compose(more_itertools.consume, print_all, func) - return functools.wraps(func)(print_results) - - -def pass_none(func): - """ - Wrap func so it's not called if its first param is None. - - >>> print_text = pass_none(print) - >>> print_text('text') - text - >>> print_text(None) - """ - - @functools.wraps(func) - def wrapper(param, /, *args, **kwargs): - if param is not None: - return func(param, *args, **kwargs) - return None - - return wrapper - - -def assign_params(func, namespace): - """ - Assign parameters from namespace where func solicits. - - >>> def func(x, y=3): - ... print(x, y) - >>> assigned = assign_params(func, dict(x=2, z=4)) - >>> assigned() - 2 3 - - The usual errors are raised if a function doesn't receive - its required parameters: - - >>> assigned = assign_params(func, dict(y=3, z=4)) - >>> assigned() - Traceback (most recent call last): - TypeError: func() ...argument... - - It even works on methods: - - >>> class Handler: - ... def meth(self, arg): - ... print(arg) - >>> assign_params(Handler().meth, dict(arg='crystal', foo='clear'))() - crystal - """ - sig = inspect.signature(func) - params = sig.parameters.keys() - call_ns = {k: namespace[k] for k in params if k in namespace} - return functools.partial(func, **call_ns) - - -def save_method_args(method): - """ - Wrap a method such that when it is called, the args and kwargs are - saved on the method. - - >>> class MyClass: - ... @save_method_args - ... def method(self, a, b): - ... print(a, b) - >>> my_ob = MyClass() - >>> my_ob.method(1, 2) - 1 2 - >>> my_ob._saved_method.args - (1, 2) - >>> my_ob._saved_method.kwargs - {} - >>> my_ob.method(a=3, b='foo') - 3 foo - >>> my_ob._saved_method.args - () - >>> my_ob._saved_method.kwargs == dict(a=3, b='foo') - True - - The arguments are stored on the instance, allowing for - different instance to save different args. - - >>> your_ob = MyClass() - >>> your_ob.method({str('x'): 3}, b=[4]) - {'x': 3} [4] - >>> your_ob._saved_method.args - ({'x': 3},) - >>> my_ob._saved_method.args - () - """ - args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') - - @functools.wraps(method) - def wrapper(self, /, *args, **kwargs): - attr_name = '_saved_' + method.__name__ - attr = args_and_kwargs(args, kwargs) - setattr(self, attr_name, attr) - return method(self, *args, **kwargs) - - return wrapper - - -def except_(*exceptions, replace=None, use=None): - """ - Replace the indicated exceptions, if raised, with the indicated - literal replacement or evaluated expression (if present). - - >>> safe_int = except_(ValueError)(int) - >>> safe_int('five') - >>> safe_int('5') - 5 - - Specify a literal replacement with ``replace``. - - >>> safe_int_r = except_(ValueError, replace=0)(int) - >>> safe_int_r('five') - 0 - - Provide an expression to ``use`` to pass through particular parameters. - - >>> safe_int_pt = except_(ValueError, use='args[0]')(int) - >>> safe_int_pt('five') - 'five' - - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except exceptions: - try: - return eval(use) - except TypeError: - return replace - - return wrapper - - return decorate - - -def identity(x): - """ - Return the argument. - - >>> o = object() - >>> identity(o) is o - True - """ - return x - - -def bypass_when(check, *, _op=identity): - """ - Decorate a function to return its parameter when ``check``. - - >>> bypassed = [] # False - - >>> @bypass_when(bypassed) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> bypassed[:] = [object()] # True - >>> double(2) - 2 - """ - - def decorate(func): - @functools.wraps(func) - def wrapper(param, /): - return param if _op(check) else func(param) - - return wrapper - - return decorate - - -def bypass_unless(check): - """ - Decorate a function to return its parameter unless ``check``. - - >>> enabled = [object()] # True - - >>> @bypass_unless(enabled) - ... def double(x): - ... return x * 2 - >>> double(2) - 4 - >>> del enabled[:] # False - >>> double(2) - 2 - """ - return bypass_when(check, _op=operator.not_) - - -@functools.singledispatch -def _splat_inner(args, func): - """Splat args to func.""" - return func(*args) - - -@_splat_inner.register -def _(args: collections.abc.Mapping, func): - """Splat kargs to func as kwargs.""" - return func(**args) - - -def splat(func): - """ - Wrap func to expect its parameters to be passed positionally in a tuple. - - Has a similar effect to that of ``itertools.starmap`` over - simple ``map``. - - >>> pairs = [(-1, 1), (0, 2)] - >>> pkg_resources.extern.more_itertools.consume(itertools.starmap(print, pairs)) - -1 1 - 0 2 - >>> pkg_resources.extern.more_itertools.consume(map(splat(print), pairs)) - -1 1 - 0 2 - - The approach generalizes to other iterators that don't have a "star" - equivalent, such as a "starfilter". - - >>> list(filter(splat(operator.add), pairs)) - [(0, 2)] - - Splat also accepts a mapping argument. - - >>> def is_nice(msg, code): - ... return "smile" in msg or code == 0 - >>> msgs = [ - ... dict(msg='smile!', code=20), - ... dict(msg='error :(', code=1), - ... dict(msg='unknown', code=0), - ... ] - >>> for msg in filter(splat(is_nice), msgs): - ... print(msg) - {'msg': 'smile!', 'code': 20} - {'msg': 'unknown', 'code': 0} - """ - return functools.wraps(func)(functools.partial(_splat_inner, func=func)) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi deleted file mode 100644 index c2b9ab1757e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/jaraco/functools/__init__.pyi +++ /dev/null @@ -1,128 +0,0 @@ -from collections.abc import Callable, Hashable, Iterator -from functools import partial -from operator import methodcaller -import sys -from typing import ( - Any, - Generic, - Protocol, - TypeVar, - overload, -) - -if sys.version_info >= (3, 10): - from typing import Concatenate, ParamSpec -else: - from typing_extensions import Concatenate, ParamSpec - -_P = ParamSpec('_P') -_R = TypeVar('_R') -_T = TypeVar('_T') -_R1 = TypeVar('_R1') -_R2 = TypeVar('_R2') -_V = TypeVar('_V') -_S = TypeVar('_S') -_R_co = TypeVar('_R_co', covariant=True) - -class _OnceCallable(Protocol[_P, _R]): - saved_result: _R - reset: Callable[[], None] - def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... - -class _ProxyMethodCacheWrapper(Protocol[_R_co]): - cache_clear: Callable[[], None] - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -class _MethodCacheWrapper(Protocol[_R_co]): - def cache_clear(self) -> None: ... - def __call__(self, *args: Hashable, **kwargs: Hashable) -> _R_co: ... - -# `compose()` overloads below will cover most use cases. - -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[_P, _R], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R1], _R], - __func3: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -@overload -def compose( - __func1: Callable[[_R], _T], - __func2: Callable[[_R2], _R], - __func3: Callable[[_R1], _R2], - __func4: Callable[_P, _R1], - /, -) -> Callable[_P, _T]: ... -def once(func: Callable[_P, _R]) -> _OnceCallable[_P, _R]: ... -def method_cache( - method: Callable[..., _R], - cache_wrapper: Callable[[Callable[..., _R]], _MethodCacheWrapper[_R]] = ..., -) -> _MethodCacheWrapper[_R] | _ProxyMethodCacheWrapper[_R]: ... -def apply( - transform: Callable[[_R], _T] -) -> Callable[[Callable[_P, _R]], Callable[_P, _T]]: ... -def result_invoke( - action: Callable[[_R], Any] -) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: ... -def invoke( - f: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... -def call_aside( - f: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs -) -> Callable[_P, _R]: ... - -class Throttler(Generic[_R]): - last_called: float - func: Callable[..., _R] - max_rate: float - def __init__( - self, func: Callable[..., _R] | Throttler[_R], max_rate: float = ... - ) -> None: ... - def reset(self) -> None: ... - def __call__(self, *args: Any, **kwargs: Any) -> _R: ... - def __get__(self, obj: Any, owner: type[Any] | None = ...) -> Callable[..., _R]: ... - -def first_invoke( - func1: Callable[..., Any], func2: Callable[_P, _R] -) -> Callable[_P, _R]: ... - -method_caller: Callable[..., methodcaller] - -def retry_call( - func: Callable[..., _R], - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> _R: ... -def retry( - cleanup: Callable[..., None] = ..., - retries: int | float = ..., - trap: type[BaseException] | tuple[type[BaseException], ...] = ..., -) -> Callable[[Callable[..., _R]], Callable[..., _R]]: ... -def print_yielded(func: Callable[_P, Iterator[Any]]) -> Callable[_P, None]: ... -def pass_none( - func: Callable[Concatenate[_T, _P], _R] -) -> Callable[Concatenate[_T, _P], _R]: ... -def assign_params( - func: Callable[..., _R], namespace: dict[str, Any] -) -> partial[_R]: ... -def save_method_args( - method: Callable[Concatenate[_S, _P], _R] -) -> Callable[Concatenate[_S, _P], _R]: ... -def except_( - *exceptions: type[BaseException], replace: Any = ..., use: Any = ... -) -> Callable[[Callable[_P, Any]], Callable[_P, Any]]: ... -def identity(x: _T) -> _T: ... -def bypass_when( - check: _V, *, _op: Callable[[_V], Any] = ... -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... -def bypass_unless( - check: Any, -) -> Callable[[Callable[[_T], _R]], Callable[[_T], _T | _R]]: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py deleted file mode 100644 index aff94a9abd0..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""More routines for operating on iterables, beyond itertools""" - -from .more import * # noqa -from .recipes import * # noqa - -__version__ = '10.2.0' diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi deleted file mode 100644 index 96f6e36c7f4..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/__init__.pyi +++ /dev/null @@ -1,2 +0,0 @@ -from .more import * -from .recipes import * diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py deleted file mode 100644 index d0957681f54..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.py +++ /dev/null @@ -1,4655 +0,0 @@ -import warnings - -from collections import Counter, defaultdict, deque, abc -from collections.abc import Sequence -from functools import cached_property, partial, reduce, wraps -from heapq import heapify, heapreplace, heappop -from itertools import ( - chain, - compress, - count, - cycle, - dropwhile, - groupby, - islice, - repeat, - starmap, - takewhile, - tee, - zip_longest, - product, -) -from math import exp, factorial, floor, log, perm, comb -from queue import Empty, Queue -from random import random, randrange, uniform -from operator import itemgetter, mul, sub, gt, lt, ge, le -from sys import hexversion, maxsize -from time import monotonic - -from .recipes import ( - _marker, - _zip_equal, - UnequalIterablesError, - consume, - flatten, - pairwise, - powerset, - take, - unique_everseen, - all_equal, - batched, -) - -__all__ = [ - 'AbortThread', - 'SequenceView', - 'UnequalIterablesError', - 'adjacent', - 'all_unique', - 'always_iterable', - 'always_reversible', - 'bucket', - 'callback_iter', - 'chunked', - 'chunked_even', - 'circular_shifts', - 'collapse', - 'combination_index', - 'combination_with_replacement_index', - 'consecutive_groups', - 'constrained_batches', - 'consumer', - 'count_cycle', - 'countable', - 'difference', - 'distinct_combinations', - 'distinct_permutations', - 'distribute', - 'divide', - 'duplicates_everseen', - 'duplicates_justseen', - 'classify_unique', - 'exactly_n', - 'filter_except', - 'filter_map', - 'first', - 'gray_product', - 'groupby_transform', - 'ichunked', - 'iequals', - 'ilen', - 'interleave', - 'interleave_evenly', - 'interleave_longest', - 'intersperse', - 'is_sorted', - 'islice_extended', - 'iterate', - 'iter_suppress', - 'last', - 'locate', - 'longest_common_prefix', - 'lstrip', - 'make_decorator', - 'map_except', - 'map_if', - 'map_reduce', - 'mark_ends', - 'minmax', - 'nth_or_last', - 'nth_permutation', - 'nth_product', - 'nth_combination_with_replacement', - 'numeric_range', - 'one', - 'only', - 'outer_product', - 'padded', - 'partial_product', - 'partitions', - 'peekable', - 'permutation_index', - 'product_index', - 'raise_', - 'repeat_each', - 'repeat_last', - 'replace', - 'rlocate', - 'rstrip', - 'run_length', - 'sample', - 'seekable', - 'set_partitions', - 'side_effect', - 'sliced', - 'sort_together', - 'split_after', - 'split_at', - 'split_before', - 'split_into', - 'split_when', - 'spy', - 'stagger', - 'strip', - 'strictly_n', - 'substrings', - 'substrings_indexes', - 'takewhile_inclusive', - 'time_limited', - 'unique_in_window', - 'unique_to_each', - 'unzip', - 'value_chain', - 'windowed', - 'windowed_complete', - 'with_iter', - 'zip_broadcast', - 'zip_equal', - 'zip_offset', -] - - -def chunked(iterable, n, strict=False): - """Break *iterable* into lists of length *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) - [[1, 2, 3], [4, 5, 6]] - - By the default, the last yielded list will have fewer than *n* elements - if the length of *iterable* is not divisible by *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) - [[1, 2, 3], [4, 5, 6], [7, 8]] - - To use a fill-in value instead, see the :func:`grouper` recipe. - - If the length of *iterable* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - list is yielded. - - """ - iterator = iter(partial(take, n, iter(iterable)), []) - if strict: - if n is None: - raise ValueError('n must not be None when using strict mode.') - - def ret(): - for chunk in iterator: - if len(chunk) != n: - raise ValueError('iterable is not divisible by n.') - yield chunk - - return iter(ret()) - else: - return iterator - - -def first(iterable, default=_marker): - """Return the first item of *iterable*, or *default* if *iterable* is - empty. - - >>> first([0, 1, 2, 3]) - 0 - >>> first([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - - :func:`first` is useful when you have a generator of expensive-to-retrieve - values and want any arbitrary one. It is marginally shorter than - ``next(iter(iterable), default)``. - - """ - for item in iterable: - return item - if default is _marker: - raise ValueError( - 'first() was called on an empty iterable, and no ' - 'default value was provided.' - ) - return default - - -def last(iterable, default=_marker): - """Return the last item of *iterable*, or *default* if *iterable* is - empty. - - >>> last([0, 1, 2, 3]) - 3 - >>> last([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - try: - if isinstance(iterable, Sequence): - return iterable[-1] - # Work around https://bugs.python.org/issue38525 - elif hasattr(iterable, '__reversed__') and (hexversion != 0x030800F0): - return next(reversed(iterable)) - else: - return deque(iterable, maxlen=1)[-1] - except (IndexError, TypeError, StopIteration): - if default is _marker: - raise ValueError( - 'last() was called on an empty iterable, and no default was ' - 'provided.' - ) - return default - - -def nth_or_last(iterable, n, default=_marker): - """Return the nth or the last item of *iterable*, - or *default* if *iterable* is empty. - - >>> nth_or_last([0, 1, 2, 3], 2) - 2 - >>> nth_or_last([0, 1], 2) - 1 - >>> nth_or_last([], 0, 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - return last(islice(iterable, n + 1), default=default) - - -class peekable: - """Wrap an iterator to allow lookahead and prepending elements. - - Call :meth:`peek` on the result to get the value that will be returned - by :func:`next`. This won't advance the iterator: - - >>> p = peekable(['a', 'b']) - >>> p.peek() - 'a' - >>> next(p) - 'a' - - Pass :meth:`peek` a default value to return that instead of raising - ``StopIteration`` when the iterator is exhausted. - - >>> p = peekable([]) - >>> p.peek('hi') - 'hi' - - peekables also offer a :meth:`prepend` method, which "inserts" items - at the head of the iterable: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> p.peek() - 11 - >>> list(p) - [11, 12, 1, 2, 3] - - peekables can be indexed. Index 0 is the item that will be returned by - :func:`next`, index 1 is the item after that, and so on: - The values up to the given index will be cached. - - >>> p = peekable(['a', 'b', 'c', 'd']) - >>> p[0] - 'a' - >>> p[1] - 'b' - >>> next(p) - 'a' - - Negative indexes are supported, but be aware that they will cache the - remaining items in the source iterator, which may require significant - storage. - - To check whether a peekable is exhausted, check its truth value: - - >>> p = peekable(['a', 'b']) - >>> if p: # peekable has items - ... list(p) - ['a', 'b'] - >>> if not p: # peekable is exhausted - ... list(p) - [] - - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self._cache = deque() - - def __iter__(self): - return self - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - """Return the item that will be next returned from ``next()``. - - Return ``default`` if there are no items left. If ``default`` is not - provided, raise ``StopIteration``. - - """ - if not self._cache: - try: - self._cache.append(next(self._it)) - except StopIteration: - if default is _marker: - raise - return default - return self._cache[0] - - def prepend(self, *items): - """Stack up items to be the next ones returned from ``next()`` or - ``self.peek()``. The items will be returned in - first in, first out order:: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> list(p) - [11, 12, 1, 2, 3] - - It is possible, by prepending items, to "resurrect" a peekable that - previously raised ``StopIteration``. - - >>> p = peekable([]) - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - >>> p.prepend(1) - >>> next(p) - 1 - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - - """ - self._cache.extendleft(reversed(items)) - - def __next__(self): - if self._cache: - return self._cache.popleft() - - return next(self._it) - - def _get_slice(self, index): - # Normalize the slice's arguments - step = 1 if (index.step is None) else index.step - if step > 0: - start = 0 if (index.start is None) else index.start - stop = maxsize if (index.stop is None) else index.stop - elif step < 0: - start = -1 if (index.start is None) else index.start - stop = (-maxsize - 1) if (index.stop is None) else index.stop - else: - raise ValueError('slice step cannot be zero') - - # If either the start or stop index is negative, we'll need to cache - # the rest of the iterable in order to slice from the right side. - if (start < 0) or (stop < 0): - self._cache.extend(self._it) - # Otherwise we'll need to find the rightmost index and cache to that - # point. - else: - n = min(max(start, stop) + 1, maxsize) - cache_len = len(self._cache) - if n >= cache_len: - self._cache.extend(islice(self._it, n - cache_len)) - - return list(self._cache)[index] - - def __getitem__(self, index): - if isinstance(index, slice): - return self._get_slice(index) - - cache_len = len(self._cache) - if index < 0: - self._cache.extend(self._it) - elif index >= cache_len: - self._cache.extend(islice(self._it, index + 1 - cache_len)) - - return self._cache[index] - - -def consumer(func): - """Decorator that automatically advances a PEP-342-style "reverse iterator" - to its first yield point so you don't have to call ``next()`` on it - manually. - - >>> @consumer - ... def tally(): - ... i = 0 - ... while True: - ... print('Thing number %s is %s.' % (i, (yield))) - ... i += 1 - ... - >>> t = tally() - >>> t.send('red') - Thing number 0 is red. - >>> t.send('fish') - Thing number 1 is fish. - - Without the decorator, you would have to call ``next(t)`` before - ``t.send()`` could be used. - - """ - - @wraps(func) - def wrapper(*args, **kwargs): - gen = func(*args, **kwargs) - next(gen) - return gen - - return wrapper - - -def ilen(iterable): - """Return the number of items in *iterable*. - - >>> ilen(x for x in range(1000000) if x % 3 == 0) - 333334 - - This consumes the iterable, so handle with care. - - """ - # This approach was selected because benchmarks showed it's likely the - # fastest of the known implementations at the time of writing. - # See GitHub tracker: #236, #230. - counter = count() - deque(zip(iterable, counter), maxlen=0) - return next(counter) - - -def iterate(func, start): - """Return ``start``, ``func(start)``, ``func(func(start))``, ... - - >>> from itertools import islice - >>> list(islice(iterate(lambda x: 2*x, 1), 10)) - [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - - """ - while True: - yield start - try: - start = func(start) - except StopIteration: - break - - -def with_iter(context_manager): - """Wrap an iterable in a ``with`` statement, so it closes once exhausted. - - For example, this will close the file when the iterator is exhausted:: - - upper_lines = (line.upper() for line in with_iter(open('foo'))) - - Any context manager which returns an iterable is a candidate for - ``with_iter``. - - """ - with context_manager as iterable: - yield from iterable - - -def one(iterable, too_short=None, too_long=None): - """Return the first item from *iterable*, which is expected to contain only - that item. Raise an exception if *iterable* is empty or has more than one - item. - - :func:`one` is useful for ensuring that an iterable contains only one item. - For example, it can be used to retrieve the result of a database query - that is expected to return a single row. - - If *iterable* is empty, ``ValueError`` will be raised. You may specify a - different exception with the *too_short* keyword: - - >>> it = [] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (expected 1)' - >>> too_short = IndexError('too few items') - >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - IndexError: too few items - - Similarly, if *iterable* contains more than one item, ``ValueError`` will - be raised. You may specify a different exception with the *too_long* - keyword: - - >>> it = ['too', 'many'] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 'too', - 'many', and perhaps more. - >>> too_long = RuntimeError - >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - Note that :func:`one` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check iterable - contents less destructively. - - """ - it = iter(iterable) - - try: - first_value = next(it) - except StopIteration as e: - raise ( - too_short or ValueError('too few items in iterable (expected 1)') - ) from e - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def raise_(exception, *args): - raise exception(*args) - - -def strictly_n(iterable, n, too_short=None, too_long=None): - """Validate that *iterable* has exactly *n* items and return them if - it does. If it has fewer than *n* items, call function *too_short* - with those items. If it has more than *n* items, call function - *too_long* with the first ``n + 1`` items. - - >>> iterable = ['a', 'b', 'c', 'd'] - >>> n = 4 - >>> list(strictly_n(iterable, n)) - ['a', 'b', 'c', 'd'] - - Note that the returned iterable must be consumed in order for the check to - be made. - - By default, *too_short* and *too_long* are functions that raise - ``ValueError``. - - >>> list(strictly_n('ab', 3)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too few items in iterable (got 2) - - >>> list(strictly_n('abc', 2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (got at least 3) - - You can instead supply functions that do something else. - *too_short* will be called with the number of items in *iterable*. - *too_long* will be called with `n + 1`. - - >>> def too_short(item_count): - ... raise RuntimeError - >>> it = strictly_n('abcd', 6, too_short=too_short) - >>> list(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - >>> def too_long(item_count): - ... print('The boss is going to hear about this') - >>> it = strictly_n('abcdef', 4, too_long=too_long) - >>> list(it) - The boss is going to hear about this - ['a', 'b', 'c', 'd'] - - """ - if too_short is None: - too_short = lambda item_count: raise_( - ValueError, - 'Too few items in iterable (got {})'.format(item_count), - ) - - if too_long is None: - too_long = lambda item_count: raise_( - ValueError, - 'Too many items in iterable (got at least {})'.format(item_count), - ) - - it = iter(iterable) - for i in range(n): - try: - item = next(it) - except StopIteration: - too_short(i) - return - else: - yield item - - try: - next(it) - except StopIteration: - pass - else: - too_long(n + 1) - - -def distinct_permutations(iterable, r=None): - """Yield successive distinct permutations of the elements in *iterable*. - - >>> sorted(distinct_permutations([1, 0, 1])) - [(0, 1, 1), (1, 0, 1), (1, 1, 0)] - - Equivalent to ``set(permutations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - Duplicate permutations arise when there are duplicated elements in the - input iterable. The number of items returned is - `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of - items input, and each `x_i` is the count of a distinct item in the input - sequence. - - If *r* is given, only the *r*-length permutations are yielded. - - >>> sorted(distinct_permutations([1, 0, 1], r=2)) - [(0, 1), (1, 0), (1, 1)] - >>> sorted(distinct_permutations(range(3), r=2)) - [(0, 1), (0, 2), (1, 0), (1, 2), (2, 0), (2, 1)] - - """ - - # Algorithm: https://w.wiki/Qai - def _full(A): - while True: - # Yield the permutation we have - yield tuple(A) - - # Find the largest index i such that A[i] < A[i + 1] - for i in range(size - 2, -1, -1): - if A[i] < A[i + 1]: - break - # If no such index exists, this permutation is the last one - else: - return - - # Find the largest index j greater than j such that A[i] < A[j] - for j in range(size - 1, i, -1): - if A[i] < A[j]: - break - - # Swap the value of A[i] with that of A[j], then reverse the - # sequence from A[i + 1] to form the new permutation - A[i], A[j] = A[j], A[i] - A[i + 1 :] = A[: i - size : -1] # A[i + 1:][::-1] - - # Algorithm: modified from the above - def _partial(A, r): - # Split A into the first r items and the last r items - head, tail = A[:r], A[r:] - right_head_indexes = range(r - 1, -1, -1) - left_tail_indexes = range(len(tail)) - - while True: - # Yield the permutation we have - yield tuple(head) - - # Starting from the right, find the first index of the head with - # value smaller than the maximum value of the tail - call it i. - pivot = tail[-1] - for i in right_head_indexes: - if head[i] < pivot: - break - pivot = head[i] - else: - return - - # Starting from the left, find the first value of the tail - # with a value greater than head[i] and swap. - for j in left_tail_indexes: - if tail[j] > head[i]: - head[i], tail[j] = tail[j], head[i] - break - # If we didn't find one, start from the right and find the first - # index of the head with a value greater than head[i] and swap. - else: - for j in right_head_indexes: - if head[j] > head[i]: - head[i], head[j] = head[j], head[i] - break - - # Reverse head[i + 1:] and swap it with tail[:r - (i + 1)] - tail += head[: i - r : -1] # head[i + 1:][::-1] - i += 1 - head[i:], tail[:] = tail[: r - i], tail[r - i :] - - items = sorted(iterable) - - size = len(items) - if r is None: - r = size - - if 0 < r <= size: - return _full(items) if (r == size) else _partial(items, r) - - return iter(() if r else ((),)) - - -def intersperse(e, iterable, n=1): - """Intersperse filler element *e* among the items in *iterable*, leaving - *n* items between each filler element. - - >>> list(intersperse('!', [1, 2, 3, 4, 5])) - [1, '!', 2, '!', 3, '!', 4, '!', 5] - - >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) - [1, 2, None, 3, 4, None, 5] - - """ - if n == 0: - raise ValueError('n must be > 0') - elif n == 1: - # interleave(repeat(e), iterable) -> e, x_0, e, x_1, e, x_2... - # islice(..., 1, None) -> x_0, e, x_1, e, x_2... - return islice(interleave(repeat(e), iterable), 1, None) - else: - # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... - # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... - # flatten(...) -> x_0, x_1, e, x_2, x_3... - filler = repeat([e]) - chunks = chunked(iterable, n) - return flatten(islice(interleave(filler, chunks), 1, None)) - - -def unique_to_each(*iterables): - """Return the elements from each of the input iterables that aren't in the - other input iterables. - - For example, suppose you have a set of packages, each with a set of - dependencies:: - - {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} - - If you remove one package, which dependencies can also be removed? - - If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not - associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for - ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: - - >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) - [['A'], ['C'], ['D']] - - If there are duplicates in one input iterable that aren't in the others - they will be duplicated in the output. Input order is preserved:: - - >>> unique_to_each("mississippi", "missouri") - [['p', 'p'], ['o', 'u', 'r']] - - It is assumed that the elements of each iterable are hashable. - - """ - pool = [list(it) for it in iterables] - counts = Counter(chain.from_iterable(map(set, pool))) - uniques = {element for element in counts if counts[element] == 1} - return [list(filter(uniques.__contains__, it)) for it in pool] - - -def windowed(seq, n, fillvalue=None, step=1): - """Return a sliding window of width *n* over the given iterable. - - >>> all_windows = windowed([1, 2, 3, 4, 5], 3) - >>> list(all_windows) - [(1, 2, 3), (2, 3, 4), (3, 4, 5)] - - When the window is larger than the iterable, *fillvalue* is used in place - of missing values: - - >>> list(windowed([1, 2, 3], 4)) - [(1, 2, 3, None)] - - Each window will advance in increments of *step*: - - >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) - [(1, 2, 3), (3, 4, 5), (5, 6, '!')] - - To slide into the iterable's items, use :func:`chain` to add filler items - to the left: - - >>> iterable = [1, 2, 3, 4] - >>> n = 3 - >>> padding = [None] * (n - 1) - >>> list(windowed(chain(padding, iterable), 3)) - [(None, None, 1), (None, 1, 2), (1, 2, 3), (2, 3, 4)] - """ - if n < 0: - raise ValueError('n must be >= 0') - if n == 0: - yield tuple() - return - if step < 1: - raise ValueError('step must be >= 1') - - window = deque(maxlen=n) - i = n - for _ in map(window.append, seq): - i -= 1 - if not i: - i = step - yield tuple(window) - - size = len(window) - if size == 0: - return - elif size < n: - yield tuple(chain(window, repeat(fillvalue, n - size))) - elif 0 < i < min(step, n): - window += (fillvalue,) * i - yield tuple(window) - - -def substrings(iterable): - """Yield all of the substrings of *iterable*. - - >>> [''.join(s) for s in substrings('more')] - ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] - - Note that non-string iterables can also be subdivided. - - >>> list(substrings([0, 1, 2])) - [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] - - """ - # The length-1 substrings - seq = [] - for item in iter(iterable): - seq.append(item) - yield (item,) - seq = tuple(seq) - item_count = len(seq) - - # And the rest - for n in range(2, item_count + 1): - for i in range(item_count - n + 1): - yield seq[i : i + n] - - -def substrings_indexes(seq, reverse=False): - """Yield all substrings and their positions in *seq* - - The items yielded will be a tuple of the form ``(substr, i, j)``, where - ``substr == seq[i:j]``. - - This function only works for iterables that support slicing, such as - ``str`` objects. - - >>> for item in substrings_indexes('more'): - ... print(item) - ('m', 0, 1) - ('o', 1, 2) - ('r', 2, 3) - ('e', 3, 4) - ('mo', 0, 2) - ('or', 1, 3) - ('re', 2, 4) - ('mor', 0, 3) - ('ore', 1, 4) - ('more', 0, 4) - - Set *reverse* to ``True`` to yield the same items in the opposite order. - - - """ - r = range(1, len(seq) + 1) - if reverse: - r = reversed(r) - return ( - (seq[i : i + L], i, i + L) for L in r for i in range(len(seq) - L + 1) - ) - - -class bucket: - """Wrap *iterable* and return an object that buckets the iterable into - child iterables based on a *key* function. - - >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] - >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character - >>> sorted(list(s)) # Get the keys - ['a', 'b', 'c'] - >>> a_iterable = s['a'] - >>> next(a_iterable) - 'a1' - >>> next(a_iterable) - 'a2' - >>> list(s['b']) - ['b1', 'b2', 'b3'] - - The original iterable will be advanced and its items will be cached until - they are used by the child iterables. This may require significant storage. - - By default, attempting to select a bucket to which no items belong will - exhaust the iterable and cache all values. - If you specify a *validator* function, selected buckets will instead be - checked against it. - - >>> from itertools import count - >>> it = count(1, 2) # Infinite sequence of odd numbers - >>> key = lambda x: x % 10 # Bucket by last digit - >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only - >>> s = bucket(it, key=key, validator=validator) - >>> 2 in s - False - >>> list(s[2]) - [] - - """ - - def __init__(self, iterable, key, validator=None): - self._it = iter(iterable) - self._key = key - self._cache = defaultdict(deque) - self._validator = validator or (lambda x: True) - - def __contains__(self, value): - if not self._validator(value): - return False - - try: - item = next(self[value]) - except StopIteration: - return False - else: - self._cache[value].appendleft(item) - - return True - - def _get_values(self, value): - """ - Helper to yield items from the parent iterator that match *value*. - Items that don't match are stored in the local cache as they - are encountered. - """ - while True: - # If we've cached some items that match the target value, emit - # the first one and evict it from the cache. - if self._cache[value]: - yield self._cache[value].popleft() - # Otherwise we need to advance the parent iterator to search for - # a matching item, caching the rest. - else: - while True: - try: - item = next(self._it) - except StopIteration: - return - item_value = self._key(item) - if item_value == value: - yield item - break - elif self._validator(item_value): - self._cache[item_value].append(item) - - def __iter__(self): - for item in self._it: - item_value = self._key(item) - if self._validator(item_value): - self._cache[item_value].append(item) - - yield from self._cache.keys() - - def __getitem__(self, value): - if not self._validator(value): - return iter(()) - - return self._get_values(value) - - -def spy(iterable, n=1): - """Return a 2-tuple with a list containing the first *n* elements of - *iterable*, and an iterator with the same items as *iterable*. - This allows you to "look ahead" at the items in the iterable without - advancing it. - - There is one item in the list by default: - - >>> iterable = 'abcdefg' - >>> head, iterable = spy(iterable) - >>> head - ['a'] - >>> list(iterable) - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - - You may use unpacking to retrieve items instead of lists: - - >>> (head,), iterable = spy('abcdefg') - >>> head - 'a' - >>> (first, second), iterable = spy('abcdefg', 2) - >>> first - 'a' - >>> second - 'b' - - The number of items requested can be larger than the number of items in - the iterable: - - >>> iterable = [1, 2, 3, 4, 5] - >>> head, iterable = spy(iterable, 10) - >>> head - [1, 2, 3, 4, 5] - >>> list(iterable) - [1, 2, 3, 4, 5] - - """ - it = iter(iterable) - head = take(n, it) - - return head.copy(), chain(head, it) - - -def interleave(*iterables): - """Return a new iterable yielding from each iterable in turn, - until the shortest is exhausted. - - >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7] - - For a version that doesn't terminate after the shortest iterable is - exhausted, see :func:`interleave_longest`. - - """ - return chain.from_iterable(zip(*iterables)) - - -def interleave_longest(*iterables): - """Return a new iterable yielding from each iterable in turn, - skipping any that are exhausted. - - >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7, 3, 8] - - This function produces the same output as :func:`roundrobin`, but may - perform better for some inputs (in particular when the number of iterables - is large). - - """ - i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) - return (x for x in i if x is not _marker) - - -def interleave_evenly(iterables, lengths=None): - """ - Interleave multiple iterables so that their elements are evenly distributed - throughout the output sequence. - - >>> iterables = [1, 2, 3, 4, 5], ['a', 'b'] - >>> list(interleave_evenly(iterables)) - [1, 2, 'a', 3, 4, 'b', 5] - - >>> iterables = [[1, 2, 3], [4, 5], [6, 7, 8]] - >>> list(interleave_evenly(iterables)) - [1, 6, 4, 2, 7, 3, 8, 5] - - This function requires iterables of known length. Iterables without - ``__len__()`` can be used by manually specifying lengths with *lengths*: - - >>> from itertools import combinations, repeat - >>> iterables = [combinations(range(4), 2), ['a', 'b', 'c']] - >>> lengths = [4 * (4 - 1) // 2, 3] - >>> list(interleave_evenly(iterables, lengths=lengths)) - [(0, 1), (0, 2), 'a', (0, 3), (1, 2), 'b', (1, 3), (2, 3), 'c'] - - Based on Bresenham's algorithm. - """ - if lengths is None: - try: - lengths = [len(it) for it in iterables] - except TypeError: - raise ValueError( - 'Iterable lengths could not be determined automatically. ' - 'Specify them with the lengths keyword.' - ) - elif len(iterables) != len(lengths): - raise ValueError('Mismatching number of iterables and lengths.') - - dims = len(lengths) - - # sort iterables by length, descending - lengths_permute = sorted( - range(dims), key=lambda i: lengths[i], reverse=True - ) - lengths_desc = [lengths[i] for i in lengths_permute] - iters_desc = [iter(iterables[i]) for i in lengths_permute] - - # the longest iterable is the primary one (Bresenham: the longest - # distance along an axis) - delta_primary, deltas_secondary = lengths_desc[0], lengths_desc[1:] - iter_primary, iters_secondary = iters_desc[0], iters_desc[1:] - errors = [delta_primary // dims] * len(deltas_secondary) - - to_yield = sum(lengths) - while to_yield: - yield next(iter_primary) - to_yield -= 1 - # update errors for each secondary iterable - errors = [e - delta for e, delta in zip(errors, deltas_secondary)] - - # those iterables for which the error is negative are yielded - # ("diagonal step" in Bresenham) - for i, e in enumerate(errors): - if e < 0: - yield next(iters_secondary[i]) - to_yield -= 1 - errors[i] += delta_primary - - -def collapse(iterable, base_type=None, levels=None): - """Flatten an iterable with multiple levels of nesting (e.g., a list of - lists of tuples) into non-iterable types. - - >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] - >>> list(collapse(iterable)) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and - will not be collapsed. - - To avoid collapsing other types, specify *base_type*: - - >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] - >>> list(collapse(iterable, base_type=tuple)) - ['ab', ('cd', 'ef'), 'gh', 'ij'] - - Specify *levels* to stop flattening after a certain level: - - >>> iterable = [('a', ['b']), ('c', ['d'])] - >>> list(collapse(iterable)) # Fully flattened - ['a', 'b', 'c', 'd'] - >>> list(collapse(iterable, levels=1)) # Only one level flattened - ['a', ['b'], 'c', ['d']] - - """ - - def walk(node, level): - if ( - ((levels is not None) and (level > levels)) - or isinstance(node, (str, bytes)) - or ((base_type is not None) and isinstance(node, base_type)) - ): - yield node - return - - try: - tree = iter(node) - except TypeError: - yield node - return - else: - for child in tree: - yield from walk(child, level + 1) - - yield from walk(iterable, 0) - - -def side_effect(func, iterable, chunk_size=None, before=None, after=None): - """Invoke *func* on each item in *iterable* (or on each *chunk_size* group - of items) before yielding the item. - - `func` must be a function that takes a single argument. Its return value - will be discarded. - - *before* and *after* are optional functions that take no arguments. They - will be executed before iteration starts and after it ends, respectively. - - `side_effect` can be used for logging, updating progress bars, or anything - that is not functionally "pure." - - Emitting a status message: - - >>> from more_itertools import consume - >>> func = lambda item: print('Received {}'.format(item)) - >>> consume(side_effect(func, range(2))) - Received 0 - Received 1 - - Operating on chunks of items: - - >>> pair_sums = [] - >>> func = lambda chunk: pair_sums.append(sum(chunk)) - >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) - [0, 1, 2, 3, 4, 5] - >>> list(pair_sums) - [1, 5, 9] - - Writing to a file-like object: - - >>> from io import StringIO - >>> from more_itertools import consume - >>> f = StringIO() - >>> func = lambda x: print(x, file=f) - >>> before = lambda: print(u'HEADER', file=f) - >>> after = f.close - >>> it = [u'a', u'b', u'c'] - >>> consume(side_effect(func, it, before=before, after=after)) - >>> f.closed - True - - """ - try: - if before is not None: - before() - - if chunk_size is None: - for item in iterable: - func(item) - yield item - else: - for chunk in chunked(iterable, chunk_size): - func(chunk) - yield from chunk - finally: - if after is not None: - after() - - -def sliced(seq, n, strict=False): - """Yield slices of length *n* from the sequence *seq*. - - >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) - [(1, 2, 3), (4, 5, 6)] - - By the default, the last yielded slice will have fewer than *n* elements - if the length of *seq* is not divisible by *n*: - - >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) - [(1, 2, 3), (4, 5, 6), (7, 8)] - - If the length of *seq* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - slice is yielded. - - This function will only work for iterables that support slicing. - For non-sliceable iterables, see :func:`chunked`. - - """ - iterator = takewhile(len, (seq[i : i + n] for i in count(0, n))) - if strict: - - def ret(): - for _slice in iterator: - if len(_slice) != n: - raise ValueError("seq is not divisible by n.") - yield _slice - - return iter(ret()) - else: - return iterator - - -def split_at(iterable, pred, maxsplit=-1, keep_separator=False): - """Yield lists of items from *iterable*, where each list is delimited by - an item where callable *pred* returns ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b')) - [['a'], ['c', 'd', 'c'], ['a']] - - >>> list(split_at(range(10), lambda n: n % 2 == 1)) - [[0], [2], [4], [6], [8], []] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_at(range(10), lambda n: n % 2 == 1, maxsplit=2)) - [[0], [2], [4, 5, 6, 7, 8, 9]] - - By default, the delimiting items are not included in the output. - To include them, set *keep_separator* to ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b', keep_separator=True)) - [['a'], ['b'], ['c', 'd', 'c'], ['b'], ['a']] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item): - yield buf - if keep_separator: - yield [item] - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - else: - buf.append(item) - yield buf - - -def split_before(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends just before - an item for which callable *pred* returns ``True``: - - >>> list(split_before('OneTwo', lambda s: s.isupper())) - [['O', 'n', 'e'], ['T', 'w', 'o']] - - >>> list(split_before(range(10), lambda n: n % 3 == 0)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_before(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]] - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield [item] + list(it) - return - buf = [] - maxsplit -= 1 - buf.append(item) - if buf: - yield buf - - -def split_after(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends with an - item where callable *pred* returns ``True``: - - >>> list(split_after('one1two2', lambda s: s.isdigit())) - [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] - - >>> list(split_after(range(10), lambda n: n % 3 == 0)) - [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_after(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0], [1, 2, 3], [4, 5, 6, 7, 8, 9]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - buf.append(item) - if pred(item) and buf: - yield buf - if maxsplit == 1: - buf = list(it) - if buf: - yield buf - return - buf = [] - maxsplit -= 1 - if buf: - yield buf - - -def split_when(iterable, pred, maxsplit=-1): - """Split *iterable* into pieces based on the output of *pred*. - *pred* should be a function that takes successive pairs of items and - returns ``True`` if the iterable should be split in between them. - - For example, to find runs of increasing numbers, split the iterable when - element ``i`` is larger than element ``i + 1``: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], lambda x, y: x > y)) - [[1, 2, 3, 3], [2, 5], [2, 4], [2]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], - ... lambda x, y: x > y, maxsplit=2)) - [[1, 2, 3, 3], [2, 5], [2, 4, 2]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - it = iter(iterable) - try: - cur_item = next(it) - except StopIteration: - return - - buf = [cur_item] - for next_item in it: - if pred(cur_item, next_item): - yield buf - if maxsplit == 1: - yield [next_item] + list(it) - return - buf = [] - maxsplit -= 1 - - buf.append(next_item) - cur_item = next_item - - yield buf - - -def split_into(iterable, sizes): - """Yield a list of sequential items from *iterable* of length 'n' for each - integer 'n' in *sizes*. - - >>> list(split_into([1,2,3,4,5,6], [1,2,3])) - [[1], [2, 3], [4, 5, 6]] - - If the sum of *sizes* is smaller than the length of *iterable*, then the - remaining items of *iterable* will not be returned. - - >>> list(split_into([1,2,3,4,5,6], [2,3])) - [[1, 2], [3, 4, 5]] - - If the sum of *sizes* is larger than the length of *iterable*, fewer items - will be returned in the iteration that overruns *iterable* and further - lists will be empty: - - >>> list(split_into([1,2,3,4], [1,2,3,4])) - [[1], [2, 3], [4], []] - - When a ``None`` object is encountered in *sizes*, the returned list will - contain items up to the end of *iterable* the same way that itertools.slice - does: - - >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) - [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] - - :func:`split_into` can be useful for grouping a series of items where the - sizes of the groups are not uniform. An example would be where in a row - from a table, multiple columns represent elements of the same feature - (e.g. a point represented by x,y,z) but, the format is not the same for - all columns. - """ - # convert the iterable argument into an iterator so its contents can - # be consumed by islice in case it is a generator - it = iter(iterable) - - for size in sizes: - if size is None: - yield list(it) - return - else: - yield list(islice(it, size)) - - -def padded(iterable, fillvalue=None, n=None, next_multiple=False): - """Yield the elements from *iterable*, followed by *fillvalue*, such that - at least *n* items are emitted. - - >>> list(padded([1, 2, 3], '?', 5)) - [1, 2, 3, '?', '?'] - - If *next_multiple* is ``True``, *fillvalue* will be emitted until the - number of items emitted is a multiple of *n*:: - - >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) - [1, 2, 3, 4, None, None] - - If *n* is ``None``, *fillvalue* will be emitted indefinitely. - - """ - it = iter(iterable) - if n is None: - yield from chain(it, repeat(fillvalue)) - elif n < 1: - raise ValueError('n must be at least 1') - else: - item_count = 0 - for item in it: - yield item - item_count += 1 - - remaining = (n - item_count) % n if next_multiple else n - item_count - for _ in range(remaining): - yield fillvalue - - -def repeat_each(iterable, n=2): - """Repeat each element in *iterable* *n* times. - - >>> list(repeat_each('ABC', 3)) - ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C', 'C'] - """ - return chain.from_iterable(map(repeat, iterable, repeat(n))) - - -def repeat_last(iterable, default=None): - """After the *iterable* is exhausted, keep yielding its last element. - - >>> list(islice(repeat_last(range(3)), 5)) - [0, 1, 2, 2, 2] - - If the iterable is empty, yield *default* forever:: - - >>> list(islice(repeat_last(range(0), 42), 5)) - [42, 42, 42, 42, 42] - - """ - item = _marker - for item in iterable: - yield item - final = default if item is _marker else item - yield from repeat(final) - - -def distribute(n, iterable): - """Distribute the items from *iterable* among *n* smaller iterables. - - >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 3, 5] - >>> list(group_2) - [2, 4, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 4, 7], [2, 5], [3, 6]] - - If the length of *iterable* is smaller than *n*, then the last returned - iterables will be empty: - - >>> children = distribute(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function uses :func:`itertools.tee` and may require significant - storage. If you need the order items in the smaller iterables to match the - original iterable, see :func:`divide`. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - children = tee(iterable, n) - return [islice(it, index, None, n) for index, it in enumerate(children)] - - -def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): - """Yield tuples whose elements are offset from *iterable*. - The amount by which the `i`-th item in each tuple is offset is given by - the `i`-th item in *offsets*. - - >>> list(stagger([0, 1, 2, 3])) - [(None, 0, 1), (0, 1, 2), (1, 2, 3)] - >>> list(stagger(range(8), offsets=(0, 2, 4))) - [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] - - By default, the sequence will end when the final element of a tuple is the - last item in the iterable. To continue until the first element of a tuple - is the last item in the iterable, set *longest* to ``True``:: - - >>> list(stagger([0, 1, 2, 3], longest=True)) - [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - children = tee(iterable, len(offsets)) - - return zip_offset( - *children, offsets=offsets, longest=longest, fillvalue=fillvalue - ) - - -def zip_equal(*iterables): - """``zip`` the input *iterables* together, but raise - ``UnequalIterablesError`` if they aren't all the same length. - - >>> it_1 = range(3) - >>> it_2 = iter('abc') - >>> list(zip_equal(it_1, it_2)) - [(0, 'a'), (1, 'b'), (2, 'c')] - - >>> it_1 = range(3) - >>> it_2 = iter('abcd') - >>> list(zip_equal(it_1, it_2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - more_itertools.more.UnequalIterablesError: Iterables have different - lengths - - """ - if hexversion >= 0x30A00A6: - warnings.warn( - ( - 'zip_equal will be removed in a future version of ' - 'more-itertools. Use the builtin zip function with ' - 'strict=True instead.' - ), - DeprecationWarning, - ) - - return _zip_equal(*iterables) - - -def zip_offset(*iterables, offsets, longest=False, fillvalue=None): - """``zip`` the input *iterables* together, but offset the `i`-th iterable - by the `i`-th item in *offsets*. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] - - This can be used as a lightweight alternative to SciPy or pandas to analyze - data sets in which some series have a lead or lag relationship. - - By default, the sequence will end when the shortest iterable is exhausted. - To continue until the longest iterable is exhausted, set *longest* to - ``True``. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - if len(iterables) != len(offsets): - raise ValueError("Number of iterables and offsets didn't match") - - staggered = [] - for it, n in zip(iterables, offsets): - if n < 0: - staggered.append(chain(repeat(fillvalue, -n), it)) - elif n > 0: - staggered.append(islice(it, n, None)) - else: - staggered.append(it) - - if longest: - return zip_longest(*staggered, fillvalue=fillvalue) - - return zip(*staggered) - - -def sort_together(iterables, key_list=(0,), key=None, reverse=False): - """Return the input iterables sorted together, with *key_list* as the - priority for sorting. All iterables are trimmed to the length of the - shortest one. - - This can be used like the sorting function in a spreadsheet. If each - iterable represents a column of data, the key list determines which - columns are used for sorting. - - By default, all iterables are sorted using the ``0``-th iterable:: - - >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] - >>> sort_together(iterables) - [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] - - Set a different key list to sort according to another iterable. - Specifying multiple keys dictates how ties are broken:: - - >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] - >>> sort_together(iterables, key_list=(1, 2)) - [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] - - To sort by a function of the elements of the iterable, pass a *key* - function. Its arguments are the elements of the iterables corresponding to - the key list:: - - >>> names = ('a', 'b', 'c') - >>> lengths = (1, 2, 3) - >>> widths = (5, 2, 1) - >>> def area(length, width): - ... return length * width - >>> sort_together([names, lengths, widths], key_list=(1, 2), key=area) - [('c', 'b', 'a'), (3, 2, 1), (1, 2, 5)] - - Set *reverse* to ``True`` to sort in descending order. - - >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) - [(3, 2, 1), ('a', 'b', 'c')] - - """ - if key is None: - # if there is no key function, the key argument to sorted is an - # itemgetter - key_argument = itemgetter(*key_list) - else: - # if there is a key function, call it with the items at the offsets - # specified by the key function as arguments - key_list = list(key_list) - if len(key_list) == 1: - # if key_list contains a single item, pass the item at that offset - # as the only argument to the key function - key_offset = key_list[0] - key_argument = lambda zipped_items: key(zipped_items[key_offset]) - else: - # if key_list contains multiple items, use itemgetter to return a - # tuple of items, which we pass as *args to the key function - get_key_items = itemgetter(*key_list) - key_argument = lambda zipped_items: key( - *get_key_items(zipped_items) - ) - - return list( - zip(*sorted(zip(*iterables), key=key_argument, reverse=reverse)) - ) - - -def unzip(iterable): - """The inverse of :func:`zip`, this function disaggregates the elements - of the zipped *iterable*. - - The ``i``-th iterable contains the ``i``-th element from each element - of the zipped iterable. The first element is used to determine the - length of the remaining elements. - - >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> letters, numbers = unzip(iterable) - >>> list(letters) - ['a', 'b', 'c', 'd'] - >>> list(numbers) - [1, 2, 3, 4] - - This is similar to using ``zip(*iterable)``, but it avoids reading - *iterable* into memory. Note, however, that this function uses - :func:`itertools.tee` and thus may require significant storage. - - """ - head, iterable = spy(iter(iterable)) - if not head: - # empty iterable, e.g. zip([], [], []) - return () - # spy returns a one-length iterable as head - head = head[0] - iterables = tee(iterable, len(head)) - - def itemgetter(i): - def getter(obj): - try: - return obj[i] - except IndexError: - # basically if we have an iterable like - # iter([(1, 2, 3), (4, 5), (6,)]) - # the second unzipped iterable would fail at the third tuple - # since it would try to access tup[1] - # same with the third unzipped iterable and the second tuple - # to support these "improperly zipped" iterables, - # we create a custom itemgetter - # which just stops the unzipped iterables - # at first length mismatch - raise StopIteration - - return getter - - return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) - - -def divide(n, iterable): - """Divide the elements from *iterable* into *n* parts, maintaining - order. - - >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 2, 3] - >>> list(group_2) - [4, 5, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 2, 3], [4, 5], [6, 7]] - - If the length of the iterable is smaller than n, then the last returned - iterables will be empty: - - >>> children = divide(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function will exhaust the iterable before returning and may require - significant storage. If order is not important, see :func:`distribute`, - which does not first pull the iterable into memory. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - try: - iterable[:0] - except TypeError: - seq = tuple(iterable) - else: - seq = iterable - - q, r = divmod(len(seq), n) - - ret = [] - stop = 0 - for i in range(1, n + 1): - start = stop - stop += q + 1 if i <= r else q - ret.append(iter(seq[start:stop])) - - return ret - - -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) - - -def adjacent(predicate, iterable, distance=1): - """Return an iterable over `(bool, item)` tuples where the `item` is - drawn from *iterable* and the `bool` indicates whether - that item satisfies the *predicate* or is adjacent to an item that does. - - For example, to find whether items are adjacent to a ``3``:: - - >>> list(adjacent(lambda x: x == 3, range(6))) - [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] - - Set *distance* to change what counts as adjacent. For example, to find - whether items are two places away from a ``3``: - - >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) - [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] - - This is useful for contextualizing the results of a search function. - For example, a code comparison tool might want to identify lines that - have changed, but also surrounding lines to give the viewer of the diff - context. - - The predicate function will only be called once for each item in the - iterable. - - See also :func:`groupby_transform`, which can be used with this function - to group ranges of items with the same `bool` value. - - """ - # Allow distance=0 mainly for testing that it reproduces results with map() - if distance < 0: - raise ValueError('distance must be at least 0') - - i1, i2 = tee(iterable) - padding = [False] * distance - selected = chain(padding, map(predicate, i1), padding) - adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) - return zip(adjacent_to_selected, i2) - - -def groupby_transform(iterable, keyfunc=None, valuefunc=None, reducefunc=None): - """An extension of :func:`itertools.groupby` that can apply transformations - to the grouped data. - - * *keyfunc* is a function computing a key value for each item in *iterable* - * *valuefunc* is a function that transforms the individual items from - *iterable* after grouping - * *reducefunc* is a function that transforms each group of items - - >>> iterable = 'aAAbBBcCC' - >>> keyfunc = lambda k: k.upper() - >>> valuefunc = lambda v: v.lower() - >>> reducefunc = lambda g: ''.join(g) - >>> list(groupby_transform(iterable, keyfunc, valuefunc, reducefunc)) - [('A', 'aaa'), ('B', 'bbb'), ('C', 'ccc')] - - Each optional argument defaults to an identity function if not specified. - - :func:`groupby_transform` is useful when grouping elements of an iterable - using a separate iterable as the key. To do this, :func:`zip` the iterables - and pass a *keyfunc* that extracts the first element and a *valuefunc* - that extracts the second element:: - - >>> from operator import itemgetter - >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] - >>> values = 'abcdefghi' - >>> iterable = zip(keys, values) - >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) - >>> [(k, ''.join(g)) for k, g in grouper] - [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] - - Note that the order of items in the iterable is significant. - Only adjacent items are grouped together, so if you don't want any - duplicate groups, you should sort the iterable by the key function. - - """ - ret = groupby(iterable, keyfunc) - if valuefunc: - ret = ((k, map(valuefunc, g)) for k, g in ret) - if reducefunc: - ret = ((k, reducefunc(g)) for k, g in ret) - - return ret - - -class numeric_range(abc.Sequence, abc.Hashable): - """An extension of the built-in ``range()`` function whose arguments can - be any orderable numeric type. - - With only *stop* specified, *start* defaults to ``0`` and *step* - defaults to ``1``. The output items will match the type of *stop*: - - >>> list(numeric_range(3.5)) - [0.0, 1.0, 2.0, 3.0] - - With only *start* and *stop* specified, *step* defaults to ``1``. The - output items will match the type of *start*: - - >>> from decimal import Decimal - >>> start = Decimal('2.1') - >>> stop = Decimal('5.1') - >>> list(numeric_range(start, stop)) - [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] - - With *start*, *stop*, and *step* specified the output items will match - the type of ``start + step``: - - >>> from fractions import Fraction - >>> start = Fraction(1, 2) # Start at 1/2 - >>> stop = Fraction(5, 2) # End at 5/2 - >>> step = Fraction(1, 2) # Count by 1/2 - >>> list(numeric_range(start, stop, step)) - [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] - - If *step* is zero, ``ValueError`` is raised. Negative steps are supported: - - >>> list(numeric_range(3, -1, -1.0)) - [3.0, 2.0, 1.0, 0.0] - - Be aware of the limitations of floating point numbers; the representation - of the yielded numbers may be surprising. - - ``datetime.datetime`` objects can be used for *start* and *stop*, if *step* - is a ``datetime.timedelta`` object: - - >>> import datetime - >>> start = datetime.datetime(2019, 1, 1) - >>> stop = datetime.datetime(2019, 1, 3) - >>> step = datetime.timedelta(days=1) - >>> items = iter(numeric_range(start, stop, step)) - >>> next(items) - datetime.datetime(2019, 1, 1, 0, 0) - >>> next(items) - datetime.datetime(2019, 1, 2, 0, 0) - - """ - - _EMPTY_HASH = hash(range(0, 0)) - - def __init__(self, *args): - argc = len(args) - if argc == 1: - (self._stop,) = args - self._start = type(self._stop)(0) - self._step = type(self._stop - self._start)(1) - elif argc == 2: - self._start, self._stop = args - self._step = type(self._stop - self._start)(1) - elif argc == 3: - self._start, self._stop, self._step = args - elif argc == 0: - raise TypeError( - 'numeric_range expected at least ' - '1 argument, got {}'.format(argc) - ) - else: - raise TypeError( - 'numeric_range expected at most ' - '3 arguments, got {}'.format(argc) - ) - - self._zero = type(self._step)(0) - if self._step == self._zero: - raise ValueError('numeric_range() arg 3 must not be zero') - self._growing = self._step > self._zero - - def __bool__(self): - if self._growing: - return self._start < self._stop - else: - return self._start > self._stop - - def __contains__(self, elem): - if self._growing: - if self._start <= elem < self._stop: - return (elem - self._start) % self._step == self._zero - else: - if self._start >= elem > self._stop: - return (self._start - elem) % (-self._step) == self._zero - - return False - - def __eq__(self, other): - if isinstance(other, numeric_range): - empty_self = not bool(self) - empty_other = not bool(other) - if empty_self or empty_other: - return empty_self and empty_other # True if both empty - else: - return ( - self._start == other._start - and self._step == other._step - and self._get_by_index(-1) == other._get_by_index(-1) - ) - else: - return False - - def __getitem__(self, key): - if isinstance(key, int): - return self._get_by_index(key) - elif isinstance(key, slice): - step = self._step if key.step is None else key.step * self._step - - if key.start is None or key.start <= -self._len: - start = self._start - elif key.start >= self._len: - start = self._stop - else: # -self._len < key.start < self._len - start = self._get_by_index(key.start) - - if key.stop is None or key.stop >= self._len: - stop = self._stop - elif key.stop <= -self._len: - stop = self._start - else: # -self._len < key.stop < self._len - stop = self._get_by_index(key.stop) - - return numeric_range(start, stop, step) - else: - raise TypeError( - 'numeric range indices must be ' - 'integers or slices, not {}'.format(type(key).__name__) - ) - - def __hash__(self): - if self: - return hash((self._start, self._get_by_index(-1), self._step)) - else: - return self._EMPTY_HASH - - def __iter__(self): - values = (self._start + (n * self._step) for n in count()) - if self._growing: - return takewhile(partial(gt, self._stop), values) - else: - return takewhile(partial(lt, self._stop), values) - - def __len__(self): - return self._len - - @cached_property - def _len(self): - if self._growing: - start = self._start - stop = self._stop - step = self._step - else: - start = self._stop - stop = self._start - step = -self._step - distance = stop - start - if distance <= self._zero: - return 0 - else: # distance > 0 and step > 0: regular euclidean division - q, r = divmod(distance, step) - return int(q) + int(r != self._zero) - - def __reduce__(self): - return numeric_range, (self._start, self._stop, self._step) - - def __repr__(self): - if self._step == 1: - return "numeric_range({}, {})".format( - repr(self._start), repr(self._stop) - ) - else: - return "numeric_range({}, {}, {})".format( - repr(self._start), repr(self._stop), repr(self._step) - ) - - def __reversed__(self): - return iter( - numeric_range( - self._get_by_index(-1), self._start - self._step, -self._step - ) - ) - - def count(self, value): - return int(value in self) - - def index(self, value): - if self._growing: - if self._start <= value < self._stop: - q, r = divmod(value - self._start, self._step) - if r == self._zero: - return int(q) - else: - if self._start >= value > self._stop: - q, r = divmod(self._start - value, -self._step) - if r == self._zero: - return int(q) - - raise ValueError("{} is not in numeric range".format(value)) - - def _get_by_index(self, i): - if i < 0: - i += self._len - if i < 0 or i >= self._len: - raise IndexError("numeric range object index out of range") - return self._start + i * self._step - - -def count_cycle(iterable, n=None): - """Cycle through the items from *iterable* up to *n* times, yielding - the number of completed cycles along with each item. If *n* is omitted the - process repeats indefinitely. - - >>> list(count_cycle('AB', 3)) - [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] - - """ - iterable = tuple(iterable) - if not iterable: - return iter(()) - counter = count() if n is None else range(n) - return ((i, item) for i in counter for item in iterable) - - -def mark_ends(iterable): - """Yield 3-tuples of the form ``(is_first, is_last, item)``. - - >>> list(mark_ends('ABC')) - [(True, False, 'A'), (False, False, 'B'), (False, True, 'C')] - - Use this when looping over an iterable to take special action on its first - and/or last items: - - >>> iterable = ['Header', 100, 200, 'Footer'] - >>> total = 0 - >>> for is_first, is_last, item in mark_ends(iterable): - ... if is_first: - ... continue # Skip the header - ... if is_last: - ... continue # Skip the footer - ... total += item - >>> print(total) - 300 - """ - it = iter(iterable) - - try: - b = next(it) - except StopIteration: - return - - try: - for i in count(): - a = b - b = next(it) - yield i == 0, False, a - - except StopIteration: - yield i == 0, True, a - - -def locate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(locate([0, 1, 1, 0, 1, 0, 0])) - [1, 2, 4] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item. - - >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) - [1, 3] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(locate(iterable, pred=pred, window_size=3)) - [1, 5, 9] - - Use with :func:`seekable` to find indexes and then retrieve the associated - items: - - >>> from itertools import count - >>> from more_itertools import seekable - >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) - >>> it = seekable(source) - >>> pred = lambda x: x > 100 - >>> indexes = locate(it, pred=pred) - >>> i = next(indexes) - >>> it.seek(i) - >>> next(it) - 106 - - """ - if window_size is None: - return compress(count(), map(pred, iterable)) - - if window_size < 1: - raise ValueError('window size must be at least 1') - - it = windowed(iterable, window_size, fillvalue=_marker) - return compress(count(), starmap(pred, it)) - - -def longest_common_prefix(iterables): - """Yield elements of the longest common prefix amongst given *iterables*. - - >>> ''.join(longest_common_prefix(['abcd', 'abc', 'abf'])) - 'ab' - - """ - return (c[0] for c in takewhile(all_equal, zip(*iterables))) - - -def lstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the beginning - for which *pred* returns ``True``. - - For example, to remove a set of items from the start of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(lstrip(iterable, pred)) - [1, 2, None, 3, False, None] - - This function is analogous to to :func:`str.lstrip`, and is essentially - an wrapper for :func:`itertools.dropwhile`. - - """ - return dropwhile(pred, iterable) - - -def rstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the end - for which *pred* returns ``True``. - - For example, to remove a set of items from the end of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(rstrip(iterable, pred)) - [None, False, None, 1, 2, None, 3] - - This function is analogous to :func:`str.rstrip`. - - """ - cache = [] - cache_append = cache.append - cache_clear = cache.clear - for x in iterable: - if pred(x): - cache_append(x) - else: - yield from cache - cache_clear() - yield x - - -def strip(iterable, pred): - """Yield the items from *iterable*, but strip any from the - beginning and end for which *pred* returns ``True``. - - For example, to remove a set of items from both ends of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(strip(iterable, pred)) - [1, 2, None, 3] - - This function is analogous to :func:`str.strip`. - - """ - return rstrip(lstrip(iterable, pred), pred) - - -class islice_extended: - """An extension of :func:`itertools.islice` that supports negative values - for *stop*, *start*, and *step*. - - >>> iterable = iter('abcdefgh') - >>> list(islice_extended(iterable, -4, -1)) - ['e', 'f', 'g'] - - Slices with negative values require some caching of *iterable*, but this - function takes care to minimize the amount of memory required. - - For example, you can use a negative step with an infinite iterator: - - >>> from itertools import count - >>> list(islice_extended(count(), 110, 99, -2)) - [110, 108, 106, 104, 102, 100] - - You can also use slice notation directly: - - >>> iterable = map(str, count()) - >>> it = islice_extended(iterable)[10:20:2] - >>> list(it) - ['10', '12', '14', '16', '18'] - - """ - - def __init__(self, iterable, *args): - it = iter(iterable) - if args: - self._iterable = _islice_helper(it, slice(*args)) - else: - self._iterable = it - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterable) - - def __getitem__(self, key): - if isinstance(key, slice): - return islice_extended(_islice_helper(self._iterable, key)) - - raise TypeError('islice_extended.__getitem__ argument must be a slice') - - -def _islice_helper(it, s): - start = s.start - stop = s.stop - if s.step == 0: - raise ValueError('step argument must be a non-zero integer or None.') - step = s.step or 1 - - if step > 0: - start = 0 if (start is None) else start - - if start < 0: - # Consume all but the last -start items - cache = deque(enumerate(it, 1), maxlen=-start) - len_iter = cache[-1][0] if cache else 0 - - # Adjust start to be positive - i = max(len_iter + start, 0) - - # Adjust stop to be positive - if stop is None: - j = len_iter - elif stop >= 0: - j = min(stop, len_iter) - else: - j = max(len_iter + stop, 0) - - # Slice the cache - n = j - i - if n <= 0: - return - - for index, item in islice(cache, 0, n, step): - yield item - elif (stop is not None) and (stop < 0): - # Advance to the start position - next(islice(it, start, start), None) - - # When stop is negative, we have to carry -stop items while - # iterating - cache = deque(islice(it, -stop), maxlen=-stop) - - for index, item in enumerate(it): - cached_item = cache.popleft() - if index % step == 0: - yield cached_item - cache.append(item) - else: - # When both start and stop are positive we have the normal case - yield from islice(it, start, stop, step) - else: - start = -1 if (start is None) else start - - if (stop is not None) and (stop < 0): - # Consume all but the last items - n = -stop - 1 - cache = deque(enumerate(it, 1), maxlen=n) - len_iter = cache[-1][0] if cache else 0 - - # If start and stop are both negative they are comparable and - # we can just slice. Otherwise we can adjust start to be negative - # and then slice. - if start < 0: - i, j = start, stop - else: - i, j = min(start - len_iter, -1), None - - for index, item in list(cache)[i:j:step]: - yield item - else: - # Advance to the stop position - if stop is not None: - m = stop + 1 - next(islice(it, m, m), None) - - # stop is positive, so if start is negative they are not comparable - # and we need the rest of the items. - if start < 0: - i = start - n = None - # stop is None and start is positive, so we just need items up to - # the start index. - elif stop is None: - i = None - n = start + 1 - # Both stop and start are positive, so they are comparable. - else: - i = None - n = start - stop - if n <= 0: - return - - cache = list(islice(it, n)) - - yield from cache[i::step] - - -def always_reversible(iterable): - """An extension of :func:`reversed` that supports all iterables, not - just those which implement the ``Reversible`` or ``Sequence`` protocols. - - >>> print(*always_reversible(x for x in range(3))) - 2 1 0 - - If the iterable is already reversible, this function returns the - result of :func:`reversed()`. If the iterable is not reversible, - this function will cache the remaining items in the iterable and - yield them in reverse order, which may require significant storage. - """ - try: - return reversed(iterable) - except TypeError: - return reversed(list(iterable)) - - -def consecutive_groups(iterable, ordering=lambda x: x): - """Yield groups of consecutive items using :func:`itertools.groupby`. - The *ordering* function determines whether two items are adjacent by - returning their position. - - By default, the ordering function is the identity function. This is - suitable for finding runs of numbers: - - >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] - >>> for group in consecutive_groups(iterable): - ... print(list(group)) - [1] - [10, 11, 12] - [20] - [30, 31, 32, 33] - [40] - - For finding runs of adjacent letters, try using the :meth:`index` method - of a string of letters: - - >>> from string import ascii_lowercase - >>> iterable = 'abcdfgilmnop' - >>> ordering = ascii_lowercase.index - >>> for group in consecutive_groups(iterable, ordering): - ... print(list(group)) - ['a', 'b', 'c', 'd'] - ['f', 'g'] - ['i'] - ['l', 'm', 'n', 'o', 'p'] - - Each group of consecutive items is an iterator that shares it source with - *iterable*. When an an output group is advanced, the previous group is - no longer available unless its elements are copied (e.g., into a ``list``). - - >>> iterable = [1, 2, 11, 12, 21, 22] - >>> saved_groups = [] - >>> for group in consecutive_groups(iterable): - ... saved_groups.append(list(group)) # Copy group elements - >>> saved_groups - [[1, 2], [11, 12], [21, 22]] - - """ - for k, g in groupby( - enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) - ): - yield map(itemgetter(1), g) - - -def difference(iterable, func=sub, *, initial=None): - """This function is the inverse of :func:`itertools.accumulate`. By default - it will compute the first difference of *iterable* using - :func:`operator.sub`: - - >>> from itertools import accumulate - >>> iterable = accumulate([0, 1, 2, 3, 4]) # produces 0, 1, 3, 6, 10 - >>> list(difference(iterable)) - [0, 1, 2, 3, 4] - - *func* defaults to :func:`operator.sub`, but other functions can be - specified. They will be applied as follows:: - - A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... - - For example, to do progressive division: - - >>> iterable = [1, 2, 6, 24, 120] - >>> func = lambda x, y: x // y - >>> list(difference(iterable, func)) - [1, 2, 3, 4, 5] - - If the *initial* keyword is set, the first element will be skipped when - computing successive differences. - - >>> it = [10, 11, 13, 16] # from accumulate([1, 2, 3], initial=10) - >>> list(difference(it, initial=10)) - [1, 2, 3] - - """ - a, b = tee(iterable) - try: - first = [next(b)] - except StopIteration: - return iter([]) - - if initial is not None: - first = [] - - return chain(first, map(func, b, a)) - - -class SequenceView(Sequence): - """Return a read-only view of the sequence object *target*. - - :class:`SequenceView` objects are analogous to Python's built-in - "dictionary view" types. They provide a dynamic view of a sequence's items, - meaning that when the sequence updates, so does the view. - - >>> seq = ['0', '1', '2'] - >>> view = SequenceView(seq) - >>> view - SequenceView(['0', '1', '2']) - >>> seq.append('3') - >>> view - SequenceView(['0', '1', '2', '3']) - - Sequence views support indexing, slicing, and length queries. They act - like the underlying sequence, except they don't allow assignment: - - >>> view[1] - '1' - >>> view[1:-1] - ['1', '2'] - >>> len(view) - 4 - - Sequence views are useful as an alternative to copying, as they don't - require (much) extra storage. - - """ - - def __init__(self, target): - if not isinstance(target, Sequence): - raise TypeError - self._target = target - - def __getitem__(self, index): - return self._target[index] - - def __len__(self): - return len(self._target) - - def __repr__(self): - return '{}({})'.format(self.__class__.__name__, repr(self._target)) - - -class seekable: - """Wrap an iterator to allow for seeking backward and forward. This - progressively caches the items in the source iterable so they can be - re-visited. - - Call :meth:`seek` with an index to seek to that position in the source - iterable. - - To "reset" an iterator, seek to ``0``: - - >>> from itertools import count - >>> it = seekable((str(n) for n in count())) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> it.seek(0) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> next(it) - '3' - - You can also seek forward: - - >>> it = seekable((str(n) for n in range(20))) - >>> it.seek(10) - >>> next(it) - '10' - >>> it.relative_seek(-2) # Seeking relative to the current position - >>> next(it) - '9' - >>> it.seek(20) # Seeking past the end of the source isn't a problem - >>> list(it) - [] - >>> it.seek(0) # Resetting works even after hitting the end - >>> next(it), next(it), next(it) - ('0', '1', '2') - - Call :meth:`peek` to look ahead one item without advancing the iterator: - - >>> it = seekable('1234') - >>> it.peek() - '1' - >>> list(it) - ['1', '2', '3', '4'] - >>> it.peek(default='empty') - 'empty' - - Before the iterator is at its end, calling :func:`bool` on it will return - ``True``. After it will return ``False``: - - >>> it = seekable('5678') - >>> bool(it) - True - >>> list(it) - ['5', '6', '7', '8'] - >>> bool(it) - False - - You may view the contents of the cache with the :meth:`elements` method. - That returns a :class:`SequenceView`, a view that updates automatically: - - >>> it = seekable((str(n) for n in range(10))) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> elements = it.elements() - >>> elements - SequenceView(['0', '1', '2']) - >>> next(it) - '3' - >>> elements - SequenceView(['0', '1', '2', '3']) - - By default, the cache grows as the source iterable progresses, so beware of - wrapping very large or infinite iterables. Supply *maxlen* to limit the - size of the cache (this of course limits how far back you can seek). - - >>> from itertools import count - >>> it = seekable((str(n) for n in count()), maxlen=2) - >>> next(it), next(it), next(it), next(it) - ('0', '1', '2', '3') - >>> list(it.elements()) - ['2', '3'] - >>> it.seek(0) - >>> next(it), next(it), next(it), next(it) - ('2', '3', '4', '5') - >>> next(it) - '6' - - """ - - def __init__(self, iterable, maxlen=None): - self._source = iter(iterable) - if maxlen is None: - self._cache = [] - else: - self._cache = deque([], maxlen) - self._index = None - - def __iter__(self): - return self - - def __next__(self): - if self._index is not None: - try: - item = self._cache[self._index] - except IndexError: - self._index = None - else: - self._index += 1 - return item - - item = next(self._source) - self._cache.append(item) - return item - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - try: - peeked = next(self) - except StopIteration: - if default is _marker: - raise - return default - if self._index is None: - self._index = len(self._cache) - self._index -= 1 - return peeked - - def elements(self): - return SequenceView(self._cache) - - def seek(self, index): - self._index = index - remainder = index - len(self._cache) - if remainder > 0: - consume(self, remainder) - - def relative_seek(self, count): - index = len(self._cache) - self.seek(max(index + count, 0)) - - -class run_length: - """ - :func:`run_length.encode` compresses an iterable with run-length encoding. - It yields groups of repeated items with the count of how many times they - were repeated: - - >>> uncompressed = 'abbcccdddd' - >>> list(run_length.encode(uncompressed)) - [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - - :func:`run_length.decode` decompresses an iterable that was previously - compressed with run-length encoding. It yields the items of the - decompressed iterable: - - >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> list(run_length.decode(compressed)) - ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] - - """ - - @staticmethod - def encode(iterable): - return ((k, ilen(g)) for k, g in groupby(iterable)) - - @staticmethod - def decode(iterable): - return chain.from_iterable(repeat(k, n) for k, n in iterable) - - -def exactly_n(iterable, n, predicate=bool): - """Return ``True`` if exactly ``n`` items in the iterable are ``True`` - according to the *predicate* function. - - >>> exactly_n([True, True, False], 2) - True - >>> exactly_n([True, True, False], 1) - False - >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) - True - - The iterable will be advanced until ``n + 1`` truthy items are encountered, - so avoid calling it on infinite iterables. - - """ - return len(take(n + 1, filter(predicate, iterable))) == n - - -def circular_shifts(iterable): - """Return a list of circular shifts of *iterable*. - - >>> circular_shifts(range(4)) - [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] - """ - lst = list(iterable) - return take(len(lst), windowed(cycle(lst), len(lst))) - - -def make_decorator(wrapping_func, result_index=0): - """Return a decorator version of *wrapping_func*, which is a function that - modifies an iterable. *result_index* is the position in that function's - signature where the iterable goes. - - This lets you use itertools on the "production end," i.e. at function - definition. This can augment what the function returns without changing the - function's code. - - For example, to produce a decorator version of :func:`chunked`: - - >>> from more_itertools import chunked - >>> chunker = make_decorator(chunked, result_index=0) - >>> @chunker(3) - ... def iter_range(n): - ... return iter(range(n)) - ... - >>> list(iter_range(9)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8]] - - To only allow truthy items to be returned: - - >>> truth_serum = make_decorator(filter, result_index=1) - >>> @truth_serum(bool) - ... def boolean_test(): - ... return [0, 1, '', ' ', False, True] - ... - >>> list(boolean_test()) - [1, ' ', True] - - The :func:`peekable` and :func:`seekable` wrappers make for practical - decorators: - - >>> from more_itertools import peekable - >>> peekable_function = make_decorator(peekable) - >>> @peekable_function() - ... def str_range(*args): - ... return (str(x) for x in range(*args)) - ... - >>> it = str_range(1, 20, 2) - >>> next(it), next(it), next(it) - ('1', '3', '5') - >>> it.peek() - '7' - >>> next(it) - '7' - - """ - - # See https://sites.google.com/site/bbayles/index/decorator_factory for - # notes on how this works. - def decorator(*wrapping_args, **wrapping_kwargs): - def outer_wrapper(f): - def inner_wrapper(*args, **kwargs): - result = f(*args, **kwargs) - wrapping_args_ = list(wrapping_args) - wrapping_args_.insert(result_index, result) - return wrapping_func(*wrapping_args_, **wrapping_kwargs) - - return inner_wrapper - - return outer_wrapper - - return decorator - - -def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): - """Return a dictionary that maps the items in *iterable* to categories - defined by *keyfunc*, transforms them with *valuefunc*, and - then summarizes them by category with *reducefunc*. - - *valuefunc* defaults to the identity function if it is unspecified. - If *reducefunc* is unspecified, no summarization takes place: - - >>> keyfunc = lambda x: x.upper() - >>> result = map_reduce('abbccc', keyfunc) - >>> sorted(result.items()) - [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] - - Specifying *valuefunc* transforms the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> result = map_reduce('abbccc', keyfunc, valuefunc) - >>> sorted(result.items()) - [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] - - Specifying *reducefunc* summarizes the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> reducefunc = sum - >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) - >>> sorted(result.items()) - [('A', 1), ('B', 2), ('C', 3)] - - You may want to filter the input iterable before applying the map/reduce - procedure: - - >>> all_items = range(30) - >>> items = [x for x in all_items if 10 <= x <= 20] # Filter - >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 - >>> categories = map_reduce(items, keyfunc=keyfunc) - >>> sorted(categories.items()) - [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] - >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) - >>> sorted(summaries.items()) - [(0, 90), (1, 75)] - - Note that all items in the iterable are gathered into a list before the - summarization step, which may require significant storage. - - The returned object is a :obj:`collections.defaultdict` with the - ``default_factory`` set to ``None``, such that it behaves like a normal - dictionary. - - """ - valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc - - ret = defaultdict(list) - for item in iterable: - key = keyfunc(item) - value = valuefunc(item) - ret[key].append(value) - - if reducefunc is not None: - for key, value_list in ret.items(): - ret[key] = reducefunc(value_list) - - ret.default_factory = None - return ret - - -def rlocate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``, starting from the right and moving left. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 - [4, 2, 1] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item: - - >>> iterable = iter('abcb') - >>> pred = lambda x: x == 'b' - >>> list(rlocate(iterable, pred)) - [3, 1] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(rlocate(iterable, pred=pred, window_size=3)) - [9, 5, 1] - - Beware, this function won't return anything for infinite iterables. - If *iterable* is reversible, ``rlocate`` will reverse it and search from - the right. Otherwise, it will search from the left and return the results - in reverse order. - - See :func:`locate` to for other example applications. - - """ - if window_size is None: - try: - len_iter = len(iterable) - return (len_iter - i - 1 for i in locate(reversed(iterable), pred)) - except TypeError: - pass - - return reversed(list(locate(iterable, pred, window_size))) - - -def replace(iterable, pred, substitutes, count=None, window_size=1): - """Yield the items from *iterable*, replacing the items for which *pred* - returns ``True`` with the items from the iterable *substitutes*. - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] - >>> pred = lambda x: x == 0 - >>> substitutes = (2, 3) - >>> list(replace(iterable, pred, substitutes)) - [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] - - If *count* is given, the number of replacements will be limited: - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] - >>> pred = lambda x: x == 0 - >>> substitutes = [None] - >>> list(replace(iterable, pred, substitutes, count=2)) - [1, 1, None, 1, 1, None, 1, 1, 0] - - Use *window_size* to control the number of items passed as arguments to - *pred*. This allows for locating and replacing subsequences. - - >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] - >>> window_size = 3 - >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred - >>> substitutes = [3, 4] # Splice in these items - >>> list(replace(iterable, pred, substitutes, window_size=window_size)) - [3, 4, 5, 3, 4, 5] - - """ - if window_size < 1: - raise ValueError('window_size must be at least 1') - - # Save the substitutes iterable, since it's used more than once - substitutes = tuple(substitutes) - - # Add padding such that the number of windows matches the length of the - # iterable - it = chain(iterable, [_marker] * (window_size - 1)) - windows = windowed(it, window_size) - - n = 0 - for w in windows: - # If the current window matches our predicate (and we haven't hit - # our maximum number of replacements), splice in the substitutes - # and then consume the following windows that overlap with this one. - # For example, if the iterable is (0, 1, 2, 3, 4...) - # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... - # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) - if pred(*w): - if (count is None) or (n < count): - n += 1 - yield from substitutes - consume(windows, window_size - 1) - continue - - # If there was no match (or we've reached the replacement limit), - # yield the first item from the window. - if w and (w[0] is not _marker): - yield w[0] - - -def partitions(iterable): - """Yield all possible order-preserving partitions of *iterable*. - - >>> iterable = 'abc' - >>> for part in partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['a', 'b', 'c'] - - This is unrelated to :func:`partition`. - - """ - sequence = list(iterable) - n = len(sequence) - for i in powerset(range(1, n)): - yield [sequence[i:j] for i, j in zip((0,) + i, i + (n,))] - - -def set_partitions(iterable, k=None): - """ - Yield the set partitions of *iterable* into *k* parts. Set partitions are - not order-preserving. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable, 2): - ... print([''.join(p) for p in part]) - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - - - If *k* is not given, every set partition is generated. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - ['a', 'b', 'c'] - - """ - L = list(iterable) - n = len(L) - if k is not None: - if k < 1: - raise ValueError( - "Can't partition in a negative or zero number of groups" - ) - elif k > n: - return - - def set_partitions_helper(L, k): - n = len(L) - if k == 1: - yield [L] - elif n == k: - yield [[s] for s in L] - else: - e, *M = L - for p in set_partitions_helper(M, k - 1): - yield [[e], *p] - for p in set_partitions_helper(M, k): - for i in range(len(p)): - yield p[:i] + [[e] + p[i]] + p[i + 1 :] - - if k is None: - for k in range(1, n + 1): - yield from set_partitions_helper(L, k) - else: - yield from set_partitions_helper(L, k) - - -class time_limited: - """ - Yield items from *iterable* until *limit_seconds* have passed. - If the time limit expires before all items have been yielded, the - ``timed_out`` parameter will be set to ``True``. - - >>> from time import sleep - >>> def generator(): - ... yield 1 - ... yield 2 - ... sleep(0.2) - ... yield 3 - >>> iterable = time_limited(0.1, generator()) - >>> list(iterable) - [1, 2] - >>> iterable.timed_out - True - - Note that the time is checked before each item is yielded, and iteration - stops if the time elapsed is greater than *limit_seconds*. If your time - limit is 1 second, but it takes 2 seconds to generate the first item from - the iterable, the function will run for 2 seconds and not yield anything. - As a special case, when *limit_seconds* is zero, the iterator never - returns anything. - - """ - - def __init__(self, limit_seconds, iterable): - if limit_seconds < 0: - raise ValueError('limit_seconds must be positive') - self.limit_seconds = limit_seconds - self._iterable = iter(iterable) - self._start_time = monotonic() - self.timed_out = False - - def __iter__(self): - return self - - def __next__(self): - if self.limit_seconds == 0: - self.timed_out = True - raise StopIteration - item = next(self._iterable) - if monotonic() - self._start_time > self.limit_seconds: - self.timed_out = True - raise StopIteration - - return item - - -def only(iterable, default=None, too_long=None): - """If *iterable* has only one item, return it. - If it has zero items, return *default*. - If it has more than one item, raise the exception given by *too_long*, - which is ``ValueError`` by default. - - >>> only([], default='missing') - 'missing' - >>> only([1]) - 1 - >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 1, 2, - and perhaps more.' - >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - TypeError - - Note that :func:`only` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check - iterable contents less destructively. - """ - it = iter(iterable) - first_value = next(it, default) - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -class _IChunk: - def __init__(self, iterable, n): - self._it = islice(iterable, n) - self._cache = deque() - - def fill_cache(self): - self._cache.extend(self._it) - - def __iter__(self): - return self - - def __next__(self): - try: - return next(self._it) - except StopIteration: - if self._cache: - return self._cache.popleft() - else: - raise - - -def ichunked(iterable, n): - """Break *iterable* into sub-iterables with *n* elements each. - :func:`ichunked` is like :func:`chunked`, but it yields iterables - instead of lists. - - If the sub-iterables are read in order, the elements of *iterable* - won't be stored in memory. - If they are read out of order, :func:`itertools.tee` is used to cache - elements as necessary. - - >>> from itertools import count - >>> all_chunks = ichunked(count(), 4) - >>> c_1, c_2, c_3 = next(all_chunks), next(all_chunks), next(all_chunks) - >>> list(c_2) # c_1's elements have been cached; c_3's haven't been - [4, 5, 6, 7] - >>> list(c_1) - [0, 1, 2, 3] - >>> list(c_3) - [8, 9, 10, 11] - - """ - source = peekable(iter(iterable)) - ichunk_marker = object() - while True: - # Check to see whether we're at the end of the source iterable - item = source.peek(ichunk_marker) - if item is ichunk_marker: - return - - chunk = _IChunk(source, n) - yield chunk - - # Advance the source iterable and fill previous chunk's cache - chunk.fill_cache() - - -def iequals(*iterables): - """Return ``True`` if all given *iterables* are equal to each other, - which means that they contain the same elements in the same order. - - The function is useful for comparing iterables of different data types - or iterables that do not support equality checks. - - >>> iequals("abc", ['a', 'b', 'c'], ('a', 'b', 'c'), iter("abc")) - True - - >>> iequals("abc", "acb") - False - - Not to be confused with :func:`all_equal`, which checks whether all - elements of iterable are equal to each other. - - """ - return all(map(all_equal, zip_longest(*iterables, fillvalue=object()))) - - -def distinct_combinations(iterable, r): - """Yield the distinct combinations of *r* items taken from *iterable*. - - >>> list(distinct_combinations([0, 0, 1], 2)) - [(0, 0), (0, 1)] - - Equivalent to ``set(combinations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - """ - if r < 0: - raise ValueError('r must be non-negative') - elif r == 0: - yield () - return - pool = tuple(iterable) - generators = [unique_everseen(enumerate(pool), key=itemgetter(1))] - current_combo = [None] * r - level = 0 - while generators: - try: - cur_idx, p = next(generators[-1]) - except StopIteration: - generators.pop() - level -= 1 - continue - current_combo[level] = p - if level + 1 == r: - yield tuple(current_combo) - else: - generators.append( - unique_everseen( - enumerate(pool[cur_idx + 1 :], cur_idx + 1), - key=itemgetter(1), - ) - ) - level += 1 - - -def filter_except(validator, iterable, *exceptions): - """Yield the items from *iterable* for which the *validator* function does - not raise one of the specified *exceptions*. - - *validator* is called for each item in *iterable*. - It should be a function that accepts one argument and raises an exception - if that item is not valid. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(filter_except(int, iterable, ValueError, TypeError)) - ['1', '2', '4'] - - If an exception other than one given by *exceptions* is raised by - *validator*, it is raised like normal. - """ - for item in iterable: - try: - validator(item) - except exceptions: - pass - else: - yield item - - -def map_except(function, iterable, *exceptions): - """Transform each item from *iterable* with *function* and yield the - result, unless *function* raises one of the specified *exceptions*. - - *function* is called to transform each item in *iterable*. - It should accept one argument. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(map_except(int, iterable, ValueError, TypeError)) - [1, 2, 4] - - If an exception other than one given by *exceptions* is raised by - *function*, it is raised like normal. - """ - for item in iterable: - try: - yield function(item) - except exceptions: - pass - - -def map_if(iterable, pred, func, func_else=lambda x: x): - """Evaluate each item from *iterable* using *pred*. If the result is - equivalent to ``True``, transform the item with *func* and yield it. - Otherwise, transform the item with *func_else* and yield it. - - *pred*, *func*, and *func_else* should each be functions that accept - one argument. By default, *func_else* is the identity function. - - >>> from math import sqrt - >>> iterable = list(range(-5, 5)) - >>> iterable - [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4] - >>> list(map_if(iterable, lambda x: x > 3, lambda x: 'toobig')) - [-5, -4, -3, -2, -1, 0, 1, 2, 3, 'toobig'] - >>> list(map_if(iterable, lambda x: x >= 0, - ... lambda x: f'{sqrt(x):.2f}', lambda x: None)) - [None, None, None, None, None, '0.00', '1.00', '1.41', '1.73', '2.00'] - """ - for item in iterable: - yield func(item) if pred(item) else func_else(item) - - -def _sample_unweighted(iterable, k): - # Implementation of "Algorithm L" from the 1994 paper by Kim-Hung Li: - # "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". - - # Fill up the reservoir (collection of samples) with the first `k` samples - reservoir = take(k, iterable) - - # Generate random number that's the largest in a sample of k U(0,1) numbers - # Largest order statistic: https://en.wikipedia.org/wiki/Order_statistic - W = exp(log(random()) / k) - - # The number of elements to skip before changing the reservoir is a random - # number with a geometric distribution. Sample it using random() and logs. - next_index = k + floor(log(random()) / log(1 - W)) - - for index, element in enumerate(iterable, k): - if index == next_index: - reservoir[randrange(k)] = element - # The new W is the largest in a sample of k U(0, `old_W`) numbers - W *= exp(log(random()) / k) - next_index += floor(log(random()) / log(1 - W)) + 1 - - return reservoir - - -def _sample_weighted(iterable, k, weights): - # Implementation of "A-ExpJ" from the 2006 paper by Efraimidis et al. : - # "Weighted random sampling with a reservoir". - - # Log-transform for numerical stability for weights that are small/large - weight_keys = (log(random()) / weight for weight in weights) - - # Fill up the reservoir (collection of samples) with the first `k` - # weight-keys and elements, then heapify the list. - reservoir = take(k, zip(weight_keys, iterable)) - heapify(reservoir) - - # The number of jumps before changing the reservoir is a random variable - # with an exponential distribution. Sample it using random() and logs. - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - - for weight, element in zip(weights, iterable): - if weight >= weights_to_skip: - # The notation here is consistent with the paper, but we store - # the weight-keys in log-space for better numerical stability. - smallest_weight_key, _ = reservoir[0] - t_w = exp(weight * smallest_weight_key) - r_2 = uniform(t_w, 1) # generate U(t_w, 1) - weight_key = log(r_2) / weight - heapreplace(reservoir, (weight_key, element)) - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - else: - weights_to_skip -= weight - - # Equivalent to [element for weight_key, element in sorted(reservoir)] - return [heappop(reservoir)[1] for _ in range(k)] - - -def sample(iterable, k, weights=None): - """Return a *k*-length list of elements chosen (without replacement) - from the *iterable*. Like :func:`random.sample`, but works on iterables - of unknown length. - - >>> iterable = range(100) - >>> sample(iterable, 5) # doctest: +SKIP - [81, 60, 96, 16, 4] - - An iterable with *weights* may also be given: - - >>> iterable = range(100) - >>> weights = (i * i + 1 for i in range(100)) - >>> sampled = sample(iterable, 5, weights=weights) # doctest: +SKIP - [79, 67, 74, 66, 78] - - The algorithm can also be used to generate weighted random permutations. - The relative weight of each item determines the probability that it - appears late in the permutation. - - >>> data = "abcdefgh" - >>> weights = range(1, len(data) + 1) - >>> sample(data, k=len(data), weights=weights) # doctest: +SKIP - ['c', 'a', 'b', 'e', 'g', 'd', 'h', 'f'] - """ - if k == 0: - return [] - - iterable = iter(iterable) - if weights is None: - return _sample_unweighted(iterable, k) - else: - weights = iter(weights) - return _sample_weighted(iterable, k, weights) - - -def is_sorted(iterable, key=None, reverse=False, strict=False): - """Returns ``True`` if the items of iterable are in sorted order, and - ``False`` otherwise. *key* and *reverse* have the same meaning that they do - in the built-in :func:`sorted` function. - - >>> is_sorted(['1', '2', '3', '4', '5'], key=int) - True - >>> is_sorted([5, 4, 3, 1, 2], reverse=True) - False - - If *strict*, tests for strict sorting, that is, returns ``False`` if equal - elements are found: - - >>> is_sorted([1, 2, 2]) - True - >>> is_sorted([1, 2, 2], strict=True) - False - - The function returns ``False`` after encountering the first out-of-order - item. If there are no out-of-order items, the iterable is exhausted. - """ - - compare = (le if reverse else ge) if strict else (lt if reverse else gt) - it = iterable if key is None else map(key, iterable) - return not any(starmap(compare, pairwise(it))) - - -class AbortThread(BaseException): - pass - - -class callback_iter: - """Convert a function that uses callbacks to an iterator. - - Let *func* be a function that takes a `callback` keyword argument. - For example: - - >>> def func(callback=None): - ... for i, c in [(1, 'a'), (2, 'b'), (3, 'c')]: - ... if callback: - ... callback(i, c) - ... return 4 - - - Use ``with callback_iter(func)`` to get an iterator over the parameters - that are delivered to the callback. - - >>> with callback_iter(func) as it: - ... for args, kwargs in it: - ... print(args) - (1, 'a') - (2, 'b') - (3, 'c') - - The function will be called in a background thread. The ``done`` property - indicates whether it has completed execution. - - >>> it.done - True - - If it completes successfully, its return value will be available - in the ``result`` property. - - >>> it.result - 4 - - Notes: - - * If the function uses some keyword argument besides ``callback``, supply - *callback_kwd*. - * If it finished executing, but raised an exception, accessing the - ``result`` property will raise the same exception. - * If it hasn't finished executing, accessing the ``result`` - property from within the ``with`` block will raise ``RuntimeError``. - * If it hasn't finished executing, accessing the ``result`` property from - outside the ``with`` block will raise a - ``more_itertools.AbortThread`` exception. - * Provide *wait_seconds* to adjust how frequently the it is polled for - output. - - """ - - def __init__(self, func, callback_kwd='callback', wait_seconds=0.1): - self._func = func - self._callback_kwd = callback_kwd - self._aborted = False - self._future = None - self._wait_seconds = wait_seconds - # Lazily import concurrent.future - self._executor = __import__( - ).futures.__import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1) - self._iterator = self._reader() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._aborted = True - self._executor.shutdown() - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterator) - - @property - def done(self): - if self._future is None: - return False - return self._future.done() - - @property - def result(self): - if not self.done: - raise RuntimeError('Function has not yet completed') - - return self._future.result() - - def _reader(self): - q = Queue() - - def callback(*args, **kwargs): - if self._aborted: - raise AbortThread('canceled by user') - - q.put((args, kwargs)) - - self._future = self._executor.submit( - self._func, **{self._callback_kwd: callback} - ) - - while True: - try: - item = q.get(timeout=self._wait_seconds) - except Empty: - pass - else: - q.task_done() - yield item - - if self._future.done(): - break - - remaining = [] - while True: - try: - item = q.get_nowait() - except Empty: - break - else: - q.task_done() - remaining.append(item) - q.join() - yield from remaining - - -def windowed_complete(iterable, n): - """ - Yield ``(beginning, middle, end)`` tuples, where: - - * Each ``middle`` has *n* items from *iterable* - * Each ``beginning`` has the items before the ones in ``middle`` - * Each ``end`` has the items after the ones in ``middle`` - - >>> iterable = range(7) - >>> n = 3 - >>> for beginning, middle, end in windowed_complete(iterable, n): - ... print(beginning, middle, end) - () (0, 1, 2) (3, 4, 5, 6) - (0,) (1, 2, 3) (4, 5, 6) - (0, 1) (2, 3, 4) (5, 6) - (0, 1, 2) (3, 4, 5) (6,) - (0, 1, 2, 3) (4, 5, 6) () - - Note that *n* must be at least 0 and most equal to the length of - *iterable*. - - This function will exhaust the iterable and may require significant - storage. - """ - if n < 0: - raise ValueError('n must be >= 0') - - seq = tuple(iterable) - size = len(seq) - - if n > size: - raise ValueError('n must be <= len(seq)') - - for i in range(size - n + 1): - beginning = seq[:i] - middle = seq[i : i + n] - end = seq[i + n :] - yield beginning, middle, end - - -def all_unique(iterable, key=None): - """ - Returns ``True`` if all the elements of *iterable* are unique (no two - elements are equal). - - >>> all_unique('ABCB') - False - - If a *key* function is specified, it will be used to make comparisons. - - >>> all_unique('ABCb') - True - >>> all_unique('ABCb', str.lower) - False - - The function returns as soon as the first non-unique element is - encountered. Iterables with a mix of hashable and unhashable items can - be used, but the function will be slower for unhashable items. - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - for element in map(key, iterable) if key else iterable: - try: - if element in seenset: - return False - seenset_add(element) - except TypeError: - if element in seenlist: - return False - seenlist_add(element) - return True - - -def nth_product(index, *args): - """Equivalent to ``list(product(*args))[index]``. - - The products of *args* can be ordered lexicographically. - :func:`nth_product` computes the product at sort position *index* without - computing the previous products. - - >>> nth_product(8, range(2), range(2), range(2), range(2)) - (1, 0, 0, 0) - - ``IndexError`` will be raised if the given *index* is invalid. - """ - pools = list(map(tuple, reversed(args))) - ns = list(map(len, pools)) - - c = reduce(mul, ns) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - result = [] - for pool, n in zip(pools, ns): - result.append(pool[index % n]) - index //= n - - return tuple(reversed(result)) - - -def nth_permutation(iterable, r, index): - """Equivalent to ``list(permutations(iterable, r))[index]``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`nth_permutation` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences. - - >>> nth_permutation('ghijk', 2, 5) - ('h', 'i') - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = list(iterable) - n = len(pool) - - if r is None or r == n: - r, c = n, factorial(n) - elif not 0 <= r < n: - raise ValueError - else: - c = perm(n, r) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - if c == 0: - return tuple() - - result = [0] * r - q = index * factorial(n) // c if r < n else index - for d in range(1, n + 1): - q, i = divmod(q, d) - if 0 <= n - d < r: - result[n - d] = i - if q == 0: - break - - return tuple(map(pool.pop, result)) - - -def nth_combination_with_replacement(iterable, r, index): - """Equivalent to - ``list(combinations_with_replacement(iterable, r))[index]``. - - - The subsequences with repetition of *iterable* that are of length *r* can - be ordered lexicographically. :func:`nth_combination_with_replacement` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences with replacement. - - >>> nth_combination_with_replacement(range(5), 3, 5) - (0, 1, 1) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = comb(n + r - 1, r) - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - i = 0 - while r: - r -= 1 - while n >= 0: - num_combs = comb(n + r - 1, r) - if index < num_combs: - break - n -= 1 - i += 1 - index -= num_combs - result.append(pool[i]) - - return tuple(result) - - -def value_chain(*args): - """Yield all arguments passed to the function in the same order in which - they were passed. If an argument itself is iterable then iterate over its - values. - - >>> list(value_chain(1, 2, 3, [4, 5, 6])) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and are emitted - as-is: - - >>> list(value_chain('12', '34', ['56', '78'])) - ['12', '34', '56', '78'] - - - Multiple levels of nesting are not flattened. - - """ - for value in args: - if isinstance(value, (str, bytes)): - yield value - continue - try: - yield from value - except TypeError: - yield value - - -def product_index(element, *args): - """Equivalent to ``list(product(*args)).index(element)`` - - The products of *args* can be ordered lexicographically. - :func:`product_index` computes the first index of *element* without - computing the previous products. - - >>> product_index([8, 2], range(10), range(5)) - 42 - - ``ValueError`` will be raised if the given *element* isn't in the product - of *args*. - """ - index = 0 - - for x, pool in zip_longest(element, args, fillvalue=_marker): - if x is _marker or pool is _marker: - raise ValueError('element is not a product of args') - - pool = tuple(pool) - index = index * len(pool) + pool.index(x) - - return index - - -def combination_index(element, iterable): - """Equivalent to ``list(combinations(iterable, r)).index(element)`` - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`combination_index` computes the index of the - first *element*, without computing the previous combinations. - - >>> combination_index('adf', 'abcdefg') - 10 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations of *iterable*. - """ - element = enumerate(element) - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = enumerate(iterable) - for n, x in pool: - if x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - else: - raise ValueError('element is not a combination of iterable') - - n, _ = last(pool, default=(n, None)) - - # Python versions below 3.8 don't have math.comb - index = 1 - for i, j in enumerate(reversed(indexes), start=1): - j = n - j - if i <= j: - index += comb(j, i) - - return comb(n + 1, k + 1) - index - - -def combination_with_replacement_index(element, iterable): - """Equivalent to - ``list(combinations_with_replacement(iterable, r)).index(element)`` - - The subsequences with repetition of *iterable* that are of length *r* can - be ordered lexicographically. :func:`combination_with_replacement_index` - computes the index of the first *element*, without computing the previous - combinations with replacement. - - >>> combination_with_replacement_index('adf', 'abcdefg') - 20 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations with replacement of *iterable*. - """ - element = tuple(element) - l = len(element) - element = enumerate(element) - - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = tuple(iterable) - for n, x in enumerate(pool): - while x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - if y is None: - break - else: - raise ValueError( - 'element is not a combination with replacement of iterable' - ) - - n = len(pool) - occupations = [0] * n - for p in indexes: - occupations[p] += 1 - - index = 0 - cumulative_sum = 0 - for k in range(1, n): - cumulative_sum += occupations[k - 1] - j = l + n - 1 - k - cumulative_sum - i = n - k - if i <= j: - index += comb(j, i) - - return index - - -def permutation_index(element, iterable): - """Equivalent to ``list(permutations(iterable, r)).index(element)``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`permutation_index` - computes the index of the first *element* directly, without computing - the previous permutations. - - >>> permutation_index([1, 3, 2], range(5)) - 19 - - ``ValueError`` will be raised if the given *element* isn't one of the - permutations of *iterable*. - """ - index = 0 - pool = list(iterable) - for i, x in zip(range(len(pool), -1, -1), element): - r = pool.index(x) - index = index * i + r - del pool[r] - - return index - - -class countable: - """Wrap *iterable* and keep a count of how many items have been consumed. - - The ``items_seen`` attribute starts at ``0`` and increments as the iterable - is consumed: - - >>> iterable = map(str, range(10)) - >>> it = countable(iterable) - >>> it.items_seen - 0 - >>> next(it), next(it) - ('0', '1') - >>> list(it) - ['2', '3', '4', '5', '6', '7', '8', '9'] - >>> it.items_seen - 10 - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self.items_seen = 0 - - def __iter__(self): - return self - - def __next__(self): - item = next(self._it) - self.items_seen += 1 - - return item - - -def chunked_even(iterable, n): - """Break *iterable* into lists of approximately length *n*. - Items are distributed such the lengths of the lists differ by at most - 1 item. - - >>> iterable = [1, 2, 3, 4, 5, 6, 7] - >>> n = 3 - >>> list(chunked_even(iterable, n)) # List lengths: 3, 2, 2 - [[1, 2, 3], [4, 5], [6, 7]] - >>> list(chunked(iterable, n)) # List lengths: 3, 3, 1 - [[1, 2, 3], [4, 5, 6], [7]] - - """ - - len_method = getattr(iterable, '__len__', None) - - if len_method is None: - return _chunked_even_online(iterable, n) - else: - return _chunked_even_finite(iterable, len_method(), n) - - -def _chunked_even_online(iterable, n): - buffer = [] - maxbuf = n + (n - 2) * (n - 1) - for x in iterable: - buffer.append(x) - if len(buffer) == maxbuf: - yield buffer[:n] - buffer = buffer[n:] - yield from _chunked_even_finite(buffer, len(buffer), n) - - -def _chunked_even_finite(iterable, N, n): - if N < 1: - return - - # Lists are either size `full_size <= n` or `partial_size = full_size - 1` - q, r = divmod(N, n) - num_lists = q + (1 if r > 0 else 0) - q, r = divmod(N, num_lists) - full_size = q + (1 if r > 0 else 0) - partial_size = full_size - 1 - num_full = N - partial_size * num_lists - num_partial = num_lists - num_full - - # Yield num_full lists of full_size - partial_start_idx = num_full * full_size - if full_size > 0: - for i in range(0, partial_start_idx, full_size): - yield list(islice(iterable, i, i + full_size)) - - # Yield num_partial lists of partial_size - if partial_size > 0: - for i in range( - partial_start_idx, - partial_start_idx + (num_partial * partial_size), - partial_size, - ): - yield list(islice(iterable, i, i + partial_size)) - - -def zip_broadcast(*objects, scalar_types=(str, bytes), strict=False): - """A version of :func:`zip` that "broadcasts" any scalar - (i.e., non-iterable) items into output tuples. - - >>> iterable_1 = [1, 2, 3] - >>> iterable_2 = ['a', 'b', 'c'] - >>> scalar = '_' - >>> list(zip_broadcast(iterable_1, iterable_2, scalar)) - [(1, 'a', '_'), (2, 'b', '_'), (3, 'c', '_')] - - The *scalar_types* keyword argument determines what types are considered - scalar. It is set to ``(str, bytes)`` by default. Set it to ``None`` to - treat strings and byte strings as iterable: - - >>> list(zip_broadcast('abc', 0, 'xyz', scalar_types=None)) - [('a', 0, 'x'), ('b', 0, 'y'), ('c', 0, 'z')] - - If the *strict* keyword argument is ``True``, then - ``UnequalIterablesError`` will be raised if any of the iterables have - different lengths. - """ - - def is_scalar(obj): - if scalar_types and isinstance(obj, scalar_types): - return True - try: - iter(obj) - except TypeError: - return True - else: - return False - - size = len(objects) - if not size: - return - - new_item = [None] * size - iterables, iterable_positions = [], [] - for i, obj in enumerate(objects): - if is_scalar(obj): - new_item[i] = obj - else: - iterables.append(iter(obj)) - iterable_positions.append(i) - - if not iterables: - yield tuple(objects) - return - - zipper = _zip_equal if strict else zip - for item in zipper(*iterables): - for i, new_item[i] in zip(iterable_positions, item): - pass - yield tuple(new_item) - - -def unique_in_window(iterable, n, key=None): - """Yield the items from *iterable* that haven't been seen recently. - *n* is the size of the lookback window. - - >>> iterable = [0, 1, 0, 2, 3, 0] - >>> n = 3 - >>> list(unique_in_window(iterable, n)) - [0, 1, 2, 3, 0] - - The *key* function, if provided, will be used to determine uniqueness: - - >>> list(unique_in_window('abAcda', 3, key=lambda x: x.lower())) - ['a', 'b', 'c', 'd', 'a'] - - The items in *iterable* must be hashable. - - """ - if n <= 0: - raise ValueError('n must be greater than 0') - - window = deque(maxlen=n) - counts = defaultdict(int) - use_key = key is not None - - for item in iterable: - if len(window) == n: - to_discard = window[0] - if counts[to_discard] == 1: - del counts[to_discard] - else: - counts[to_discard] -= 1 - - k = key(item) if use_key else item - if k not in counts: - yield item - counts[k] += 1 - window.append(k) - - -def duplicates_everseen(iterable, key=None): - """Yield duplicate elements after their first appearance. - - >>> list(duplicates_everseen('mississippi')) - ['s', 'i', 's', 's', 'i', 'p', 'i'] - >>> list(duplicates_everseen('AaaBbbCccAaa', str.lower)) - ['a', 'a', 'b', 'b', 'c', 'c', 'A', 'a', 'a'] - - This function is analogous to :func:`unique_everseen` and is subject to - the same performance considerations. - - """ - seen_set = set() - seen_list = [] - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seen_set: - seen_set.add(k) - else: - yield element - except TypeError: - if k not in seen_list: - seen_list.append(k) - else: - yield element - - -def duplicates_justseen(iterable, key=None): - """Yields serially-duplicate elements after their first appearance. - - >>> list(duplicates_justseen('mississippi')) - ['s', 's', 'p'] - >>> list(duplicates_justseen('AaaBbbCccAaa', str.lower)) - ['a', 'a', 'b', 'b', 'c', 'c', 'a', 'a'] - - This function is analogous to :func:`unique_justseen`. - - """ - return flatten(g for _, g in groupby(iterable, key) for _ in g) - - -def classify_unique(iterable, key=None): - """Classify each element in terms of its uniqueness. - - For each element in the input iterable, return a 3-tuple consisting of: - - 1. The element itself - 2. ``False`` if the element is equal to the one preceding it in the input, - ``True`` otherwise (i.e. the equivalent of :func:`unique_justseen`) - 3. ``False`` if this element has been seen anywhere in the input before, - ``True`` otherwise (i.e. the equivalent of :func:`unique_everseen`) - - >>> list(classify_unique('otto')) # doctest: +NORMALIZE_WHITESPACE - [('o', True, True), - ('t', True, True), - ('t', False, False), - ('o', True, False)] - - This function is analogous to :func:`unique_everseen` and is subject to - the same performance considerations. - - """ - seen_set = set() - seen_list = [] - use_key = key is not None - previous = None - - for i, element in enumerate(iterable): - k = key(element) if use_key else element - is_unique_justseen = not i or previous != k - previous = k - is_unique_everseen = False - try: - if k not in seen_set: - seen_set.add(k) - is_unique_everseen = True - except TypeError: - if k not in seen_list: - seen_list.append(k) - is_unique_everseen = True - yield element, is_unique_justseen, is_unique_everseen - - -def minmax(iterable_or_value, *others, key=None, default=_marker): - """Returns both the smallest and largest items in an iterable - or the largest of two or more arguments. - - >>> minmax([3, 1, 5]) - (1, 5) - - >>> minmax(4, 2, 6) - (2, 6) - - If a *key* function is provided, it will be used to transform the input - items for comparison. - - >>> minmax([5, 30], key=str) # '30' sorts before '5' - (30, 5) - - If a *default* value is provided, it will be returned if there are no - input items. - - >>> minmax([], default=(0, 0)) - (0, 0) - - Otherwise ``ValueError`` is raised. - - This function is based on the - `recipe <http://code.activestate.com/recipes/577916/>`__ by - Raymond Hettinger and takes care to minimize the number of comparisons - performed. - """ - iterable = (iterable_or_value, *others) if others else iterable_or_value - - it = iter(iterable) - - try: - lo = hi = next(it) - except StopIteration as e: - if default is _marker: - raise ValueError( - '`minmax()` argument is an empty iterable. ' - 'Provide a `default` value to suppress this error.' - ) from e - return default - - # Different branches depending on the presence of key. This saves a lot - # of unimportant copies which would slow the "key=None" branch - # significantly down. - if key is None: - for x, y in zip_longest(it, it, fillvalue=lo): - if y < x: - x, y = y, x - if x < lo: - lo = x - if hi < y: - hi = y - - else: - lo_key = hi_key = key(lo) - - for x, y in zip_longest(it, it, fillvalue=lo): - x_key, y_key = key(x), key(y) - - if y_key < x_key: - x, y, x_key, y_key = y, x, y_key, x_key - if x_key < lo_key: - lo, lo_key = x, x_key - if hi_key < y_key: - hi, hi_key = y, y_key - - return lo, hi - - -def constrained_batches( - iterable, max_size, max_count=None, get_len=len, strict=True -): - """Yield batches of items from *iterable* with a combined size limited by - *max_size*. - - >>> iterable = [b'12345', b'123', b'12345678', b'1', b'1', b'12', b'1'] - >>> list(constrained_batches(iterable, 10)) - [(b'12345', b'123'), (b'12345678', b'1', b'1'), (b'12', b'1')] - - If a *max_count* is supplied, the number of items per batch is also - limited: - - >>> iterable = [b'12345', b'123', b'12345678', b'1', b'1', b'12', b'1'] - >>> list(constrained_batches(iterable, 10, max_count = 2)) - [(b'12345', b'123'), (b'12345678', b'1'), (b'1', b'12'), (b'1',)] - - If a *get_len* function is supplied, use that instead of :func:`len` to - determine item size. - - If *strict* is ``True``, raise ``ValueError`` if any single item is bigger - than *max_size*. Otherwise, allow single items to exceed *max_size*. - """ - if max_size <= 0: - raise ValueError('maximum size must be greater than zero') - - batch = [] - batch_size = 0 - batch_count = 0 - for item in iterable: - item_len = get_len(item) - if strict and item_len > max_size: - raise ValueError('item size exceeds maximum size') - - reached_count = batch_count == max_count - reached_size = item_len + batch_size > max_size - if batch_count and (reached_size or reached_count): - yield tuple(batch) - batch.clear() - batch_size = 0 - batch_count = 0 - - batch.append(item) - batch_size += item_len - batch_count += 1 - - if batch: - yield tuple(batch) - - -def gray_product(*iterables): - """Like :func:`itertools.product`, but return tuples in an order such - that only one element in the generated tuple changes from one iteration - to the next. - - >>> list(gray_product('AB','CD')) - [('A', 'C'), ('B', 'C'), ('B', 'D'), ('A', 'D')] - - This function consumes all of the input iterables before producing output. - If any of the input iterables have fewer than two items, ``ValueError`` - is raised. - - For information on the algorithm, see - `this section <https://www-cs-faculty.stanford.edu/~knuth/fasc2a.ps.gz>`__ - of Donald Knuth's *The Art of Computer Programming*. - """ - all_iterables = tuple(tuple(x) for x in iterables) - iterable_count = len(all_iterables) - for iterable in all_iterables: - if len(iterable) < 2: - raise ValueError("each iterable must have two or more items") - - # This is based on "Algorithm H" from section 7.2.1.1, page 20. - # a holds the indexes of the source iterables for the n-tuple to be yielded - # f is the array of "focus pointers" - # o is the array of "directions" - a = [0] * iterable_count - f = list(range(iterable_count + 1)) - o = [1] * iterable_count - while True: - yield tuple(all_iterables[i][a[i]] for i in range(iterable_count)) - j = f[0] - f[0] = 0 - if j == iterable_count: - break - a[j] = a[j] + o[j] - if a[j] == 0 or a[j] == len(all_iterables[j]) - 1: - o[j] = -o[j] - f[j] = f[j + 1] - f[j + 1] = j + 1 - - -def partial_product(*iterables): - """Yields tuples containing one item from each iterator, with subsequent - tuples changing a single item at a time by advancing each iterator until it - is exhausted. This sequence guarantees every value in each iterable is - output at least once without generating all possible combinations. - - This may be useful, for example, when testing an expensive function. - - >>> list(partial_product('AB', 'C', 'DEF')) - [('A', 'C', 'D'), ('B', 'C', 'D'), ('B', 'C', 'E'), ('B', 'C', 'F')] - """ - - iterators = list(map(iter, iterables)) - - try: - prod = [next(it) for it in iterators] - except StopIteration: - return - yield tuple(prod) - - for i, it in enumerate(iterators): - for prod[i] in it: - yield tuple(prod) - - -def takewhile_inclusive(predicate, iterable): - """A variant of :func:`takewhile` that yields one additional element. - - >>> list(takewhile_inclusive(lambda x: x < 5, [1, 4, 6, 4, 1])) - [1, 4, 6] - - :func:`takewhile` would return ``[1, 4]``. - """ - for x in iterable: - yield x - if not predicate(x): - break - - -def outer_product(func, xs, ys, *args, **kwargs): - """A generalized outer product that applies a binary function to all - pairs of items. Returns a 2D matrix with ``len(xs)`` rows and ``len(ys)`` - columns. - Also accepts ``*args`` and ``**kwargs`` that are passed to ``func``. - - Multiplication table: - - >>> list(outer_product(mul, range(1, 4), range(1, 6))) - [(1, 2, 3, 4, 5), (2, 4, 6, 8, 10), (3, 6, 9, 12, 15)] - - Cross tabulation: - - >>> xs = ['A', 'B', 'A', 'A', 'B', 'B', 'A', 'A', 'B', 'B'] - >>> ys = ['X', 'X', 'X', 'Y', 'Z', 'Z', 'Y', 'Y', 'Z', 'Z'] - >>> rows = list(zip(xs, ys)) - >>> count_rows = lambda x, y: rows.count((x, y)) - >>> list(outer_product(count_rows, sorted(set(xs)), sorted(set(ys)))) - [(2, 3, 0), (1, 0, 4)] - - Usage with ``*args`` and ``**kwargs``: - - >>> animals = ['cat', 'wolf', 'mouse'] - >>> list(outer_product(min, animals, animals, key=len)) - [('cat', 'cat', 'cat'), ('cat', 'wolf', 'wolf'), ('cat', 'wolf', 'mouse')] - """ - ys = tuple(ys) - return batched( - starmap(lambda x, y: func(x, y, *args, **kwargs), product(xs, ys)), - n=len(ys), - ) - - -def iter_suppress(iterable, *exceptions): - """Yield each of the items from *iterable*. If the iteration raises one of - the specified *exceptions*, that exception will be suppressed and iteration - will stop. - - >>> from itertools import chain - >>> def breaks_at_five(x): - ... while True: - ... if x >= 5: - ... raise RuntimeError - ... yield x - ... x += 1 - >>> it_1 = iter_suppress(breaks_at_five(1), RuntimeError) - >>> it_2 = iter_suppress(breaks_at_five(2), RuntimeError) - >>> list(chain(it_1, it_2)) - [1, 2, 3, 4, 2, 3, 4] - """ - try: - yield from iterable - except exceptions: - return - - -def filter_map(func, iterable): - """Apply *func* to every element of *iterable*, yielding only those which - are not ``None``. - - >>> elems = ['1', 'a', '2', 'b', '3'] - >>> list(filter_map(lambda s: int(s) if s.isnumeric() else None, elems)) - [1, 2, 3] - """ - for x in iterable: - y = func(x) - if y is not None: - yield y diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi deleted file mode 100644 index 9a5fc911a3e..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/more.pyi +++ /dev/null @@ -1,695 +0,0 @@ -"""Stubs for more_itertools.more""" -from __future__ import annotations - -from types import TracebackType -from typing import ( - Any, - Callable, - Container, - ContextManager, - Generic, - Hashable, - Iterable, - Iterator, - overload, - Reversible, - Sequence, - Sized, - Type, - TypeVar, - type_check_only, -) -from typing_extensions import Protocol - -# Type and type variable definitions -_T = TypeVar('_T') -_T1 = TypeVar('_T1') -_T2 = TypeVar('_T2') -_U = TypeVar('_U') -_V = TypeVar('_V') -_W = TypeVar('_W') -_T_co = TypeVar('_T_co', covariant=True) -_GenFn = TypeVar('_GenFn', bound=Callable[..., Iterator[Any]]) -_Raisable = BaseException | Type[BaseException] - -@type_check_only -class _SizedIterable(Protocol[_T_co], Sized, Iterable[_T_co]): ... - -@type_check_only -class _SizedReversible(Protocol[_T_co], Sized, Reversible[_T_co]): ... - -@type_check_only -class _SupportsSlicing(Protocol[_T_co]): - def __getitem__(self, __k: slice) -> _T_co: ... - -def chunked( - iterable: Iterable[_T], n: int | None, strict: bool = ... -) -> Iterator[list[_T]]: ... -@overload -def first(iterable: Iterable[_T]) -> _T: ... -@overload -def first(iterable: Iterable[_T], default: _U) -> _T | _U: ... -@overload -def last(iterable: Iterable[_T]) -> _T: ... -@overload -def last(iterable: Iterable[_T], default: _U) -> _T | _U: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int) -> _T: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int, default: _U) -> _T | _U: ... - -class peekable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> peekable[_T]: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> _T | _U: ... - def prepend(self, *items: _T) -> None: ... - def __next__(self) -> _T: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> list[_T]: ... - -def consumer(func: _GenFn) -> _GenFn: ... -def ilen(iterable: Iterable[_T]) -> int: ... -def iterate(func: Callable[[_T], _T], start: _T) -> Iterator[_T]: ... -def with_iter( - context_manager: ContextManager[Iterable[_T]], -) -> Iterator[_T]: ... -def one( - iterable: Iterable[_T], - too_short: _Raisable | None = ..., - too_long: _Raisable | None = ..., -) -> _T: ... -def raise_(exception: _Raisable, *args: Any) -> None: ... -def strictly_n( - iterable: Iterable[_T], - n: int, - too_short: _GenFn | None = ..., - too_long: _GenFn | None = ..., -) -> list[_T]: ... -def distinct_permutations( - iterable: Iterable[_T], r: int | None = ... -) -> Iterator[tuple[_T, ...]]: ... -def intersperse( - e: _U, iterable: Iterable[_T], n: int = ... -) -> Iterator[_T | _U]: ... -def unique_to_each(*iterables: Iterable[_T]) -> list[list[_T]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, *, step: int = ... -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, fillvalue: _U, step: int = ... -) -> Iterator[tuple[_T | _U, ...]]: ... -def substrings(iterable: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def substrings_indexes( - seq: Sequence[_T], reverse: bool = ... -) -> Iterator[tuple[Sequence[_T], int, int]]: ... - -class bucket(Generic[_T, _U], Container[_U]): - def __init__( - self, - iterable: Iterable[_T], - key: Callable[[_T], _U], - validator: Callable[[_U], object] | None = ..., - ) -> None: ... - def __contains__(self, value: object) -> bool: ... - def __iter__(self) -> Iterator[_U]: ... - def __getitem__(self, value: object) -> Iterator[_T]: ... - -def spy( - iterable: Iterable[_T], n: int = ... -) -> tuple[list[_T], Iterator[_T]]: ... -def interleave(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_longest(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_evenly( - iterables: list[Iterable[_T]], lengths: list[int] | None = ... -) -> Iterator[_T]: ... -def collapse( - iterable: Iterable[Any], - base_type: type | None = ..., - levels: int | None = ..., -) -> Iterator[Any]: ... -@overload -def side_effect( - func: Callable[[_T], object], - iterable: Iterable[_T], - chunk_size: None = ..., - before: Callable[[], object] | None = ..., - after: Callable[[], object] | None = ..., -) -> Iterator[_T]: ... -@overload -def side_effect( - func: Callable[[list[_T]], object], - iterable: Iterable[_T], - chunk_size: int, - before: Callable[[], object] | None = ..., - after: Callable[[], object] | None = ..., -) -> Iterator[_T]: ... -def sliced( - seq: _SupportsSlicing[_T], n: int, strict: bool = ... -) -> Iterator[_T]: ... -def split_at( - iterable: Iterable[_T], - pred: Callable[[_T], object], - maxsplit: int = ..., - keep_separator: bool = ..., -) -> Iterator[list[_T]]: ... -def split_before( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[list[_T]]: ... -def split_after( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[list[_T]]: ... -def split_when( - iterable: Iterable[_T], - pred: Callable[[_T, _T], object], - maxsplit: int = ..., -) -> Iterator[list[_T]]: ... -def split_into( - iterable: Iterable[_T], sizes: Iterable[int | None] -) -> Iterator[list[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - *, - n: int | None = ..., - next_multiple: bool = ..., -) -> Iterator[_T | None]: ... -@overload -def padded( - iterable: Iterable[_T], - fillvalue: _U, - n: int | None = ..., - next_multiple: bool = ..., -) -> Iterator[_T | _U]: ... -@overload -def repeat_last(iterable: Iterable[_T]) -> Iterator[_T]: ... -@overload -def repeat_last(iterable: Iterable[_T], default: _U) -> Iterator[_T | _U]: ... -def distribute(n: int, iterable: Iterable[_T]) -> list[Iterator[_T]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., - fillvalue: _U = ..., -) -> Iterator[tuple[_T | _U, ...]]: ... - -class UnequalIterablesError(ValueError): - def __init__(self, details: tuple[int, int, int] | None = ...) -> None: ... - -@overload -def zip_equal(__iter1: Iterable[_T1]) -> Iterator[tuple[_T1]]: ... -@overload -def zip_equal( - __iter1: Iterable[_T1], __iter2: Iterable[_T2] -) -> Iterator[tuple[_T1, _T2]]: ... -@overload -def zip_equal( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], -) -> Iterator[tuple[_T, ...]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T1 | None]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - __iter2: Iterable[_T2], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T1 | None, _T2 | None]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: None = None, -) -> Iterator[tuple[_T | None, ...]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T1 | _U]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T1], - __iter2: Iterable[_T2], - *, - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T1 | _U, _T2 | _U]]: ... -@overload -def zip_offset( - __iter1: Iterable[_T], - __iter2: Iterable[_T], - __iter3: Iterable[_T], - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U, -) -> Iterator[tuple[_T | _U, ...]]: ... -def sort_together( - iterables: Iterable[Iterable[_T]], - key_list: Iterable[int] = ..., - key: Callable[..., Any] | None = ..., - reverse: bool = ..., -) -> list[tuple[_T, ...]]: ... -def unzip(iterable: Iterable[Sequence[_T]]) -> tuple[Iterator[_T], ...]: ... -def divide(n: int, iterable: Iterable[_T]) -> list[Iterator[_T]]: ... -def always_iterable( - obj: object, - base_type: type | tuple[type | tuple[Any, ...], ...] | None = ..., -) -> Iterator[Any]: ... -def adjacent( - predicate: Callable[[_T], bool], - iterable: Iterable[_T], - distance: int = ..., -) -> Iterator[tuple[bool, _T]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None = None, - valuefunc: None = None, - reducefunc: None = None, -) -> Iterator[tuple[_T, Iterator[_T]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None, - reducefunc: None, -) -> Iterator[tuple[_U, Iterator[_T]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: Callable[[_T], _V], - reducefunc: None, -) -> Iterable[tuple[_T, Iterable[_V]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None, -) -> Iterable[tuple[_U, Iterator[_V]]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: None, - reducefunc: Callable[[Iterator[_T]], _W], -) -> Iterable[tuple[_T, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None, - reducefunc: Callable[[Iterator[_T]], _W], -) -> Iterable[tuple[_U, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: None, - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[Iterable[_V]], _W], -) -> Iterable[tuple[_T, _W]]: ... -@overload -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[Iterable[_V]], _W], -) -> Iterable[tuple[_U, _W]]: ... - -class numeric_range(Generic[_T, _U], Sequence[_T], Hashable, Reversible[_T]): - @overload - def __init__(self, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T, __step: _U) -> None: ... - def __bool__(self) -> bool: ... - def __contains__(self, elem: object) -> bool: ... - def __eq__(self, other: object) -> bool: ... - @overload - def __getitem__(self, key: int) -> _T: ... - @overload - def __getitem__(self, key: slice) -> numeric_range[_T, _U]: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[_T]: ... - def __len__(self) -> int: ... - def __reduce__( - self, - ) -> tuple[Type[numeric_range[_T, _U]], tuple[_T, _T, _U]]: ... - def __repr__(self) -> str: ... - def __reversed__(self) -> Iterator[_T]: ... - def count(self, value: _T) -> int: ... - def index(self, value: _T) -> int: ... # type: ignore - -def count_cycle( - iterable: Iterable[_T], n: int | None = ... -) -> Iterable[tuple[int, _T]]: ... -def mark_ends( - iterable: Iterable[_T], -) -> Iterable[tuple[bool, bool, _T]]: ... -def locate( - iterable: Iterable[_T], - pred: Callable[..., Any] = ..., - window_size: int | None = ..., -) -> Iterator[int]: ... -def lstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def rstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def strip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... - -class islice_extended(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T], *args: int | None) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - def __getitem__(self, index: slice) -> islice_extended[_T]: ... - -def always_reversible(iterable: Iterable[_T]) -> Iterator[_T]: ... -def consecutive_groups( - iterable: Iterable[_T], ordering: Callable[[_T], int] = ... -) -> Iterator[Iterator[_T]]: ... -@overload -def difference( - iterable: Iterable[_T], - func: Callable[[_T, _T], _U] = ..., - *, - initial: None = ..., -) -> Iterator[_T | _U]: ... -@overload -def difference( - iterable: Iterable[_T], func: Callable[[_T, _T], _U] = ..., *, initial: _U -) -> Iterator[_U]: ... - -class SequenceView(Generic[_T], Sequence[_T]): - def __init__(self, target: Sequence[_T]) -> None: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> Sequence[_T]: ... - def __len__(self) -> int: ... - -class seekable(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], maxlen: int | None = ... - ) -> None: ... - def __iter__(self) -> seekable[_T]: ... - def __next__(self) -> _T: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> _T | _U: ... - def elements(self) -> SequenceView[_T]: ... - def seek(self, index: int) -> None: ... - def relative_seek(self, count: int) -> None: ... - -class run_length: - @staticmethod - def encode(iterable: Iterable[_T]) -> Iterator[tuple[_T, int]]: ... - @staticmethod - def decode(iterable: Iterable[tuple[_T, int]]) -> Iterator[_T]: ... - -def exactly_n( - iterable: Iterable[_T], n: int, predicate: Callable[[_T], object] = ... -) -> bool: ... -def circular_shifts(iterable: Iterable[_T]) -> list[tuple[_T, ...]]: ... -def make_decorator( - wrapping_func: Callable[..., _U], result_index: int = ... -) -> Callable[..., Callable[[Callable[..., Any]], Callable[..., _U]]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: None = ..., -) -> dict[_U, list[_T]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None = ..., -) -> dict[_U, list[_V]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: Callable[[list[_T]], _W] = ..., -) -> dict[_U, _W]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[list[_V]], _W], -) -> dict[_U, _W]: ... -def rlocate( - iterable: Iterable[_T], - pred: Callable[..., object] = ..., - window_size: int | None = ..., -) -> Iterator[int]: ... -def replace( - iterable: Iterable[_T], - pred: Callable[..., object], - substitutes: Iterable[_U], - count: int | None = ..., - window_size: int = ..., -) -> Iterator[_T | _U]: ... -def partitions(iterable: Iterable[_T]) -> Iterator[list[list[_T]]]: ... -def set_partitions( - iterable: Iterable[_T], k: int | None = ... -) -> Iterator[list[list[_T]]]: ... - -class time_limited(Generic[_T], Iterator[_T]): - def __init__( - self, limit_seconds: float, iterable: Iterable[_T] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - -@overload -def only( - iterable: Iterable[_T], *, too_long: _Raisable | None = ... -) -> _T | None: ... -@overload -def only( - iterable: Iterable[_T], default: _U, too_long: _Raisable | None = ... -) -> _T | _U: ... -def ichunked(iterable: Iterable[_T], n: int) -> Iterator[Iterator[_T]]: ... -def distinct_combinations( - iterable: Iterable[_T], r: int -) -> Iterator[tuple[_T, ...]]: ... -def filter_except( - validator: Callable[[Any], object], - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_T]: ... -def map_except( - function: Callable[[Any], _U], - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_U]: ... -def map_if( - iterable: Iterable[Any], - pred: Callable[[Any], bool], - func: Callable[[Any], Any], - func_else: Callable[[Any], Any] | None = ..., -) -> Iterator[Any]: ... -def sample( - iterable: Iterable[_T], - k: int, - weights: Iterable[float] | None = ..., -) -> list[_T]: ... -def is_sorted( - iterable: Iterable[_T], - key: Callable[[_T], _U] | None = ..., - reverse: bool = False, - strict: bool = False, -) -> bool: ... - -class AbortThread(BaseException): - pass - -class callback_iter(Generic[_T], Iterator[_T]): - def __init__( - self, - func: Callable[..., Any], - callback_kwd: str = ..., - wait_seconds: float = ..., - ) -> None: ... - def __enter__(self) -> callback_iter[_T]: ... - def __exit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool | None: ... - def __iter__(self) -> callback_iter[_T]: ... - def __next__(self) -> _T: ... - def _reader(self) -> Iterator[_T]: ... - @property - def done(self) -> bool: ... - @property - def result(self) -> Any: ... - -def windowed_complete( - iterable: Iterable[_T], n: int -) -> Iterator[tuple[_T, ...]]: ... -def all_unique( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> bool: ... -def nth_product(index: int, *args: Iterable[_T]) -> tuple[_T, ...]: ... -def nth_combination_with_replacement( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def nth_permutation( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def value_chain(*args: _T | Iterable[_T]) -> Iterable[_T]: ... -def product_index(element: Iterable[_T], *args: Iterable[_T]) -> int: ... -def combination_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def combination_with_replacement_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def permutation_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def repeat_each(iterable: Iterable[_T], n: int = ...) -> Iterator[_T]: ... - -class countable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> countable[_T]: ... - def __next__(self) -> _T: ... - -def chunked_even(iterable: Iterable[_T], n: int) -> Iterator[list[_T]]: ... -def zip_broadcast( - *objects: _T | Iterable[_T], - scalar_types: type | tuple[type | tuple[Any, ...], ...] | None = ..., - strict: bool = ..., -) -> Iterable[tuple[_T, ...]]: ... -def unique_in_window( - iterable: Iterable[_T], n: int, key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def duplicates_everseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def duplicates_justseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def classify_unique( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[tuple[_T, bool, bool]]: ... - -class _SupportsLessThan(Protocol): - def __lt__(self, __other: Any) -> bool: ... - -_SupportsLessThanT = TypeVar("_SupportsLessThanT", bound=_SupportsLessThan) - -@overload -def minmax( - iterable_or_value: Iterable[_SupportsLessThanT], *, key: None = None -) -> tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: Iterable[_T], *, key: Callable[[_T], _SupportsLessThan] -) -> tuple[_T, _T]: ... -@overload -def minmax( - iterable_or_value: Iterable[_SupportsLessThanT], - *, - key: None = None, - default: _U, -) -> _U | tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: Iterable[_T], - *, - key: Callable[[_T], _SupportsLessThan], - default: _U, -) -> _U | tuple[_T, _T]: ... -@overload -def minmax( - iterable_or_value: _SupportsLessThanT, - __other: _SupportsLessThanT, - *others: _SupportsLessThanT, -) -> tuple[_SupportsLessThanT, _SupportsLessThanT]: ... -@overload -def minmax( - iterable_or_value: _T, - __other: _T, - *others: _T, - key: Callable[[_T], _SupportsLessThan], -) -> tuple[_T, _T]: ... -def longest_common_prefix( - iterables: Iterable[Iterable[_T]], -) -> Iterator[_T]: ... -def iequals(*iterables: Iterable[Any]) -> bool: ... -def constrained_batches( - iterable: Iterable[_T], - max_size: int, - max_count: int | None = ..., - get_len: Callable[[_T], object] = ..., - strict: bool = ..., -) -> Iterator[tuple[_T]]: ... -def gray_product(*iterables: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def partial_product(*iterables: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def takewhile_inclusive( - predicate: Callable[[_T], bool], iterable: Iterable[_T] -) -> Iterator[_T]: ... -def outer_product( - func: Callable[[_T, _U], _V], - xs: Iterable[_T], - ys: Iterable[_U], - *args: Any, - **kwargs: Any, -) -> Iterator[tuple[_V, ...]]: ... -def iter_suppress( - iterable: Iterable[_T], - *exceptions: Type[BaseException], -) -> Iterator[_T]: ... -def filter_map( - func: Callable[[_T], _V | None], - iterable: Iterable[_T], -) -> Iterator[_V]: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py deleted file mode 100644 index 145e3cb5bd6..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.py +++ /dev/null @@ -1,1012 +0,0 @@ -"""Imported from the recipes section of the itertools documentation. - -All functions taken from the recipes section of the itertools library docs -[1]_. -Some backward-compatible usability improvements have been made. - -.. [1] http://docs.python.org/library/itertools.html#recipes - -""" -import math -import operator - -from collections import deque -from collections.abc import Sized -from functools import partial, reduce -from itertools import ( - chain, - combinations, - compress, - count, - cycle, - groupby, - islice, - product, - repeat, - starmap, - tee, - zip_longest, -) -from random import randrange, sample, choice -from sys import hexversion - -__all__ = [ - 'all_equal', - 'batched', - 'before_and_after', - 'consume', - 'convolve', - 'dotproduct', - 'first_true', - 'factor', - 'flatten', - 'grouper', - 'iter_except', - 'iter_index', - 'matmul', - 'ncycles', - 'nth', - 'nth_combination', - 'padnone', - 'pad_none', - 'pairwise', - 'partition', - 'polynomial_eval', - 'polynomial_from_roots', - 'polynomial_derivative', - 'powerset', - 'prepend', - 'quantify', - 'reshape', - 'random_combination_with_replacement', - 'random_combination', - 'random_permutation', - 'random_product', - 'repeatfunc', - 'roundrobin', - 'sieve', - 'sliding_window', - 'subslices', - 'sum_of_squares', - 'tabulate', - 'tail', - 'take', - 'totient', - 'transpose', - 'triplewise', - 'unique_everseen', - 'unique_justseen', -] - -_marker = object() - - -# zip with strict is available for Python 3.10+ -try: - zip(strict=True) -except TypeError: - _zip_strict = zip -else: - _zip_strict = partial(zip, strict=True) - -# math.sumprod is available for Python 3.12+ -_sumprod = getattr(math, 'sumprod', lambda x, y: dotproduct(x, y)) - - -def take(n, iterable): - """Return first *n* items of the iterable as a list. - - >>> take(3, range(10)) - [0, 1, 2] - - If there are fewer than *n* items in the iterable, all of them are - returned. - - >>> take(10, range(3)) - [0, 1, 2] - - """ - return list(islice(iterable, n)) - - -def tabulate(function, start=0): - """Return an iterator over the results of ``func(start)``, - ``func(start + 1)``, ``func(start + 2)``... - - *func* should be a function that accepts one integer argument. - - If *start* is not specified it defaults to 0. It will be incremented each - time the iterator is advanced. - - >>> square = lambda x: x ** 2 - >>> iterator = tabulate(square, -3) - >>> take(4, iterator) - [9, 4, 1, 0] - - """ - return map(function, count(start)) - - -def tail(n, iterable): - """Return an iterator over the last *n* items of *iterable*. - - >>> t = tail(3, 'ABCDEFG') - >>> list(t) - ['E', 'F', 'G'] - - """ - # If the given iterable has a length, then we can use islice to get its - # final elements. Note that if the iterable is not actually Iterable, - # either islice or deque will throw a TypeError. This is why we don't - # check if it is Iterable. - if isinstance(iterable, Sized): - yield from islice(iterable, max(0, len(iterable) - n), None) - else: - yield from iter(deque(iterable, maxlen=n)) - - -def consume(iterator, n=None): - """Advance *iterable* by *n* steps. If *n* is ``None``, consume it - entirely. - - Efficiently exhausts an iterator without returning values. Defaults to - consuming the whole iterator, but an optional second argument may be - provided to limit consumption. - - >>> i = (x for x in range(10)) - >>> next(i) - 0 - >>> consume(i, 3) - >>> next(i) - 4 - >>> consume(i) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - If the iterator has fewer items remaining than the provided limit, the - whole iterator will be consumed. - - >>> i = (x for x in range(3)) - >>> consume(i, 5) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - """ - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - -def nth(iterable, n, default=None): - """Returns the nth item or a default value. - - >>> l = range(10) - >>> nth(l, 3) - 3 - >>> nth(l, 20, "zebra") - 'zebra' - - """ - return next(islice(iterable, n, None), default) - - -def all_equal(iterable): - """ - Returns ``True`` if all the elements are equal to each other. - - >>> all_equal('aaaa') - True - >>> all_equal('aaab') - False - - """ - g = groupby(iterable) - return next(g, True) and not next(g, False) - - -def quantify(iterable, pred=bool): - """Return the how many times the predicate is true. - - >>> quantify([True, False, True]) - 2 - - """ - return sum(map(pred, iterable)) - - -def pad_none(iterable): - """Returns the sequence of elements and then returns ``None`` indefinitely. - - >>> take(5, pad_none(range(3))) - [0, 1, 2, None, None] - - Useful for emulating the behavior of the built-in :func:`map` function. - - See also :func:`padded`. - - """ - return chain(iterable, repeat(None)) - - -padnone = pad_none - - -def ncycles(iterable, n): - """Returns the sequence elements *n* times - - >>> list(ncycles(["a", "b"], 3)) - ['a', 'b', 'a', 'b', 'a', 'b'] - - """ - return chain.from_iterable(repeat(tuple(iterable), n)) - - -def dotproduct(vec1, vec2): - """Returns the dot product of the two iterables. - - >>> dotproduct([10, 10], [20, 20]) - 400 - - """ - return sum(map(operator.mul, vec1, vec2)) - - -def flatten(listOfLists): - """Return an iterator flattening one level of nesting in a list of lists. - - >>> list(flatten([[0, 1], [2, 3]])) - [0, 1, 2, 3] - - See also :func:`collapse`, which can flatten multiple levels of nesting. - - """ - return chain.from_iterable(listOfLists) - - -def repeatfunc(func, times=None, *args): - """Call *func* with *args* repeatedly, returning an iterable over the - results. - - If *times* is specified, the iterable will terminate after that many - repetitions: - - >>> from operator import add - >>> times = 4 - >>> args = 3, 5 - >>> list(repeatfunc(add, times, *args)) - [8, 8, 8, 8] - - If *times* is ``None`` the iterable will not terminate: - - >>> from random import randrange - >>> times = None - >>> args = 1, 11 - >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP - [2, 4, 8, 1, 8, 4] - - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - -def _pairwise(iterable): - """Returns an iterator of paired items, overlapping, from the original - - >>> take(4, pairwise(count())) - [(0, 1), (1, 2), (2, 3), (3, 4)] - - On Python 3.10 and above, this is an alias for :func:`itertools.pairwise`. - - """ - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - -try: - from itertools import pairwise as itertools_pairwise -except ImportError: - pairwise = _pairwise -else: - - def pairwise(iterable): - return itertools_pairwise(iterable) - - pairwise.__doc__ = _pairwise.__doc__ - - -class UnequalIterablesError(ValueError): - def __init__(self, details=None): - msg = 'Iterables have different lengths' - if details is not None: - msg += (': index 0 has length {}; index {} has length {}').format( - *details - ) - - super().__init__(msg) - - -def _zip_equal_generator(iterables): - for combo in zip_longest(*iterables, fillvalue=_marker): - for val in combo: - if val is _marker: - raise UnequalIterablesError() - yield combo - - -def _zip_equal(*iterables): - # Check whether the iterables are all the same size. - try: - first_size = len(iterables[0]) - for i, it in enumerate(iterables[1:], 1): - size = len(it) - if size != first_size: - raise UnequalIterablesError(details=(first_size, i, size)) - # All sizes are equal, we can use the built-in zip. - return zip(*iterables) - # If any one of the iterables didn't have a length, start reading - # them until one runs out. - except TypeError: - return _zip_equal_generator(iterables) - - -def grouper(iterable, n, incomplete='fill', fillvalue=None): - """Group elements from *iterable* into fixed-length groups of length *n*. - - >>> list(grouper('ABCDEF', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F')] - - The keyword arguments *incomplete* and *fillvalue* control what happens for - iterables whose length is not a multiple of *n*. - - When *incomplete* is `'fill'`, the last group will contain instances of - *fillvalue*. - - >>> list(grouper('ABCDEFG', 3, incomplete='fill', fillvalue='x')) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] - - When *incomplete* is `'ignore'`, the last group will not be emitted. - - >>> list(grouper('ABCDEFG', 3, incomplete='ignore', fillvalue='x')) - [('A', 'B', 'C'), ('D', 'E', 'F')] - - When *incomplete* is `'strict'`, a subclass of `ValueError` will be raised. - - >>> it = grouper('ABCDEFG', 3, incomplete='strict') - >>> list(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - UnequalIterablesError - - """ - args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': - return _zip_equal(*args) - if incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') - - -def roundrobin(*iterables): - """Yields an item from each iterable, alternating between them. - - >>> list(roundrobin('ABC', 'D', 'EF')) - ['A', 'D', 'E', 'B', 'F', 'C'] - - This function produces the same output as :func:`interleave_longest`, but - may perform better for some inputs (in particular when the number of - iterables is small). - - """ - # Recipe credited to George Sakkis - pending = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while pending: - try: - for next in nexts: - yield next() - except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) - - -def partition(pred, iterable): - """ - Returns a 2-tuple of iterables derived from the input iterable. - The first yields the items that have ``pred(item) == False``. - The second yields the items that have ``pred(item) == True``. - - >>> is_odd = lambda x: x % 2 != 0 - >>> iterable = range(10) - >>> even_items, odd_items = partition(is_odd, iterable) - >>> list(even_items), list(odd_items) - ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) - - If *pred* is None, :func:`bool` is used. - - >>> iterable = [0, 1, False, True, '', ' '] - >>> false_items, true_items = partition(None, iterable) - >>> list(false_items), list(true_items) - ([0, False, ''], [1, True, ' ']) - - """ - if pred is None: - pred = bool - - t1, t2, p = tee(iterable, 3) - p1, p2 = tee(map(pred, p)) - return (compress(t1, map(operator.not_, p1)), compress(t2, p2)) - - -def powerset(iterable): - """Yields all possible subsets of the iterable. - - >>> list(powerset([1, 2, 3])) - [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - - :func:`powerset` will operate on iterables that aren't :class:`set` - instances, so repeated elements in the input will produce repeated elements - in the output. Use :func:`unique_everseen` on the input to avoid generating - duplicates: - - >>> seq = [1, 1, 0] - >>> list(powerset(seq)) - [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] - >>> from more_itertools import unique_everseen - >>> list(powerset(unique_everseen(seq))) - [(), (1,), (0,), (1, 0)] - - """ - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) - - -def unique_everseen(iterable, key=None): - """ - Yield unique elements, preserving order. - - >>> list(unique_everseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'D'] - - Sequences with a mix of hashable and unhashable items can be used. - The function will be slower (i.e., `O(n^2)`) for unhashable items. - - Remember that ``list`` objects are unhashable - you can use the *key* - parameter to transform the list to a tuple (which is hashable) to - avoid a slowdown. - - >>> iterable = ([1, 2], [2, 3], [1, 2]) - >>> list(unique_everseen(iterable)) # Slow - [[1, 2], [2, 3]] - >>> list(unique_everseen(iterable, key=tuple)) # Faster - [[1, 2], [2, 3]] - - Similarly, you may want to convert unhashable ``set`` objects with - ``key=frozenset``. For ``dict`` objects, - ``key=lambda x: frozenset(x.items())`` can be used. - - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seenset: - seenset_add(k) - yield element - except TypeError: - if k not in seenlist: - seenlist_add(k) - yield element - - -def unique_justseen(iterable, key=None): - """Yields elements in order, ignoring serial duplicates - - >>> list(unique_justseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'A', 'D'] - - """ - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - -def iter_except(func, exception, first=None): - """Yields results from a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel - to end the loop. - - >>> l = [0, 1, 2] - >>> list(iter_except(l.pop, IndexError)) - [2, 1, 0] - - Multiple exceptions can be specified as a stopping condition: - - >>> l = [1, 2, 3, '...', 4, 5, 6] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [7, 6, 5] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [4, 3, 2] - >>> list(iter_except(lambda: 1 + l.pop(), (IndexError, TypeError))) - [] - - """ - try: - if first is not None: - yield first() - while 1: - yield func() - except exception: - pass - - -def first_true(iterable, default=None, pred=None): - """ - Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item for which - ``pred(item) == True`` . - - >>> first_true(range(10)) - 1 - >>> first_true(range(10), pred=lambda x: x > 5) - 6 - >>> first_true(range(10), default='missing', pred=lambda x: x > 9) - 'missing' - - """ - return next(filter(pred, iterable), default) - - -def random_product(*args, repeat=1): - """Draw an item at random from each of the input iterables. - - >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP - ('c', 3, 'Z') - - If *repeat* is provided as a keyword argument, that many items will be - drawn from each iterable. - - >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP - ('a', 2, 'd', 3) - - This equivalent to taking a random selection from - ``itertools.product(*args, **kwarg)``. - - """ - pools = [tuple(pool) for pool in args] * repeat - return tuple(choice(pool) for pool in pools) - - -def random_permutation(iterable, r=None): - """Return a random *r* length permutation of the elements in *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length of - *iterable*. - - >>> random_permutation(range(5)) # doctest:+SKIP - (3, 4, 0, 1, 2) - - This equivalent to taking a random selection from - ``itertools.permutations(iterable, r)``. - - """ - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(sample(pool, r)) - - -def random_combination(iterable, r): - """Return a random *r* length subsequence of the elements in *iterable*. - - >>> random_combination(range(5), 3) # doctest:+SKIP - (2, 3, 4) - - This equivalent to taking a random selection from - ``itertools.combinations(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(sample(range(n), r)) - return tuple(pool[i] for i in indices) - - -def random_combination_with_replacement(iterable, r): - """Return a random *r* length subsequence of elements in *iterable*, - allowing individual elements to be repeated. - - >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP - (0, 0, 1, 2, 2) - - This equivalent to taking a random selection from - ``itertools.combinations_with_replacement(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(randrange(n) for i in range(r)) - return tuple(pool[i] for i in indices) - - -def nth_combination(iterable, r, index): - """Equivalent to ``list(combinations(iterable, r))[index]``. - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`nth_combination` computes the subsequence at - sort position *index* directly, without computing the previous - subsequences. - - >>> nth_combination(range(5), 3, 5) - (0, 3, 4) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = 1 - k = min(r, n - r) - for i in range(1, k + 1): - c = c * (n - k + i) // i - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - while r: - c, n, r = c * r // n, n - 1, r - 1 - while index >= c: - index -= c - c, n = c * (n - r) // n, n - 1 - result.append(pool[-1 - n]) - - return tuple(result) - - -def prepend(value, iterator): - """Yield *value*, followed by the elements in *iterator*. - - >>> value = '0' - >>> iterator = ['1', '2', '3'] - >>> list(prepend(value, iterator)) - ['0', '1', '2', '3'] - - To prepend multiple values, see :func:`itertools.chain` - or :func:`value_chain`. - - """ - return chain([value], iterator) - - -def convolve(signal, kernel): - """Convolve the iterable *signal* with the iterable *kernel*. - - >>> signal = (1, 2, 3, 4, 5) - >>> kernel = [3, 2, 1] - >>> list(convolve(signal, kernel)) - [3, 8, 14, 20, 26, 14, 5] - - Note: the input arguments are not interchangeable, as the *kernel* - is immediately consumed and stored. - - """ - # This implementation intentionally doesn't match the one in the itertools - # documentation. - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n - 1)): - window.append(x) - yield _sumprod(kernel, window) - - -def before_and_after(predicate, it): - """A variant of :func:`takewhile` that allows complete access to the - remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the first iterator must be fully consumed before the second - iterator can generate valid results. - """ - it = iter(it) - transition = [] - - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - - # Note: this is different from itertools recipes to allow nesting - # before_and_after remainders into before_and_after again. See tests - # for an example. - remainder_iterator = chain(transition, it) - - return true_iterator(), remainder_iterator - - -def triplewise(iterable): - """Return overlapping triplets from *iterable*. - - >>> list(triplewise('ABCDE')) - [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E')] - - """ - for (a, _), (b, c) in pairwise(pairwise(iterable)): - yield a, b, c - - -def sliding_window(iterable, n): - """Return a sliding window of width *n* over *iterable*. - - >>> list(sliding_window(range(6), 4)) - [(0, 1, 2, 3), (1, 2, 3, 4), (2, 3, 4, 5)] - - If *iterable* has fewer than *n* items, then nothing is yielded: - - >>> list(sliding_window(range(3), 4)) - [] - - For a variant with more features, see :func:`windowed`. - """ - it = iter(iterable) - window = deque(islice(it, n - 1), maxlen=n) - for x in it: - window.append(x) - yield tuple(window) - - -def subslices(iterable): - """Return all contiguous non-empty subslices of *iterable*. - - >>> list(subslices('ABC')) - [['A'], ['A', 'B'], ['A', 'B', 'C'], ['B'], ['B', 'C'], ['C']] - - This is similar to :func:`substrings`, but emits items in a different - order. - """ - seq = list(iterable) - slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) - - -def polynomial_from_roots(roots): - """Compute a polynomial's coefficients from its roots. - - >>> roots = [5, -4, 3] # (x - 5) * (x + 4) * (x - 3) - >>> polynomial_from_roots(roots) # x^3 - 4 * x^2 - 17 * x + 60 - [1, -4, -17, 60] - """ - factors = zip(repeat(1), map(operator.neg, roots)) - return list(reduce(convolve, factors, [1])) - - -def iter_index(iterable, value, start=0, stop=None): - """Yield the index of each place in *iterable* that *value* occurs, - beginning with index *start* and ending before index *stop*. - - See :func:`locate` for a more general means of finding the indexes - associated with particular values. - - >>> list(iter_index('AABCADEAF', 'A')) - [0, 1, 4, 7] - >>> list(iter_index('AABCADEAF', 'A', 1)) # start index is inclusive - [1, 4, 7] - >>> list(iter_index('AABCADEAF', 'A', 1, 7)) # stop index is not inclusive - [1, 4] - """ - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Slow path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i - else: - # Fast path for sequences - stop = len(iterable) if stop is None else stop - i = start - 1 - try: - while True: - yield (i := seq_index(value, i + 1, stop)) - except ValueError: - pass - - -def sieve(n): - """Yield the primes less than n. - - >>> list(sieve(30)) - [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] - """ - if n > 2: - yield 2 - start = 3 - data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p * p) - data[p * p : n : p + p] = bytes(len(range(p * p, n, p + p))) - start = p * p - yield from iter_index(data, 1, start) - - -def _batched(iterable, n, *, strict=False): - """Batch data into tuples of length *n*. If the number of items in - *iterable* is not divisible by *n*: - * The last batch will be shorter if *strict* is ``False``. - * :exc:`ValueError` will be raised if *strict* is ``True``. - - >>> list(batched('ABCDEFG', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] - - On Python 3.13 and above, this is an alias for :func:`itertools.batched`. - """ - if n < 1: - raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): - if strict and len(batch) != n: - raise ValueError('batched(): incomplete batch') - yield batch - - -if hexversion >= 0x30D00A2: - from itertools import batched as itertools_batched - - def batched(iterable, n, *, strict=False): - return itertools_batched(iterable, n, strict=strict) - -else: - batched = _batched - - batched.__doc__ = _batched.__doc__ - - -def transpose(it): - """Swap the rows and columns of the input matrix. - - >>> list(transpose([(1, 2, 3), (11, 22, 33)])) - [(1, 11), (2, 22), (3, 33)] - - The caller should ensure that the dimensions of the input are compatible. - If the input is empty, no output will be produced. - """ - return _zip_strict(*it) - - -def reshape(matrix, cols): - """Reshape the 2-D input *matrix* to have a column count given by *cols*. - - >>> matrix = [(0, 1), (2, 3), (4, 5)] - >>> cols = 3 - >>> list(reshape(matrix, cols)) - [(0, 1, 2), (3, 4, 5)] - """ - return batched(chain.from_iterable(matrix), cols) - - -def matmul(m1, m2): - """Multiply two matrices. - - >>> list(matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)])) - [(49, 80), (41, 60)] - - The caller should ensure that the dimensions of the input matrices are - compatible with each other. - """ - n = len(m2[0]) - return batched(starmap(_sumprod, product(m1, transpose(m2))), n) - - -def factor(n): - """Yield the prime factors of n. - - >>> list(factor(360)) - [2, 2, 2, 3, 3, 5] - """ - for prime in sieve(math.isqrt(n) + 1): - while not n % prime: - yield prime - n //= prime - if n == 1: - return - if n > 1: - yield n - - -def polynomial_eval(coefficients, x): - """Evaluate a polynomial at a specific value. - - Example: evaluating x^3 - 4 * x^2 - 17 * x + 60 at x = 2.5: - - >>> coefficients = [1, -4, -17, 60] - >>> x = 2.5 - >>> polynomial_eval(coefficients, x) - 8.125 - """ - n = len(coefficients) - if n == 0: - return x * 0 # coerce zero to the type of x - powers = map(pow, repeat(x), reversed(range(n))) - return _sumprod(coefficients, powers) - - -def sum_of_squares(it): - """Return the sum of the squares of the input values. - - >>> sum_of_squares([10, 20, 30]) - 1400 - """ - return _sumprod(*tee(it)) - - -def polynomial_derivative(coefficients): - """Compute the first derivative of a polynomial. - - Example: evaluating the derivative of x^3 - 4 * x^2 - 17 * x + 60 - - >>> coefficients = [1, -4, -17, 60] - >>> derivative_coefficients = polynomial_derivative(coefficients) - >>> derivative_coefficients - [3, -8, -17] - """ - n = len(coefficients) - powers = reversed(range(1, n)) - return list(map(operator.mul, coefficients, powers)) - - -def totient(n): - """Return the count of natural numbers up to *n* that are coprime with *n*. - - >>> totient(9) - 6 - >>> totient(12) - 4 - """ - for p in unique_justseen(factor(n)): - n = n // p * (p - 1) - - return n diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi b/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi deleted file mode 100644 index ed4c19db49b..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/more_itertools/recipes.pyi +++ /dev/null @@ -1,128 +0,0 @@ -"""Stubs for more_itertools.recipes""" -from __future__ import annotations - -from typing import ( - Any, - Callable, - Iterable, - Iterator, - overload, - Sequence, - Type, - TypeVar, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_T1 = TypeVar('_T1') -_T2 = TypeVar('_T2') -_U = TypeVar('_U') - -def take(n: int, iterable: Iterable[_T]) -> list[_T]: ... -def tabulate( - function: Callable[[int], _T], start: int = ... -) -> Iterator[_T]: ... -def tail(n: int, iterable: Iterable[_T]) -> Iterator[_T]: ... -def consume(iterator: Iterable[_T], n: int | None = ...) -> None: ... -@overload -def nth(iterable: Iterable[_T], n: int) -> _T | None: ... -@overload -def nth(iterable: Iterable[_T], n: int, default: _U) -> _T | _U: ... -def all_equal(iterable: Iterable[_T]) -> bool: ... -def quantify( - iterable: Iterable[_T], pred: Callable[[_T], bool] = ... -) -> int: ... -def pad_none(iterable: Iterable[_T]) -> Iterator[_T | None]: ... -def padnone(iterable: Iterable[_T]) -> Iterator[_T | None]: ... -def ncycles(iterable: Iterable[_T], n: int) -> Iterator[_T]: ... -def dotproduct(vec1: Iterable[_T1], vec2: Iterable[_T2]) -> Any: ... -def flatten(listOfLists: Iterable[Iterable[_T]]) -> Iterator[_T]: ... -def repeatfunc( - func: Callable[..., _U], times: int | None = ..., *args: Any -) -> Iterator[_U]: ... -def pairwise(iterable: Iterable[_T]) -> Iterator[tuple[_T, _T]]: ... -def grouper( - iterable: Iterable[_T], - n: int, - incomplete: str = ..., - fillvalue: _U = ..., -) -> Iterator[tuple[_T | _U, ...]]: ... -def roundrobin(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def partition( - pred: Callable[[_T], object] | None, iterable: Iterable[_T] -) -> tuple[Iterator[_T], Iterator[_T]]: ... -def powerset(iterable: Iterable[_T]) -> Iterator[tuple[_T, ...]]: ... -def unique_everseen( - iterable: Iterable[_T], key: Callable[[_T], _U] | None = ... -) -> Iterator[_T]: ... -def unique_justseen( - iterable: Iterable[_T], key: Callable[[_T], object] | None = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException] | tuple[Type[BaseException], ...], - first: None = ..., -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException] | tuple[Type[BaseException], ...], - first: Callable[[], _U], -) -> Iterator[_T | _U]: ... -@overload -def first_true( - iterable: Iterable[_T], *, pred: Callable[[_T], object] | None = ... -) -> _T | None: ... -@overload -def first_true( - iterable: Iterable[_T], - default: _U, - pred: Callable[[_T], object] | None = ..., -) -> _T | _U: ... -def random_product( - *args: Iterable[_T], repeat: int = ... -) -> tuple[_T, ...]: ... -def random_permutation( - iterable: Iterable[_T], r: int | None = ... -) -> tuple[_T, ...]: ... -def random_combination(iterable: Iterable[_T], r: int) -> tuple[_T, ...]: ... -def random_combination_with_replacement( - iterable: Iterable[_T], r: int -) -> tuple[_T, ...]: ... -def nth_combination( - iterable: Iterable[_T], r: int, index: int -) -> tuple[_T, ...]: ... -def prepend(value: _T, iterator: Iterable[_U]) -> Iterator[_T | _U]: ... -def convolve(signal: Iterable[_T], kernel: Iterable[_T]) -> Iterator[_T]: ... -def before_and_after( - predicate: Callable[[_T], bool], it: Iterable[_T] -) -> tuple[Iterator[_T], Iterator[_T]]: ... -def triplewise(iterable: Iterable[_T]) -> Iterator[tuple[_T, _T, _T]]: ... -def sliding_window( - iterable: Iterable[_T], n: int -) -> Iterator[tuple[_T, ...]]: ... -def subslices(iterable: Iterable[_T]) -> Iterator[list[_T]]: ... -def polynomial_from_roots(roots: Sequence[_T]) -> list[_T]: ... -def iter_index( - iterable: Iterable[_T], - value: Any, - start: int | None = ..., - stop: int | None = ..., -) -> Iterator[int]: ... -def sieve(n: int) -> Iterator[int]: ... -def batched( - iterable: Iterable[_T], n: int, *, strict: bool = False -) -> Iterator[tuple[_T]]: ... -def transpose( - it: Iterable[Iterable[_T]], -) -> Iterator[tuple[_T, ...]]: ... -def reshape( - matrix: Iterable[Iterable[_T]], cols: int -) -> Iterator[tuple[_T, ...]]: ... -def matmul(m1: Sequence[_T], m2: Sequence[_T]) -> Iterator[tuple[_T]]: ... -def factor(n: int) -> Iterator[int]: ... -def polynomial_eval(coefficients: Sequence[_T], x: _U) -> _U: ... -def sum_of_squares(it: Iterable[_T]) -> _T: ... -def polynomial_derivative(coefficients: Sequence[_T]) -> list[_T]: ... -def totient(n: int) -> int: ... diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py deleted file mode 100644 index e7c0aa12ca9..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -__title__ = "packaging" -__summary__ = "Core utilities for Python packages" -__uri__ = "https://github.com/pypa/packaging" - -__version__ = "24.0" - -__author__ = "Donald Stufft and individual contributors" -__email__ = "donald@stufft.io" - -__license__ = "BSD-2-Clause or Apache-2.0" -__copyright__ = "2014 %s" % __author__ diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py deleted file mode 100644 index ad62505f3ff..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_manylinux.py +++ /dev/null @@ -1,260 +0,0 @@ -import collections -import contextlib -import functools -import os -import re -import sys -import warnings -from typing import Dict, Generator, Iterator, NamedTuple, Optional, Sequence, Tuple - -from ._elffile import EIClass, EIData, ELFFile, EMachine - -EF_ARM_ABIMASK = 0xFF000000 -EF_ARM_ABI_VER5 = 0x05000000 -EF_ARM_ABI_FLOAT_HARD = 0x00000400 - - -# `os.PathLike` not a generic type until Python 3.9, so sticking with `str` -# as the type for `path` until then. -@contextlib.contextmanager -def _parse_elf(path: str) -> Generator[Optional[ELFFile], None, None]: - try: - with open(path, "rb") as f: - yield ELFFile(f) - except (OSError, TypeError, ValueError): - yield None - - -def _is_linux_armhf(executable: str) -> bool: - # hard-float ABI can be detected from the ELF header of the running - # process - # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.Arm - and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5 - and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD - ) - - -def _is_linux_i686(executable: str) -> bool: - with _parse_elf(executable) as f: - return ( - f is not None - and f.capacity == EIClass.C32 - and f.encoding == EIData.Lsb - and f.machine == EMachine.I386 - ) - - -def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: - if "armv7l" in archs: - return _is_linux_armhf(executable) - if "i686" in archs: - return _is_linux_i686(executable) - allowed_archs = { - "x86_64", - "aarch64", - "ppc64", - "ppc64le", - "s390x", - "loongarch64", - "riscv64", - } - return any(arch in allowed_archs for arch in archs) - - -# If glibc ever changes its major version, we need to know what the last -# minor version was, so we can build the complete list of all versions. -# For now, guess what the highest minor version might be, assume it will -# be 50 for testing. Once this actually happens, update the dictionary -# with the actual value. -_LAST_GLIBC_MINOR: Dict[int, int] = collections.defaultdict(lambda: 50) - - -class _GLibCVersion(NamedTuple): - major: int - minor: int - - -def _glibc_version_string_confstr() -> Optional[str]: - """ - Primary implementation of glibc_version_string using os.confstr. - """ - # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely - # to be broken or missing. This strategy is used in the standard library - # platform module. - # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183 - try: - # Should be a string like "glibc 2.17". - version_string: Optional[str] = os.confstr("CS_GNU_LIBC_VERSION") - assert version_string is not None - _, version = version_string.rsplit() - except (AssertionError, AttributeError, OSError, ValueError): - # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... - return None - return version - - -def _glibc_version_string_ctypes() -> Optional[str]: - """ - Fallback implementation of glibc_version_string using ctypes. - """ - try: - import ctypes - except ImportError: - return None - - # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen - # manpage says, "If filename is NULL, then the returned handle is for the - # main program". This way we can let the linker do the work to figure out - # which libc our process is actually using. - # - # We must also handle the special case where the executable is not a - # dynamically linked executable. This can occur when using musl libc, - # for example. In this situation, dlopen() will error, leading to an - # OSError. Interestingly, at least in the case of musl, there is no - # errno set on the OSError. The single string argument used to construct - # OSError comes from libc itself and is therefore not portable to - # hard code here. In any case, failure to call dlopen() means we - # can proceed, so we bail on our attempt. - try: - process_namespace = ctypes.CDLL(None) - except OSError: - return None - - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return None - - # Call gnu_get_libc_version, which returns a string like "2.5" - gnu_get_libc_version.restype = ctypes.c_char_p - version_str: str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") - - return version_str - - -def _glibc_version_string() -> Optional[str]: - """Returns glibc version string, or None if not using glibc.""" - return _glibc_version_string_confstr() or _glibc_version_string_ctypes() - - -def _parse_glibc_version(version_str: str) -> Tuple[int, int]: - """Parse glibc version. - - We use a regexp instead of str.split because we want to discard any - random junk that might come after the minor version -- this might happen - in patched/forked versions of glibc (e.g. Linaro's version of glibc - uses version strings like "2.20-2014.11"). See gh-3588. - """ - m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) - if not m: - warnings.warn( - f"Expected glibc version with 2 components major.minor," - f" got: {version_str}", - RuntimeWarning, - ) - return -1, -1 - return int(m.group("major")), int(m.group("minor")) - - -@functools.lru_cache() -def _get_glibc_version() -> Tuple[int, int]: - version_str = _glibc_version_string() - if version_str is None: - return (-1, -1) - return _parse_glibc_version(version_str) - - -# From PEP 513, PEP 600 -def _is_compatible(arch: str, version: _GLibCVersion) -> bool: - sys_glibc = _get_glibc_version() - if sys_glibc < version: - return False - # Check for presence of _manylinux module. - try: - import _manylinux - except ImportError: - return True - if hasattr(_manylinux, "manylinux_compatible"): - result = _manylinux.manylinux_compatible(version[0], version[1], arch) - if result is not None: - return bool(result) - return True - if version == _GLibCVersion(2, 5): - if hasattr(_manylinux, "manylinux1_compatible"): - return bool(_manylinux.manylinux1_compatible) - if version == _GLibCVersion(2, 12): - if hasattr(_manylinux, "manylinux2010_compatible"): - return bool(_manylinux.manylinux2010_compatible) - if version == _GLibCVersion(2, 17): - if hasattr(_manylinux, "manylinux2014_compatible"): - return bool(_manylinux.manylinux2014_compatible) - return True - - -_LEGACY_MANYLINUX_MAP = { - # CentOS 7 w/ glibc 2.17 (PEP 599) - (2, 17): "manylinux2014", - # CentOS 6 w/ glibc 2.12 (PEP 571) - (2, 12): "manylinux2010", - # CentOS 5 w/ glibc 2.5 (PEP 513) - (2, 5): "manylinux1", -} - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate manylinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be manylinux-compatible. - - :returns: An iterator of compatible manylinux tags. - """ - if not _have_compatible_abi(sys.executable, archs): - return - # Oldest glibc to be supported regardless of architecture is (2, 17). - too_old_glibc2 = _GLibCVersion(2, 16) - if set(archs) & {"x86_64", "i686"}: - # On x86/i686 also oldest glibc to be supported is (2, 5). - too_old_glibc2 = _GLibCVersion(2, 4) - current_glibc = _GLibCVersion(*_get_glibc_version()) - glibc_max_list = [current_glibc] - # We can assume compatibility across glibc major versions. - # https://sourceware.org/bugzilla/show_bug.cgi?id=24636 - # - # Build a list of maximum glibc versions so that we can - # output the canonical list of all glibc from current_glibc - # down to too_old_glibc2, including all intermediary versions. - for glibc_major in range(current_glibc.major - 1, 1, -1): - glibc_minor = _LAST_GLIBC_MINOR[glibc_major] - glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor)) - for arch in archs: - for glibc_max in glibc_max_list: - if glibc_max.major == too_old_glibc2.major: - min_minor = too_old_glibc2.minor - else: - # For other glibc major versions oldest supported is (x, 0). - min_minor = -1 - for glibc_minor in range(glibc_max.minor, min_minor, -1): - glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) - tag = "manylinux_{}_{}".format(*glibc_version) - if _is_compatible(arch, glibc_version): - yield f"{tag}_{arch}" - # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. - if glibc_version in _LEGACY_MANYLINUX_MAP: - legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] - if _is_compatible(arch, glibc_version): - yield f"{legacy_tag}_{arch}" diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py deleted file mode 100644 index 684df75457c..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_parser.py +++ /dev/null @@ -1,356 +0,0 @@ -"""Handwritten parser of dependency specifiers. - -The docstring for each __parse_* function contains ENBF-inspired grammar representing -the implementation. -""" - -import ast -from typing import Any, List, NamedTuple, Optional, Tuple, Union - -from ._tokenizer import DEFAULT_RULES, Tokenizer - - -class Node: - def __init__(self, value: str) -> None: - self.value = value - - def __str__(self) -> str: - return self.value - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" - - def serialize(self) -> str: - raise NotImplementedError - - -class Variable(Node): - def serialize(self) -> str: - return str(self) - - -class Value(Node): - def serialize(self) -> str: - return f'"{self}"' - - -class Op(Node): - def serialize(self) -> str: - return str(self) - - -MarkerVar = Union[Variable, Value] -MarkerItem = Tuple[MarkerVar, Op, MarkerVar] -# MarkerAtom = Union[MarkerItem, List["MarkerAtom"]] -# MarkerList = List[Union["MarkerList", MarkerAtom, str]] -# mypy does not support recursive type definition -# https://github.com/python/mypy/issues/731 -MarkerAtom = Any -MarkerList = List[Any] - - -class ParsedRequirement(NamedTuple): - name: str - url: str - extras: List[str] - specifier: str - marker: Optional[MarkerList] - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for dependency specifier -# -------------------------------------------------------------------------------------- -def parse_requirement(source: str) -> ParsedRequirement: - return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: - """ - requirement = WS? IDENTIFIER WS? extras WS? requirement_details - """ - tokenizer.consume("WS") - - name_token = tokenizer.expect( - "IDENTIFIER", expected="package name at the start of dependency specifier" - ) - name = name_token.text - tokenizer.consume("WS") - - extras = _parse_extras(tokenizer) - tokenizer.consume("WS") - - url, specifier, marker = _parse_requirement_details(tokenizer) - tokenizer.expect("END", expected="end of dependency specifier") - - return ParsedRequirement(name, url, extras, specifier, marker) - - -def _parse_requirement_details( - tokenizer: Tokenizer, -) -> Tuple[str, str, Optional[MarkerList]]: - """ - requirement_details = AT URL (WS requirement_marker?)? - | specifier WS? (requirement_marker)? - """ - - specifier = "" - url = "" - marker = None - - if tokenizer.check("AT"): - tokenizer.read() - tokenizer.consume("WS") - - url_start = tokenizer.position - url = tokenizer.expect("URL", expected="URL after @").text - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - tokenizer.expect("WS", expected="whitespace after URL") - - # The input might end after whitespace. - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, span_start=url_start, after="URL and whitespace" - ) - else: - specifier_start = tokenizer.position - specifier = _parse_specifier(tokenizer) - tokenizer.consume("WS") - - if tokenizer.check("END", peek=True): - return (url, specifier, marker) - - marker = _parse_requirement_marker( - tokenizer, - span_start=specifier_start, - after=( - "version specifier" - if specifier - else "name and no valid version specifier" - ), - ) - - return (url, specifier, marker) - - -def _parse_requirement_marker( - tokenizer: Tokenizer, *, span_start: int, after: str -) -> MarkerList: - """ - requirement_marker = SEMICOLON marker WS? - """ - - if not tokenizer.check("SEMICOLON"): - tokenizer.raise_syntax_error( - f"Expected end or semicolon (after {after})", - span_start=span_start, - ) - tokenizer.read() - - marker = _parse_marker(tokenizer) - tokenizer.consume("WS") - - return marker - - -def _parse_extras(tokenizer: Tokenizer) -> List[str]: - """ - extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? - """ - if not tokenizer.check("LEFT_BRACKET", peek=True): - return [] - - with tokenizer.enclosing_tokens( - "LEFT_BRACKET", - "RIGHT_BRACKET", - around="extras", - ): - tokenizer.consume("WS") - extras = _parse_extras_list(tokenizer) - tokenizer.consume("WS") - - return extras - - -def _parse_extras_list(tokenizer: Tokenizer) -> List[str]: - """ - extras_list = identifier (wsp* ',' wsp* identifier)* - """ - extras: List[str] = [] - - if not tokenizer.check("IDENTIFIER"): - return extras - - extras.append(tokenizer.read().text) - - while True: - tokenizer.consume("WS") - if tokenizer.check("IDENTIFIER", peek=True): - tokenizer.raise_syntax_error("Expected comma between extra names") - elif not tokenizer.check("COMMA"): - break - - tokenizer.read() - tokenizer.consume("WS") - - extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") - extras.append(extra_token.text) - - return extras - - -def _parse_specifier(tokenizer: Tokenizer) -> str: - """ - specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS - | WS? version_many WS? - """ - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="version specifier", - ): - tokenizer.consume("WS") - parsed_specifiers = _parse_version_many(tokenizer) - tokenizer.consume("WS") - - return parsed_specifiers - - -def _parse_version_many(tokenizer: Tokenizer) -> str: - """ - version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? - """ - parsed_specifiers = "" - while tokenizer.check("SPECIFIER"): - span_start = tokenizer.position - parsed_specifiers += tokenizer.read().text - if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): - tokenizer.raise_syntax_error( - ".* suffix can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position + 1, - ) - if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): - tokenizer.raise_syntax_error( - "Local version label can only be used with `==` or `!=` operators", - span_start=span_start, - span_end=tokenizer.position, - ) - tokenizer.consume("WS") - if not tokenizer.check("COMMA"): - break - parsed_specifiers += tokenizer.read().text - tokenizer.consume("WS") - - return parsed_specifiers - - -# -------------------------------------------------------------------------------------- -# Recursive descent parser for marker expression -# -------------------------------------------------------------------------------------- -def parse_marker(source: str) -> MarkerList: - return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES)) - - -def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList: - retval = _parse_marker(tokenizer) - tokenizer.expect("END", expected="end of marker expression") - return retval - - -def _parse_marker(tokenizer: Tokenizer) -> MarkerList: - """ - marker = marker_atom (BOOLOP marker_atom)+ - """ - expression = [_parse_marker_atom(tokenizer)] - while tokenizer.check("BOOLOP"): - token = tokenizer.read() - expr_right = _parse_marker_atom(tokenizer) - expression.extend((token.text, expr_right)) - return expression - - -def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: - """ - marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? - | WS? marker_item WS? - """ - - tokenizer.consume("WS") - if tokenizer.check("LEFT_PARENTHESIS", peek=True): - with tokenizer.enclosing_tokens( - "LEFT_PARENTHESIS", - "RIGHT_PARENTHESIS", - around="marker expression", - ): - tokenizer.consume("WS") - marker: MarkerAtom = _parse_marker(tokenizer) - tokenizer.consume("WS") - else: - marker = _parse_marker_item(tokenizer) - tokenizer.consume("WS") - return marker - - -def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: - """ - marker_item = WS? marker_var WS? marker_op WS? marker_var WS? - """ - tokenizer.consume("WS") - marker_var_left = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - marker_op = _parse_marker_op(tokenizer) - tokenizer.consume("WS") - marker_var_right = _parse_marker_var(tokenizer) - tokenizer.consume("WS") - return (marker_var_left, marker_op, marker_var_right) - - -def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: - """ - marker_var = VARIABLE | QUOTED_STRING - """ - if tokenizer.check("VARIABLE"): - return process_env_var(tokenizer.read().text.replace(".", "_")) - elif tokenizer.check("QUOTED_STRING"): - return process_python_str(tokenizer.read().text) - else: - tokenizer.raise_syntax_error( - message="Expected a marker variable or quoted string" - ) - - -def process_env_var(env_var: str) -> Variable: - if env_var in ("platform_python_implementation", "python_implementation"): - return Variable("platform_python_implementation") - else: - return Variable(env_var) - - -def process_python_str(python_str: str) -> Value: - value = ast.literal_eval(python_str) - return Value(str(value)) - - -def _parse_marker_op(tokenizer: Tokenizer) -> Op: - """ - marker_op = IN | NOT IN | OP - """ - if tokenizer.check("IN"): - tokenizer.read() - return Op("in") - elif tokenizer.check("NOT"): - tokenizer.read() - tokenizer.expect("WS", expected="whitespace after 'not'") - tokenizer.expect("IN", expected="'in' after 'not'") - return Op("not in") - elif tokenizer.check("OP"): - return Op(tokenizer.read().text) - else: - return tokenizer.raise_syntax_error( - "Expected marker operator, one of " - "<=, <, !=, ==, >=, >, ~=, ===, in, not in" - ) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py deleted file mode 100644 index 8b98fca7233..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/markers.py +++ /dev/null @@ -1,252 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import operator -import os -import platform -import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -from ._parser import ( - MarkerAtom, - MarkerList, - Op, - Value, - Variable, - parse_marker as _parse_marker, -) -from ._tokenizer import ParserSyntaxError -from .specifiers import InvalidSpecifier, Specifier -from .utils import canonicalize_name - -__all__ = [ - "InvalidMarker", - "UndefinedComparison", - "UndefinedEnvironmentName", - "Marker", - "default_environment", -] - -Operator = Callable[[str, str], bool] - - -class InvalidMarker(ValueError): - """ - An invalid marker was found, users should refer to PEP 508. - """ - - -class UndefinedComparison(ValueError): - """ - An invalid operation was attempted on a value that doesn't support it. - """ - - -class UndefinedEnvironmentName(ValueError): - """ - A name was attempted to be used that does not exist inside of the - environment. - """ - - -def _normalize_extra_values(results: Any) -> Any: - """ - Normalize extra values. - """ - if isinstance(results[0], tuple): - lhs, op, rhs = results[0] - if isinstance(lhs, Variable) and lhs.value == "extra": - normalized_extra = canonicalize_name(rhs.value) - rhs = Value(normalized_extra) - elif isinstance(rhs, Variable) and rhs.value == "extra": - normalized_extra = canonicalize_name(lhs.value) - lhs = Value(normalized_extra) - results[0] = lhs, op, rhs - return results - - -def _format_marker( - marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True -) -> str: - - assert isinstance(marker, (list, tuple, str)) - - # Sometimes we have a structure like [[...]] which is a single item list - # where the single item is itself it's own list. In that case we want skip - # the rest of this function so that we don't get extraneous () on the - # outside. - if ( - isinstance(marker, list) - and len(marker) == 1 - and isinstance(marker[0], (list, tuple)) - ): - return _format_marker(marker[0]) - - if isinstance(marker, list): - inner = (_format_marker(m, first=False) for m in marker) - if first: - return " ".join(inner) - else: - return "(" + " ".join(inner) + ")" - elif isinstance(marker, tuple): - return " ".join([m.serialize() for m in marker]) - else: - return marker - - -_operators: Dict[str, Operator] = { - "in": lambda lhs, rhs: lhs in rhs, - "not in": lambda lhs, rhs: lhs not in rhs, - "<": operator.lt, - "<=": operator.le, - "==": operator.eq, - "!=": operator.ne, - ">=": operator.ge, - ">": operator.gt, -} - - -def _eval_op(lhs: str, op: Op, rhs: str) -> bool: - try: - spec = Specifier("".join([op.serialize(), rhs])) - except InvalidSpecifier: - pass - else: - return spec.contains(lhs, prereleases=True) - - oper: Optional[Operator] = _operators.get(op.serialize()) - if oper is None: - raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.") - - return oper(lhs, rhs) - - -def _normalize(*values: str, key: str) -> Tuple[str, ...]: - # PEP 685 – Comparison of extra names for optional distribution dependencies - # https://peps.python.org/pep-0685/ - # > When comparing extra names, tools MUST normalize the names being - # > compared using the semantics outlined in PEP 503 for names - if key == "extra": - return tuple(canonicalize_name(v) for v in values) - - # other environment markers don't have such standards - return values - - -def _evaluate_markers(markers: MarkerList, environment: Dict[str, str]) -> bool: - groups: List[List[bool]] = [[]] - - for marker in markers: - assert isinstance(marker, (list, tuple, str)) - - if isinstance(marker, list): - groups[-1].append(_evaluate_markers(marker, environment)) - elif isinstance(marker, tuple): - lhs, op, rhs = marker - - if isinstance(lhs, Variable): - environment_key = lhs.value - lhs_value = environment[environment_key] - rhs_value = rhs.value - else: - lhs_value = lhs.value - environment_key = rhs.value - rhs_value = environment[environment_key] - - lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key) - groups[-1].append(_eval_op(lhs_value, op, rhs_value)) - else: - assert marker in ["and", "or"] - if marker == "or": - groups.append([]) - - return any(all(item) for item in groups) - - -def format_full_version(info: "sys._version_info") -> str: - version = "{0.major}.{0.minor}.{0.micro}".format(info) - kind = info.releaselevel - if kind != "final": - version += kind[0] + str(info.serial) - return version - - -def default_environment() -> Dict[str, str]: - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name - return { - "implementation_name": implementation_name, - "implementation_version": iver, - "os_name": os.name, - "platform_machine": platform.machine(), - "platform_release": platform.release(), - "platform_system": platform.system(), - "platform_version": platform.version(), - "python_full_version": platform.python_version(), - "platform_python_implementation": platform.python_implementation(), - "python_version": ".".join(platform.python_version_tuple()[:2]), - "sys_platform": sys.platform, - } - - -class Marker: - def __init__(self, marker: str) -> None: - # Note: We create a Marker object without calling this constructor in - # packaging.requirements.Requirement. If any additional logic is - # added here, make sure to mirror/adapt Requirement. - try: - self._markers = _normalize_extra_values(_parse_marker(marker)) - # The attribute `_markers` can be described in terms of a recursive type: - # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]] - # - # For example, the following expression: - # python_version > "3.6" or (python_version == "3.6" and os_name == "unix") - # - # is parsed into: - # [ - # (<Variable('python_version')>, <Op('>')>, <Value('3.6')>), - # 'and', - # [ - # (<Variable('python_version')>, <Op('==')>, <Value('3.6')>), - # 'or', - # (<Variable('os_name')>, <Op('==')>, <Value('unix')>) - # ] - # ] - except ParserSyntaxError as e: - raise InvalidMarker(str(e)) from e - - def __str__(self) -> str: - return _format_marker(self._markers) - - def __repr__(self) -> str: - return f"<Marker('{self}')>" - - def __hash__(self) -> int: - return hash((self.__class__.__name__, str(self))) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Marker): - return NotImplemented - - return str(self) == str(other) - - def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool: - """Evaluate a marker. - - Return the boolean from evaluating the given marker against the - environment. environment is an optional argument to override all or - part of the determined environment. - - The environment is determined from the current Python process. - """ - current_environment = default_environment() - current_environment["extra"] = "" - if environment is not None: - current_environment.update(environment) - # The API used to allow setting extra to None. We need to handle this - # case for backwards compatibility. - if current_environment["extra"] is None: - current_environment["extra"] = "" - - return _evaluate_markers(self._markers, current_environment) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py deleted file mode 100644 index fb274930799..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/metadata.py +++ /dev/null @@ -1,825 +0,0 @@ -import email.feedparser -import email.header -import email.message -import email.parser -import email.policy -import sys -import typing -from typing import ( - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - Union, - cast, -) - -from . import requirements, specifiers, utils, version as version_module - -T = typing.TypeVar("T") -if sys.version_info[:2] >= (3, 8): # pragma: no cover - from typing import Literal, TypedDict -else: # pragma: no cover - if typing.TYPE_CHECKING: - from typing_extensions import Literal, TypedDict - else: - try: - from typing_extensions import Literal, TypedDict - except ImportError: - - class Literal: - def __init_subclass__(*_args, **_kwargs): - pass - - class TypedDict: - def __init_subclass__(*_args, **_kwargs): - pass - - -try: - ExceptionGroup -except NameError: # pragma: no cover - - class ExceptionGroup(Exception): # noqa: N818 - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: List[Exception] - - def __init__(self, message: str, exceptions: List[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" - -else: # pragma: no cover - ExceptionGroup = ExceptionGroup - - -class InvalidMetadata(ValueError): - """A metadata field contains invalid data.""" - - field: str - """The name of the field that contains invalid data.""" - - def __init__(self, field: str, message: str) -> None: - self.field = field - super().__init__(message) - - -# The RawMetadata class attempts to make as few assumptions about the underlying -# serialization formats as possible. The idea is that as long as a serialization -# formats offer some very basic primitives in *some* way then we can support -# serializing to and from that format. -class RawMetadata(TypedDict, total=False): - """A dictionary of raw core metadata. - - Each field in core metadata maps to a key of this dictionary (when data is - provided). The key is lower-case and underscores are used instead of dashes - compared to the equivalent core metadata field. Any core metadata field that - can be specified multiple times or can hold multiple values in a single - field have a key with a plural name. See :class:`Metadata` whose attributes - match the keys of this dictionary. - - Core metadata fields that can be specified multiple times are stored as a - list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - - """ - - # Metadata 1.0 - PEP 241 - metadata_version: str - name: str - version: str - platforms: List[str] - summary: str - description: str - keywords: List[str] - home_page: str - author: str - author_email: str - license: str - - # Metadata 1.1 - PEP 314 - supported_platforms: List[str] - download_url: str - classifiers: List[str] - requires: List[str] - provides: List[str] - obsoletes: List[str] - - # Metadata 1.2 - PEP 345 - maintainer: str - maintainer_email: str - requires_dist: List[str] - provides_dist: List[str] - obsoletes_dist: List[str] - requires_python: str - requires_external: List[str] - project_urls: Dict[str, str] - - # Metadata 2.0 - # PEP 426 attempted to completely revamp the metadata format - # but got stuck without ever being able to build consensus on - # it and ultimately ended up withdrawn. - # - # However, a number of tools had started emitting METADATA with - # `2.0` Metadata-Version, so for historical reasons, this version - # was skipped. - - # Metadata 2.1 - PEP 566 - description_content_type: str - provides_extra: List[str] - - # Metadata 2.2 - PEP 643 - dynamic: List[str] - - # Metadata 2.3 - PEP 685 - # No new fields were added in PEP 685, just some edge case were - # tightened up to provide better interoptability. - - -_STRING_FIELDS = { - "author", - "author_email", - "description", - "description_content_type", - "download_url", - "home_page", - "license", - "maintainer", - "maintainer_email", - "metadata_version", - "name", - "requires_python", - "summary", - "version", -} - -_LIST_FIELDS = { - "classifiers", - "dynamic", - "obsoletes", - "obsoletes_dist", - "platforms", - "provides", - "provides_dist", - "provides_extra", - "requires", - "requires_dist", - "requires_external", - "supported_platforms", -} - -_DICT_FIELDS = { - "project_urls", -} - - -def _parse_keywords(data: str) -> List[str]: - """Split a string of comma-separate keyboards into a list of keywords.""" - return [k.strip() for k in data.split(",")] - - -def _parse_project_urls(data: List[str]) -> Dict[str, str]: - """Parse a list of label/URL string pairings separated by a comma.""" - urls = {} - for pair in data: - # Our logic is slightly tricky here as we want to try and do - # *something* reasonable with malformed data. - # - # The main thing that we have to worry about, is data that does - # not have a ',' at all to split the label from the Value. There - # isn't a singular right answer here, and we will fail validation - # later on (if the caller is validating) so it doesn't *really* - # matter, but since the missing value has to be an empty str - # and our return value is dict[str, str], if we let the key - # be the missing value, then they'd have multiple '' values that - # overwrite each other in a accumulating dict. - # - # The other potentional issue is that it's possible to have the - # same label multiple times in the metadata, with no solid "right" - # answer with what to do in that case. As such, we'll do the only - # thing we can, which is treat the field as unparseable and add it - # to our list of unparsed fields. - parts = [p.strip() for p in pair.split(",", 1)] - parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items - - # TODO: The spec doesn't say anything about if the keys should be - # considered case sensitive or not... logically they should - # be case-preserving and case-insensitive, but doing that - # would open up more cases where we might have duplicate - # entries. - label, url = parts - if label in urls: - # The label already exists in our set of urls, so this field - # is unparseable, and we can just add the whole thing to our - # unparseable data and stop processing it. - raise KeyError("duplicate labels in project urls") - urls[label] = url - - return urls - - -def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: - """Get the body of the message.""" - # If our source is a str, then our caller has managed encodings for us, - # and we don't need to deal with it. - if isinstance(source, str): - payload: str = msg.get_payload() - return payload - # If our source is a bytes, then we're managing the encoding and we need - # to deal with it. - else: - bpayload: bytes = msg.get_payload(decode=True) - try: - return bpayload.decode("utf8", "strict") - except UnicodeDecodeError: - raise ValueError("payload in an invalid encoding") - - -# The various parse_FORMAT functions here are intended to be as lenient as -# possible in their parsing, while still returning a correctly typed -# RawMetadata. -# -# To aid in this, we also generally want to do as little touching of the -# data as possible, except where there are possibly some historic holdovers -# that make valid data awkward to work with. -# -# While this is a lower level, intermediate format than our ``Metadata`` -# class, some light touch ups can make a massive difference in usability. - -# Map METADATA fields to RawMetadata. -_EMAIL_TO_RAW_MAPPING = { - "author": "author", - "author-email": "author_email", - "classifier": "classifiers", - "description": "description", - "description-content-type": "description_content_type", - "download-url": "download_url", - "dynamic": "dynamic", - "home-page": "home_page", - "keywords": "keywords", - "license": "license", - "maintainer": "maintainer", - "maintainer-email": "maintainer_email", - "metadata-version": "metadata_version", - "name": "name", - "obsoletes": "obsoletes", - "obsoletes-dist": "obsoletes_dist", - "platform": "platforms", - "project-url": "project_urls", - "provides": "provides", - "provides-dist": "provides_dist", - "provides-extra": "provides_extra", - "requires": "requires", - "requires-dist": "requires_dist", - "requires-external": "requires_external", - "requires-python": "requires_python", - "summary": "summary", - "supported-platform": "supported_platforms", - "version": "version", -} -_RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()} - - -def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: - """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``). - - This function returns a two-item tuple of dicts. The first dict is of - recognized fields from the core metadata specification. Fields that can be - parsed and translated into Python's built-in types are converted - appropriately. All other fields are left as-is. Fields that are allowed to - appear multiple times are stored as lists. - - The second dict contains all other fields from the metadata. This includes - any unrecognized fields. It also includes any fields which are expected to - be parsed into a built-in type but were not formatted appropriately. Finally, - any fields that are expected to appear only once but are repeated are - included in this dict. - - """ - raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} - unparsed: Dict[str, List[str]] = {} - - if isinstance(data, str): - parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) - else: - parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) - - # We have to wrap parsed.keys() in a set, because in the case of multiple - # values for a key (a list), the key will appear multiple times in the - # list of keys, but we're avoiding that by using get_all(). - for name in frozenset(parsed.keys()): - # Header names in RFC are case insensitive, so we'll normalize to all - # lower case to make comparisons easier. - name = name.lower() - - # We use get_all() here, even for fields that aren't multiple use, - # because otherwise someone could have e.g. two Name fields, and we - # would just silently ignore it rather than doing something about it. - headers = parsed.get_all(name) or [] - - # The way the email module works when parsing bytes is that it - # unconditionally decodes the bytes as ascii using the surrogateescape - # handler. When you pull that data back out (such as with get_all() ), - # it looks to see if the str has any surrogate escapes, and if it does - # it wraps it in a Header object instead of returning the string. - # - # As such, we'll look for those Header objects, and fix up the encoding. - value = [] - # Flag if we have run into any issues processing the headers, thus - # signalling that the data belongs in 'unparsed'. - valid_encoding = True - for h in headers: - # It's unclear if this can return more types than just a Header or - # a str, so we'll just assert here to make sure. - assert isinstance(h, (email.header.Header, str)) - - # If it's a header object, we need to do our little dance to get - # the real data out of it. In cases where there is invalid data - # we're going to end up with mojibake, but there's no obvious, good - # way around that without reimplementing parts of the Header object - # ourselves. - # - # That should be fine since, if mojibacked happens, this key is - # going into the unparsed dict anyways. - if isinstance(h, email.header.Header): - # The Header object stores it's data as chunks, and each chunk - # can be independently encoded, so we'll need to check each - # of them. - chunks: List[Tuple[bytes, Optional[str]]] = [] - for bin, encoding in email.header.decode_header(h): - try: - bin.decode("utf8", "strict") - except UnicodeDecodeError: - # Enable mojibake. - encoding = "latin1" - valid_encoding = False - else: - encoding = "utf8" - chunks.append((bin, encoding)) - - # Turn our chunks back into a Header object, then let that - # Header object do the right thing to turn them into a - # string for us. - value.append(str(email.header.make_header(chunks))) - # This is already a string, so just add it. - else: - value.append(h) - - # We've processed all of our values to get them into a list of str, - # but we may have mojibake data, in which case this is an unparsed - # field. - if not valid_encoding: - unparsed[name] = value - continue - - raw_name = _EMAIL_TO_RAW_MAPPING.get(name) - if raw_name is None: - # This is a bit of a weird situation, we've encountered a key that - # we don't know what it means, so we don't know whether it's meant - # to be a list or not. - # - # Since we can't really tell one way or another, we'll just leave it - # as a list, even though it may be a single item list, because that's - # what makes the most sense for email headers. - unparsed[name] = value - continue - - # If this is one of our string fields, then we'll check to see if our - # value is a list of a single item. If it is then we'll assume that - # it was emitted as a single string, and unwrap the str from inside - # the list. - # - # If it's any other kind of data, then we haven't the faintest clue - # what we should parse it as, and we have to just add it to our list - # of unparsed stuff. - if raw_name in _STRING_FIELDS and len(value) == 1: - raw[raw_name] = value[0] - # If this is one of our list of string fields, then we can just assign - # the value, since email *only* has strings, and our get_all() call - # above ensures that this is a list. - elif raw_name in _LIST_FIELDS: - raw[raw_name] = value - # Special Case: Keywords - # The keywords field is implemented in the metadata spec as a str, - # but it conceptually is a list of strings, and is serialized using - # ", ".join(keywords), so we'll do some light data massaging to turn - # this into what it logically is. - elif raw_name == "keywords" and len(value) == 1: - raw[raw_name] = _parse_keywords(value[0]) - # Special Case: Project-URL - # The project urls is implemented in the metadata spec as a list of - # specially-formatted strings that represent a key and a value, which - # is fundamentally a mapping, however the email format doesn't support - # mappings in a sane way, so it was crammed into a list of strings - # instead. - # - # We will do a little light data massaging to turn this into a map as - # it logically should be. - elif raw_name == "project_urls": - try: - raw[raw_name] = _parse_project_urls(value) - except KeyError: - unparsed[name] = value - # Nothing that we've done has managed to parse this, so it'll just - # throw it in our unparseable data and move on. - else: - unparsed[name] = value - - # We need to support getting the Description from the message payload in - # addition to getting it from the the headers. This does mean, though, there - # is the possibility of it being set both ways, in which case we put both - # in 'unparsed' since we don't know which is right. - try: - payload = _get_payload(parsed, data) - except ValueError: - unparsed.setdefault("description", []).append( - parsed.get_payload(decode=isinstance(data, bytes)) - ) - else: - if payload: - # Check to see if we've already got a description, if so then both - # it, and this body move to unparseable. - if "description" in raw: - description_header = cast(str, raw.pop("description")) - unparsed.setdefault("description", []).extend( - [description_header, payload] - ) - elif "description" in unparsed: - unparsed["description"].append(payload) - else: - raw["description"] = payload - - # We need to cast our `raw` to a metadata, because a TypedDict only support - # literal key names, but we're computing our key names on purpose, but the - # way this function is implemented, our `TypedDict` can only have valid key - # names. - return cast(RawMetadata, raw), unparsed - - -_NOT_FOUND = object() - - -# Keep the two values in sync. -_VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] -_MetadataVersion = Literal["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] - -_REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"]) - - -class _Validator(Generic[T]): - """Validate a metadata field. - - All _process_*() methods correspond to a core metadata field. The method is - called with the field's raw value. If the raw value is valid it is returned - in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field). - If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause - as appropriate). - """ - - name: str - raw_name: str - added: _MetadataVersion - - def __init__( - self, - *, - added: _MetadataVersion = "1.0", - ) -> None: - self.added = added - - def __set_name__(self, _owner: "Metadata", name: str) -> None: - self.name = name - self.raw_name = _RAW_TO_EMAIL_MAPPING[name] - - def __get__(self, instance: "Metadata", _owner: Type["Metadata"]) -> T: - # With Python 3.8, the caching can be replaced with functools.cached_property(). - # No need to check the cache as attribute lookup will resolve into the - # instance's __dict__ before __get__ is called. - cache = instance.__dict__ - value = instance._raw.get(self.name) - - # To make the _process_* methods easier, we'll check if the value is None - # and if this field is NOT a required attribute, and if both of those - # things are true, we'll skip the the converter. This will mean that the - # converters never have to deal with the None union. - if self.name in _REQUIRED_ATTRS or value is not None: - try: - converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}") - except AttributeError: - pass - else: - value = converter(value) - - cache[self.name] = value - try: - del instance._raw[self.name] # type: ignore[misc] - except KeyError: - pass - - return cast(T, value) - - def _invalid_metadata( - self, msg: str, cause: Optional[Exception] = None - ) -> InvalidMetadata: - exc = InvalidMetadata( - self.raw_name, msg.format_map({"field": repr(self.raw_name)}) - ) - exc.__cause__ = cause - return exc - - def _process_metadata_version(self, value: str) -> _MetadataVersion: - # Implicitly makes Metadata-Version required. - if value not in _VALID_METADATA_VERSIONS: - raise self._invalid_metadata(f"{value!r} is not a valid metadata version") - return cast(_MetadataVersion, value) - - def _process_name(self, value: str) -> str: - if not value: - raise self._invalid_metadata("{field} is a required field") - # Validate the name as a side-effect. - try: - utils.canonicalize_name(value, validate=True) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - else: - return value - - def _process_version(self, value: str) -> version_module.Version: - if not value: - raise self._invalid_metadata("{field} is a required field") - try: - return version_module.parse(value) - except version_module.InvalidVersion as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_summary(self, value: str) -> str: - """Check the field contains no newlines.""" - if "\n" in value: - raise self._invalid_metadata("{field} must be a single line") - return value - - def _process_description_content_type(self, value: str) -> str: - content_types = {"text/plain", "text/x-rst", "text/markdown"} - message = email.message.EmailMessage() - message["content-type"] = value - - content_type, parameters = ( - # Defaults to `text/plain` if parsing failed. - message.get_content_type().lower(), - message["content-type"].params, - ) - # Check if content-type is valid or defaulted to `text/plain` and thus was - # not parseable. - if content_type not in content_types or content_type not in value.lower(): - raise self._invalid_metadata( - f"{{field}} must be one of {list(content_types)}, not {value!r}" - ) - - charset = parameters.get("charset", "UTF-8") - if charset != "UTF-8": - raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" - ) - - markdown_variants = {"GFM", "CommonMark"} - variant = parameters.get("variant", "GFM") # Use an acceptable default. - if content_type == "text/markdown" and variant not in markdown_variants: - raise self._invalid_metadata( - f"valid Markdown variants for {{field}} are {list(markdown_variants)}, " - f"not {variant!r}", - ) - return value - - def _process_dynamic(self, value: List[str]) -> List[str]: - for dynamic_field in map(str.lower, value): - if dynamic_field in {"name", "version", "metadata-version"}: - raise self._invalid_metadata( - f"{value!r} is not allowed as a dynamic field" - ) - elif dynamic_field not in _EMAIL_TO_RAW_MAPPING: - raise self._invalid_metadata(f"{value!r} is not a valid dynamic field") - return list(map(str.lower, value)) - - def _process_provides_extra( - self, - value: List[str], - ) -> List[utils.NormalizedName]: - normalized_names = [] - try: - for name in value: - normalized_names.append(utils.canonicalize_name(name, validate=True)) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{name!r} is invalid for {{field}}", cause=exc - ) - else: - return normalized_names - - def _process_requires_python(self, value: str) -> specifiers.SpecifierSet: - try: - return specifiers.SpecifierSet(value) - except specifiers.InvalidSpecifier as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_requires_dist( - self, - value: List[str], - ) -> List[requirements.Requirement]: - reqs = [] - try: - for req in value: - reqs.append(requirements.Requirement(req)) - except requirements.InvalidRequirement as exc: - raise self._invalid_metadata(f"{req!r} is invalid for {{field}}", cause=exc) - else: - return reqs - - -class Metadata: - """Representation of distribution metadata. - - Compared to :class:`RawMetadata`, this class provides objects representing - metadata fields instead of only using built-in types. Any invalid metadata - will cause :exc:`InvalidMetadata` to be raised (with a - :py:attr:`~BaseException.__cause__` attribute as appropriate). - """ - - _raw: RawMetadata - - @classmethod - def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> "Metadata": - """Create an instance from :class:`RawMetadata`. - - If *validate* is true, all metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - ins = cls() - ins._raw = data.copy() # Mutations occur due to caching enriched values. - - if validate: - exceptions: List[Exception] = [] - try: - metadata_version = ins.metadata_version - metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None - - # Make sure to check for the fields that are present, the required - # fields (so their absence can be reported). - fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS - # Remove fields that have already been checked. - fields_to_check -= {"metadata_version"} - - for key in fields_to_check: - try: - if metadata_version: - # Can't use getattr() as that triggers descriptor protocol which - # will fail due to no value for the instance argument. - try: - field_metadata_version = cls.__dict__[key].added - except KeyError: - exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) - continue - field_age = _VALID_METADATA_VERSIONS.index( - field_metadata_version - ) - if field_age > metadata_age: - field = _RAW_TO_EMAIL_MAPPING[key] - exc = InvalidMetadata( - field, - "{field} introduced in metadata version " - "{field_metadata_version}, not {metadata_version}", - ) - exceptions.append(exc) - continue - getattr(ins, key) - except InvalidMetadata as exc: - exceptions.append(exc) - - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) - - return ins - - @classmethod - def from_email( - cls, data: Union[bytes, str], *, validate: bool = True - ) -> "Metadata": - """Parse metadata from email headers. - - If *validate* is true, the metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - raw, unparsed = parse_email(data) - - if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) - - try: - return cls.from_raw(raw, validate=validate) - except ExceptionGroup as exc_group: - raise ExceptionGroup( - "invalid or unparsed metadata", exc_group.exceptions - ) from None - - metadata_version: _Validator[_MetadataVersion] = _Validator() - """:external:ref:`core-metadata-metadata-version` - (required; validated to be a valid metadata version)""" - name: _Validator[str] = _Validator() - """:external:ref:`core-metadata-name` - (required; validated using :func:`~packaging.utils.canonicalize_name` and its - *validate* parameter)""" - version: _Validator[version_module.Version] = _Validator() - """:external:ref:`core-metadata-version` (required)""" - dynamic: _Validator[Optional[List[str]]] = _Validator( - added="2.2", - ) - """:external:ref:`core-metadata-dynamic` - (validated against core metadata field names and lowercased)""" - platforms: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-platform`""" - supported_platforms: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-supported-platform`""" - summary: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-summary` (validated to contain no newlines)""" - description: _Validator[Optional[str]] = _Validator() # TODO 2.1: can be in body - """:external:ref:`core-metadata-description`""" - description_content_type: _Validator[Optional[str]] = _Validator(added="2.1") - """:external:ref:`core-metadata-description-content-type` (validated)""" - keywords: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-keywords`""" - home_page: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-home-page`""" - download_url: _Validator[Optional[str]] = _Validator(added="1.1") - """:external:ref:`core-metadata-download-url`""" - author: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author`""" - author_email: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author-email`""" - maintainer: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer`""" - maintainer_email: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer-email`""" - license: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-license`""" - classifiers: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-classifier`""" - requires_dist: _Validator[Optional[List[requirements.Requirement]]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-dist`""" - requires_python: _Validator[Optional[specifiers.SpecifierSet]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-python`""" - # Because `Requires-External` allows for non-PEP 440 version specifiers, we - # don't do any processing on the values. - requires_external: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-requires-external`""" - project_urls: _Validator[Optional[Dict[str, str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-project-url`""" - # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation - # regardless of metadata version. - provides_extra: _Validator[Optional[List[utils.NormalizedName]]] = _Validator( - added="2.1", - ) - """:external:ref:`core-metadata-provides-extra`""" - provides_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-provides-dist`""" - obsoletes_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-obsoletes-dist`""" - requires: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Requires`` (deprecated)""" - provides: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Provides`` (deprecated)""" - obsoletes: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Obsoletes`` (deprecated)""" diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py deleted file mode 100644 index 2d015bab595..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/specifiers.py +++ /dev/null @@ -1,1017 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.specifiers import Specifier, SpecifierSet, InvalidSpecifier - from packaging.version import Version -""" - -import abc -import itertools -import re -from typing import Callable, Iterable, Iterator, List, Optional, Tuple, TypeVar, Union - -from .utils import canonicalize_version -from .version import Version - -UnparsedVersion = Union[Version, str] -UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) -CallableOperator = Callable[[Version, str], bool] - - -def _coerce_version(version: UnparsedVersion) -> Version: - if not isinstance(version, Version): - version = Version(version) - return version - - -class InvalidSpecifier(ValueError): - """ - Raised when attempting to create a :class:`Specifier` with a specifier - string that is invalid. - - >>> Specifier("lolwat") - Traceback (most recent call last): - ... - packaging.specifiers.InvalidSpecifier: Invalid specifier: 'lolwat' - """ - - -class BaseSpecifier(metaclass=abc.ABCMeta): - @abc.abstractmethod - def __str__(self) -> str: - """ - Returns the str representation of this Specifier-like object. This - should be representative of the Specifier itself. - """ - - @abc.abstractmethod - def __hash__(self) -> int: - """ - Returns a hash value for this Specifier-like object. - """ - - @abc.abstractmethod - def __eq__(self, other: object) -> bool: - """ - Returns a boolean representing whether or not the two Specifier-like - objects are equal. - - :param other: The other object to check against. - """ - - @property - @abc.abstractmethod - def prereleases(self) -> Optional[bool]: - """Whether or not pre-releases as a whole are allowed. - - This can be set to either ``True`` or ``False`` to explicitly enable or disable - prereleases or it can be set to ``None`` (the default) to use default semantics. - """ - - @prereleases.setter - def prereleases(self, value: bool) -> None: - """Setter for :attr:`prereleases`. - - :param value: The value to set. - """ - - @abc.abstractmethod - def contains(self, item: str, prereleases: Optional[bool] = None) -> bool: - """ - Determines if the given item is contained within this specifier. - """ - - @abc.abstractmethod - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """ - Takes an iterable of items and filters them so that only items which - are contained within this specifier are allowed in it. - """ - - -class Specifier(BaseSpecifier): - """This class abstracts handling of version specifiers. - - .. tip:: - - It is generally not required to instantiate this manually. You should instead - prefer to work with :class:`SpecifierSet` instead, which can parse - comma-separated version specifiers (which is what package metadata contains). - """ - - _operator_regex_str = r""" - (?P<operator>(~=|==|!=|<=|>=|<|>|===)) - """ - _version_regex_str = r""" - (?P<version> - (?: - # The identity operators allow for an escape hatch that will - # do an exact string match of the version you wish to install. - # This will not be parsed by PEP 440 and we cannot determine - # any semantic meaning from it. This operator is discouraged - # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator - \s* - [^\s;)]* # The arbitrary version can be just about anything, - # we match everything except for whitespace, a - # semi-colon for marker support, and a closing paren - # since versions can be enclosed in them. - ) - | - (?: - # The (non)equality operators allow for wild card and local - # versions to be specified so we have to define these two - # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - - # You cannot use a wild card and a pre-release, post-release, a dev or - # local version together so group them with a | and make them optional. - (?: - \.\* # Wild card syntax of .* - | - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local - )? - ) - | - (?: - # The compatible operator requires at least two digits in the - # release segment. - (?<=~=) # Only match for the compatible operator - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - | - (?: - # All other operators only allow a sub set of what the - # (non)equality operators do. Specifically they do not allow - # local versions to be specified nor do they allow the prefix - # matching wild cards. - (?<!==|!=|~=) # We have special cases for these - # operators so we want to make sure they - # don't match here. - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - (?: # pre release - [-_\.]? - (alpha|beta|preview|pre|a|b|c|rc) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - ) - """ - - _regex = re.compile( - r"^\s*" + _operator_regex_str + _version_regex_str + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) - - _operators = { - "~=": "compatible", - "==": "equal", - "!=": "not_equal", - "<=": "less_than_equal", - ">=": "greater_than_equal", - "<": "less_than", - ">": "greater_than", - "===": "arbitrary", - } - - def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None: - """Initialize a Specifier instance. - - :param spec: - The string representation of a specifier which will be parsed and - normalized before use. - :param prereleases: - This tells the specifier if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - :raises InvalidSpecifier: - If the given specifier is invalid (i.e. bad syntax). - """ - match = self._regex.search(spec) - if not match: - raise InvalidSpecifier(f"Invalid specifier: '{spec}'") - - self._spec: Tuple[str, str] = ( - match.group("operator").strip(), - match.group("version").strip(), - ) - - # Store whether or not this Specifier should accept prereleases - self._prereleases = prereleases - - # https://github.com/python/mypy/pull/13475#pullrequestreview-1079784515 - @property # type: ignore[override] - def prereleases(self) -> bool: - # If there is an explicit prereleases set for this, then we'll just - # blindly use that. - if self._prereleases is not None: - return self._prereleases - - # Look at all of our specifiers and determine if they are inclusive - # operators, and if they are if they are including an explicit - # prerelease. - operator, version = self._spec - if operator in ["==", ">=", "<=", "~=", "==="]: - # The == specifier can include a trailing .*, if it does we - # want to remove before parsing. - if operator == "==" and version.endswith(".*"): - version = version[:-2] - - # Parse the version, and if it is a pre-release than this - # specifier allows pre-releases. - if Version(version).is_prerelease: - return True - - return False - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - @property - def operator(self) -> str: - """The operator of this specifier. - - >>> Specifier("==1.2.3").operator - '==' - """ - return self._spec[0] - - @property - def version(self) -> str: - """The version of this specifier. - - >>> Specifier("==1.2.3").version - '1.2.3' - """ - return self._spec[1] - - def __repr__(self) -> str: - """A representation of the Specifier that shows all internal state. - - >>> Specifier('>=1.0.0') - <Specifier('>=1.0.0')> - >>> Specifier('>=1.0.0', prereleases=False) - <Specifier('>=1.0.0', prereleases=False)> - >>> Specifier('>=1.0.0', prereleases=True) - <Specifier('>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<{self.__class__.__name__}({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the Specifier that can be round-tripped. - - >>> str(Specifier('>=1.0.0')) - '>=1.0.0' - >>> str(Specifier('>=1.0.0', prereleases=False)) - '>=1.0.0' - """ - return "{}{}".format(*self._spec) - - @property - def _canonical_spec(self) -> Tuple[str, str]: - canonical_version = canonicalize_version( - self._spec[1], - strip_trailing_zero=(self._spec[0] != "~="), - ) - return self._spec[0], canonical_version - - def __hash__(self) -> int: - return hash(self._canonical_spec) - - def __eq__(self, other: object) -> bool: - """Whether or not the two Specifier-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> Specifier("==1.2.3") == Specifier("== 1.2.3.0") - True - >>> (Specifier("==1.2.3", prereleases=False) == - ... Specifier("==1.2.3", prereleases=True)) - True - >>> Specifier("==1.2.3") == "==1.2.3" - True - >>> Specifier("==1.2.3") == Specifier("==1.2.4") - False - >>> Specifier("==1.2.3") == Specifier("~=1.2.3") - False - """ - if isinstance(other, str): - try: - other = self.__class__(str(other)) - except InvalidSpecifier: - return NotImplemented - elif not isinstance(other, self.__class__): - return NotImplemented - - return self._canonical_spec == other._canonical_spec - - def _get_operator(self, op: str) -> CallableOperator: - operator_callable: CallableOperator = getattr( - self, f"_compare_{self._operators[op]}" - ) - return operator_callable - - def _compare_compatible(self, prospective: Version, spec: str) -> bool: - - # Compatible releases have an equivalent combination of >= and ==. That - # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to - # implement this in terms of the other specifiers instead of - # implementing it ourselves. The only thing we need to do is construct - # the other specifiers. - - # We want everything but the last item in the version, but we want to - # ignore suffix segments. - prefix = _version_join( - list(itertools.takewhile(_is_not_suffix, _version_split(spec)))[:-1] - ) - - # Add the prefix notation to the end of our string - prefix += ".*" - - return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( - prospective, prefix - ) - - def _compare_equal(self, prospective: Version, spec: str) -> bool: - - # We need special logic to handle prefix matching - if spec.endswith(".*"): - # In the case of prefix matching we want to ignore local segment. - normalized_prospective = canonicalize_version( - prospective.public, strip_trailing_zero=False - ) - # Get the normalized version string ignoring the trailing .* - normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) - # Split the spec out by bangs and dots, and pretend that there is - # an implicit dot in between a release segment and a pre-release segment. - split_spec = _version_split(normalized_spec) - - # Split the prospective version out by bangs and dots, and pretend - # that there is an implicit dot in between a release segment and - # a pre-release segment. - split_prospective = _version_split(normalized_prospective) - - # 0-pad the prospective version before shortening it to get the correct - # shortened version. - padded_prospective, _ = _pad_version(split_prospective, split_spec) - - # Shorten the prospective version to be the same length as the spec - # so that we can determine if the specifier is a prefix of the - # prospective version or not. - shortened_prospective = padded_prospective[: len(split_spec)] - - return shortened_prospective == split_spec - else: - # Convert our spec string into a Version - spec_version = Version(spec) - - # If the specifier does not have a local segment, then we want to - # act as if the prospective version also does not have a local - # segment. - if not spec_version.local: - prospective = Version(prospective.public) - - return prospective == spec_version - - def _compare_not_equal(self, prospective: Version, spec: str) -> bool: - return not self._compare_equal(prospective, spec) - - def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) <= Version(spec) - - def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: - - # NB: Local version identifiers are NOT permitted in the version - # specifier, so local version labels can be universally removed from - # the prospective version. - return Version(prospective.public) >= Version(spec) - - def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is less than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective < spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a pre-release version, that we do not accept pre-release - # versions for the version mentioned in the specifier (e.g. <3.1 should - # not match 3.1.dev0, but should match 3.0.dev0). - if not spec.is_prerelease and prospective.is_prerelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # less than the spec version *and* it's not a pre-release of the same - # version in the spec. - return True - - def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: - - # Convert our spec to a Version instance, since we'll want to work with - # it as a version. - spec = Version(spec_str) - - # Check to see if the prospective version is greater than the spec - # version. If it's not we can short circuit and just return False now - # instead of doing extra unneeded work. - if not prospective > spec: - return False - - # This special case is here so that, unless the specifier itself - # includes is a post-release version, that we do not accept - # post-release versions for the version mentioned in the specifier - # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). - if not spec.is_postrelease and prospective.is_postrelease: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is technically greater than, to match. - if prospective.local is not None: - if Version(prospective.base_version) == Version(spec.base_version): - return False - - # If we've gotten to here, it means that prospective version is both - # greater than the spec version *and* it's not a pre-release of the - # same version in the spec. - return True - - def _compare_arbitrary(self, prospective: Version, spec: str) -> bool: - return str(prospective).lower() == str(spec).lower() - - def __contains__(self, item: Union[str, Version]) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in Specifier(">=1.2.3") - True - >>> Version("1.2.3") in Specifier(">=1.2.3") - True - >>> "1.0.0" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3") - False - >>> "1.3.0a1" in Specifier(">=1.2.3", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, item: UnparsedVersion, prereleases: Optional[bool] = None - ) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this Specifier. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> Specifier(">=1.2.3").contains("1.2.3") - True - >>> Specifier(">=1.2.3").contains(Version("1.2.3")) - True - >>> Specifier(">=1.2.3").contains("1.0.0") - False - >>> Specifier(">=1.2.3").contains("1.3.0a1") - False - >>> Specifier(">=1.2.3", prereleases=True).contains("1.3.0a1") - True - >>> Specifier(">=1.2.3").contains("1.3.0a1", prereleases=True) - True - """ - - # Determine if prereleases are to be allowed or not. - if prereleases is None: - prereleases = self.prereleases - - # Normalize item to a Version, this allows us to have a shortcut for - # "2.0" in Specifier(">=2") - normalized_item = _coerce_version(item) - - # Determine if we should be supporting prereleases in this specifier - # or not, if we do not support prereleases than we can short circuit - # logic if this version is a prereleases. - if normalized_item.is_prerelease and not prereleases: - return False - - # Actually do the comparison to determine if this item is contained - # within this Specifier or not. - operator_callable: CallableOperator = self._get_operator(self.operator) - return operator_callable(normalized_item, self.version) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifier. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(Specifier().contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.2.3", "1.3", Version("1.4")])) - ['1.2.3', '1.3', <Version('1.4')>] - >>> list(Specifier(">=1.2.3").filter(["1.2", "1.5a1"])) - ['1.5a1'] - >>> list(Specifier(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - """ - - yielded = False - found_prereleases = [] - - kw = {"prereleases": prereleases if prereleases is not None else True} - - # Attempt to iterate over all the values in the iterable and if any of - # them match, yield them. - for version in iterable: - parsed_version = _coerce_version(version) - - if self.contains(parsed_version, **kw): - # If our version is a prerelease, and we were not set to allow - # prereleases, then we'll store it for later in case nothing - # else matches this specifier. - if parsed_version.is_prerelease and not ( - prereleases or self.prereleases - ): - found_prereleases.append(version) - # Either this is not a prerelease, or we should have been - # accepting prereleases from the beginning. - else: - yielded = True - yield version - - # Now that we've iterated over everything, determine if we've yielded - # any values, and if we have not and we have any prereleases stored up - # then we will go ahead and yield the prereleases. - if not yielded and found_prereleases: - for version in found_prereleases: - yield version - - -_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") - - -def _version_split(version: str) -> List[str]: - """Split version into components. - - The split components are intended for version comparison. The logic does - not attempt to retain the original version string, so joining the - components back with :func:`_version_join` may not produce the original - version string. - """ - result: List[str] = [] - - epoch, _, rest = version.rpartition("!") - result.append(epoch or "0") - - for item in rest.split("."): - match = _prefix_regex.search(item) - if match: - result.extend(match.groups()) - else: - result.append(item) - return result - - -def _version_join(components: List[str]) -> str: - """Join split version components into a version string. - - This function assumes the input came from :func:`_version_split`, where the - first component must be the epoch (either empty or numeric), and all other - components numeric. - """ - epoch, *rest = components - return f"{epoch}!{'.'.join(rest)}" - - -def _is_not_suffix(segment: str) -> bool: - return not any( - segment.startswith(prefix) for prefix in ("dev", "a", "b", "rc", "post") - ) - - -def _pad_version(left: List[str], right: List[str]) -> Tuple[List[str], List[str]]: - left_split, right_split = [], [] - - # Get the release segment of our versions - left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) - right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) - - # Get the rest of our versions - left_split.append(left[len(left_split[0]) :]) - right_split.append(right[len(right_split[0]) :]) - - # Insert our padding - left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) - right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - - return ( - list(itertools.chain.from_iterable(left_split)), - list(itertools.chain.from_iterable(right_split)), - ) - - -class SpecifierSet(BaseSpecifier): - """This class abstracts handling of a set of version specifiers. - - It can be passed a single specifier (``>=3.0``), a comma-separated list of - specifiers (``>=3.0,!=3.1``), or no specifier at all. - """ - - def __init__( - self, specifiers: str = "", prereleases: Optional[bool] = None - ) -> None: - """Initialize a SpecifierSet instance. - - :param specifiers: - The string representation of a specifier or a comma-separated list of - specifiers which will be parsed and normalized before use. - :param prereleases: - This tells the SpecifierSet if it should accept prerelease versions if - applicable or not. The default of ``None`` will autodetect it from the - given specifiers. - - :raises InvalidSpecifier: - If the given ``specifiers`` are not parseable than this exception will be - raised. - """ - - # Split on `,` to break each individual specifier into it's own item, and - # strip each item to remove leading/trailing whitespace. - split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] - - # Make each individual specifier a Specifier and save in a frozen set for later. - self._specs = frozenset(map(Specifier, split_specifiers)) - - # Store our prereleases value so we can use it later to determine if - # we accept prereleases or not. - self._prereleases = prereleases - - @property - def prereleases(self) -> Optional[bool]: - # If we have been given an explicit prerelease modifier, then we'll - # pass that through here. - if self._prereleases is not None: - return self._prereleases - - # If we don't have any specifiers, and we don't have a forced value, - # then we'll just return None since we don't know if this should have - # pre-releases or not. - if not self._specs: - return None - - # Otherwise we'll see if any of the given specifiers accept - # prereleases, if any of them do we'll return True, otherwise False. - return any(s.prereleases for s in self._specs) - - @prereleases.setter - def prereleases(self, value: bool) -> None: - self._prereleases = value - - def __repr__(self) -> str: - """A representation of the specifier set that shows all internal state. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> SpecifierSet('>=1.0.0,!=2.0.0') - <SpecifierSet('!=2.0.0,>=1.0.0')> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=False) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=False)> - >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=True) - <SpecifierSet('!=2.0.0,>=1.0.0', prereleases=True)> - """ - pre = ( - f", prereleases={self.prereleases!r}" - if self._prereleases is not None - else "" - ) - - return f"<SpecifierSet({str(self)!r}{pre})>" - - def __str__(self) -> str: - """A string representation of the specifier set that can be round-tripped. - - Note that the ordering of the individual specifiers within the set may not - match the input string. - - >>> str(SpecifierSet(">=1.0.0,!=1.0.1")) - '!=1.0.1,>=1.0.0' - >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) - '!=1.0.1,>=1.0.0' - """ - return ",".join(sorted(str(s) for s in self._specs)) - - def __hash__(self) -> int: - return hash(self._specs) - - def __and__(self, other: Union["SpecifierSet", str]) -> "SpecifierSet": - """Return a SpecifierSet which is a combination of the two sets. - - :param other: The other object to combine with. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") & '<=2.0.0,!=2.0.1' - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - >>> SpecifierSet(">=1.0.0,!=1.0.1") & SpecifierSet('<=2.0.0,!=2.0.1') - <SpecifierSet('!=1.0.1,!=2.0.1,<=2.0.0,>=1.0.0')> - """ - if isinstance(other, str): - other = SpecifierSet(other) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - specifier = SpecifierSet() - specifier._specs = frozenset(self._specs | other._specs) - - if self._prereleases is None and other._prereleases is not None: - specifier._prereleases = other._prereleases - elif self._prereleases is not None and other._prereleases is None: - specifier._prereleases = self._prereleases - elif self._prereleases == other._prereleases: - specifier._prereleases = self._prereleases - else: - raise ValueError( - "Cannot combine SpecifierSets with True and False prerelease " - "overrides." - ) - - return specifier - - def __eq__(self, other: object) -> bool: - """Whether or not the two SpecifierSet-like objects are equal. - - :param other: The other object to check against. - - The value of :attr:`prereleases` is ignored. - - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> (SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False) == - ... SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True)) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == ">=1.0.0,!=1.0.1" - True - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.2") - False - """ - if isinstance(other, (str, Specifier)): - other = SpecifierSet(str(other)) - elif not isinstance(other, SpecifierSet): - return NotImplemented - - return self._specs == other._specs - - def __len__(self) -> int: - """Returns the number of specifiers in this specifier set.""" - return len(self._specs) - - def __iter__(self) -> Iterator[Specifier]: - """ - Returns an iterator over all the underlying :class:`Specifier` instances - in this specifier set. - - >>> sorted(SpecifierSet(">=1.0.0,!=1.0.1"), key=str) - [<Specifier('!=1.0.1')>, <Specifier('>=1.0.0')>] - """ - return iter(self._specs) - - def __contains__(self, item: UnparsedVersion) -> bool: - """Return whether or not the item is contained in this specifier. - - :param item: The item to check for. - - This is used for the ``in`` operator and behaves the same as - :meth:`contains` with no ``prereleases`` argument passed. - - >>> "1.2.3" in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> Version("1.2.3") in SpecifierSet(">=1.0.0,!=1.0.1") - True - >>> "1.0.1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1") - False - >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True) - True - """ - return self.contains(item) - - def contains( - self, - item: UnparsedVersion, - prereleases: Optional[bool] = None, - installed: Optional[bool] = None, - ) -> bool: - """Return whether or not the item is contained in this SpecifierSet. - - :param item: - The item to check for, which can be a version string or a - :class:`Version` instance. - :param prereleases: - Whether or not to match prereleases with this SpecifierSet. If set to - ``None`` (the default), it uses :attr:`prereleases` to determine - whether or not prereleases are allowed. - - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.2.3") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains(Version("1.2.3")) - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.0.1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1") - False - >>> SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True).contains("1.3.0a1") - True - >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1", prereleases=True) - True - """ - # Ensure that our item is a Version instance. - if not isinstance(item, Version): - item = Version(item) - - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # We can determine if we're going to allow pre-releases by looking to - # see if any of the underlying items supports them. If none of them do - # and this item is a pre-release then we do not allow it and we can - # short circuit that here. - # Note: This means that 1.0.dev1 would not be contained in something - # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 - if not prereleases and item.is_prerelease: - return False - - if installed and item.is_prerelease: - item = Version(item.base_version) - - # We simply dispatch to the underlying specs here to make sure that the - # given version is contained within all of them. - # Note: This use of all() here means that an empty set of specifiers - # will always return True, this is an explicit design decision. - return all(s.contains(item, prereleases=prereleases) for s in self._specs) - - def filter( - self, iterable: Iterable[UnparsedVersionVar], prereleases: Optional[bool] = None - ) -> Iterator[UnparsedVersionVar]: - """Filter items in the given iterable, that match the specifiers in this set. - - :param iterable: - An iterable that can contain version strings and :class:`Version` instances. - The items in the iterable will be filtered according to the specifier. - :param prereleases: - Whether or not to allow prereleases in the returned iterator. If set to - ``None`` (the default), it will be intelligently decide whether to allow - prereleases or not (based on the :attr:`prereleases` attribute, and - whether the only versions matching are prereleases). - - This method is smarter than just ``filter(SpecifierSet(...).contains, [...])`` - because it implements the rule from :pep:`440` that a prerelease item - SHOULD be accepted if no other versions match the given specifier. - - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", Version("1.4")])) - ['1.3', <Version('1.4')>] - >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.5a1"])) - [] - >>> list(SpecifierSet(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - - An "empty" SpecifierSet will filter items based on the presence of prerelease - versions in the set. - - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"])) - ['1.3'] - >>> list(SpecifierSet("").filter(["1.5a1"])) - ['1.5a1'] - >>> list(SpecifierSet("", prereleases=True).filter(["1.3", "1.5a1"])) - ['1.3', '1.5a1'] - >>> list(SpecifierSet("").filter(["1.3", "1.5a1"], prereleases=True)) - ['1.3', '1.5a1'] - """ - # Determine if we're forcing a prerelease or not, if we're not forcing - # one for this particular filter call, then we'll use whatever the - # SpecifierSet thinks for whether or not we should support prereleases. - if prereleases is None: - prereleases = self.prereleases - - # If we have any specifiers, then we want to wrap our iterable in the - # filter method for each one, this will act as a logical AND amongst - # each specifier. - if self._specs: - for spec in self._specs: - iterable = spec.filter(iterable, prereleases=bool(prereleases)) - return iter(iterable) - # If we do not have any specifiers, then we need to have a rough filter - # which will filter out any pre-releases, unless there are no final - # releases. - else: - filtered: List[UnparsedVersionVar] = [] - found_prereleases: List[UnparsedVersionVar] = [] - - for item in iterable: - parsed_version = _coerce_version(item) - - # Store any item which is a pre-release for later unless we've - # already found a final version or we are accepting prereleases - if parsed_version.is_prerelease and not prereleases: - if not filtered: - found_prereleases.append(item) - else: - filtered.append(item) - - # If we've found no items except for pre-releases, then we'll go - # ahead and use the pre-releases - if not filtered and found_prereleases and prereleases is None: - return iter(found_prereleases) - - return iter(filtered) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py deleted file mode 100644 index aef2821b83f..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -Utilities for determining application-specific dirs. See <https://github.com/platformdirs/platformdirs> for details and -usage. -""" -from __future__ import annotations - -import os -import sys -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no cover (py38+) - from typing import Literal -else: # pragma: no cover (py38+) - from ..typing_extensions import Literal - -from .api import PlatformDirsABC -from .version import __version__ -from .version import __version_tuple__ as __version_info__ - - -def _set_platform_dir_class() -> type[PlatformDirsABC]: - if sys.platform == "win32": - from .windows import Windows as Result - elif sys.platform == "darwin": - from .macos import MacOS as Result - else: - from .unix import Unix as Result - - if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system": - - if os.getenv("SHELL") or os.getenv("PREFIX"): - return Result - - from .android import _android_folder - - if _android_folder() is not None: - from .android import Android - - return Android # return to avoid redefinition of result - - return Result - - -PlatformDirs = _set_platform_dir_class() #: Currently active platform -AppDirs = PlatformDirs #: Backwards compatibility with appdirs - - -def user_data_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: data directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_dir - - -def site_data_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: data directory shared by users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_dir - - -def user_config_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: config directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_dir - - -def site_config_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: config directory shared by the users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_dir - - -def user_cache_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: cache directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_dir - - -def user_state_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: state directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_dir - - -def user_log_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: log directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_dir - - -def user_documents_dir() -> str: - """ - :returns: documents directory tied to the user - """ - return PlatformDirs().user_documents_dir - - -def user_runtime_dir( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> str: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: runtime directory tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_dir - - -def user_data_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: data path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_path - - -def site_data_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `multipath <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: data path shared by users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_path - - -def user_config_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: config path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_path - - -def site_config_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - multipath: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param multipath: See `roaming <platformdirs.api.PlatformDirsABC.multipath>`. - :returns: config path shared by the users - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_path - - -def user_cache_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: cache path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_path - - -def user_state_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param roaming: See `roaming <platformdirs.api.PlatformDirsABC.version>`. - :returns: state path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_path - - -def user_log_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `roaming <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: log path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_path - - -def user_documents_path() -> Path: - """ - :returns: documents path tied to the user - """ - return PlatformDirs().user_documents_path - - -def user_runtime_path( - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - opinion: bool = True, -) -> Path: - """ - :param appname: See `appname <platformdirs.api.PlatformDirsABC.appname>`. - :param appauthor: See `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`. - :param version: See `version <platformdirs.api.PlatformDirsABC.version>`. - :param opinion: See `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - :returns: runtime path tied to the user - """ - return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_path - - -__all__ = [ - "__version__", - "__version_info__", - "PlatformDirs", - "AppDirs", - "PlatformDirsABC", - "user_data_dir", - "user_config_dir", - "user_cache_dir", - "user_state_dir", - "user_log_dir", - "user_documents_dir", - "user_runtime_dir", - "site_data_dir", - "site_config_dir", - "user_data_path", - "user_config_path", - "user_cache_path", - "user_state_path", - "user_log_path", - "user_documents_path", - "user_runtime_path", - "site_data_path", - "site_config_path", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py deleted file mode 100644 index 0fc1edd59cf..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/__main__.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from platformdirs import PlatformDirs, __version__ - -PROPS = ( - "user_data_dir", - "user_config_dir", - "user_cache_dir", - "user_state_dir", - "user_log_dir", - "user_documents_dir", - "user_runtime_dir", - "site_data_dir", - "site_config_dir", -) - - -def main() -> None: - app_name = "MyApp" - app_author = "MyCompany" - - print(f"-- platformdirs {__version__} --") - - print("-- app dirs (with optional 'version')") - dirs = PlatformDirs(app_name, app_author, version="1.0") - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (without optional 'version')") - dirs = PlatformDirs(app_name, app_author) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (without optional 'appauthor')") - dirs = PlatformDirs(app_name) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - print("\n-- app dirs (with disabled 'appauthor')") - dirs = PlatformDirs(app_name, appauthor=False) - for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") - - -if __name__ == "__main__": - main() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py deleted file mode 100644 index eda80935123..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/android.py +++ /dev/null @@ -1,120 +0,0 @@ -from __future__ import annotations - -import os -import re -import sys -from functools import lru_cache -from typing import cast - -from .api import PlatformDirsABC - - -class Android(PlatformDirsABC): - """ - Follows the guidance `from here <https://android.stackexchange.com/a/216132>`_. Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>` and - `version <platformdirs.api.PlatformDirsABC.version>`. - """ - - @property - def user_data_dir(self) -> str: - """:return: data directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/files/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "files") - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_config_dir(self) -> str: - """ - :return: config directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/shared_prefs/<AppName>`` - """ - return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, same as `user_config_dir`""" - return self.user_config_dir - - @property - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user, e.g. e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "cache") - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_cache_dir` if not opinionated else ``log`` in it, - e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/log`` - """ - path = self.user_cache_dir - if self.opinion: - path = os.path.join(path, "log") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents`` - """ - return _android_documents_folder() - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, same as `user_cache_dir` if not opinionated else ``tmp`` in it, - e.g. ``/data/user/<userid>/<packagename>/cache/<AppName>/tmp`` - """ - path = self.user_cache_dir - if self.opinion: - path = os.path.join(path, "tmp") - return path - - -@lru_cache(maxsize=1) -def _android_folder() -> str | None: - """:return: base folder for the Android OS or None if cannot be found""" - try: - # First try to get path to android app via pyjnius - from jnius import autoclass - - Context = autoclass("android.content.Context") # noqa: N806 - result: str | None = Context.getFilesDir().getParentFile().getAbsolutePath() - except Exception: - # if fails find an android folder looking path on the sys.path - pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files") - for path in sys.path: - if pattern.match(path): - result = path.split("/files")[0] - break - else: - result = None - return result - - -@lru_cache(maxsize=1) -def _android_documents_folder() -> str: - """:return: documents folder for the Android OS""" - # Get directories with pyjnius - try: - from jnius import autoclass - - Context = autoclass("android.content.Context") # noqa: N806 - Environment = autoclass("android.os.Environment") # noqa: N806 - documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath() - except Exception: - documents_dir = "/storage/emulated/0/Documents" - - return documents_dir - - -__all__ = [ - "Android", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py deleted file mode 100644 index 6f6e2c2c69d..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/api.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import os -import sys -from abc import ABC, abstractmethod -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no branch - from typing import Literal # pragma: no cover - - -class PlatformDirsABC(ABC): - """ - Abstract base class for platform directories. - """ - - def __init__( - self, - appname: str | None = None, - appauthor: str | None | Literal[False] = None, - version: str | None = None, - roaming: bool = False, - multipath: bool = False, - opinion: bool = True, - ): - """ - Create a new platform directory. - - :param appname: See `appname`. - :param appauthor: See `appauthor`. - :param version: See `version`. - :param roaming: See `roaming`. - :param multipath: See `multipath`. - :param opinion: See `opinion`. - """ - self.appname = appname #: The name of application. - self.appauthor = appauthor - """ - The name of the app author or distributing body for this application. Typically, it is the owning company name. - Defaults to `appname`. You may pass ``False`` to disable it. - """ - self.version = version - """ - An optional version path element to append to the path. You might want to use this if you want multiple versions - of your app to be able to run independently. If used, this would typically be ``<major>.<minor>``. - """ - self.roaming = roaming - """ - Whether to use the roaming appdata directory on Windows. That means that for users on a Windows network setup - for roaming profiles, this user data will be synced on login (see - `here <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>`_). - """ - self.multipath = multipath - """ - An optional parameter only applicable to Unix/Linux which indicates that the entire list of data dirs should be - returned. By default, the first item would only be returned. - """ - self.opinion = opinion #: A flag to indicating to use opinionated values. - - def _append_app_name_and_version(self, *base: str) -> str: - params = list(base[1:]) - if self.appname: - params.append(self.appname) - if self.version: - params.append(self.version) - return os.path.join(base[0], *params) - - @property - @abstractmethod - def user_data_dir(self) -> str: - """:return: data directory tied to the user""" - - @property - @abstractmethod - def site_data_dir(self) -> str: - """:return: data directory shared by users""" - - @property - @abstractmethod - def user_config_dir(self) -> str: - """:return: config directory tied to the user""" - - @property - @abstractmethod - def site_config_dir(self) -> str: - """:return: config directory shared by the users""" - - @property - @abstractmethod - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user""" - - @property - @abstractmethod - def user_state_dir(self) -> str: - """:return: state directory tied to the user""" - - @property - @abstractmethod - def user_log_dir(self) -> str: - """:return: log directory tied to the user""" - - @property - @abstractmethod - def user_documents_dir(self) -> str: - """:return: documents directory tied to the user""" - - @property - @abstractmethod - def user_runtime_dir(self) -> str: - """:return: runtime directory tied to the user""" - - @property - def user_data_path(self) -> Path: - """:return: data path tied to the user""" - return Path(self.user_data_dir) - - @property - def site_data_path(self) -> Path: - """:return: data path shared by users""" - return Path(self.site_data_dir) - - @property - def user_config_path(self) -> Path: - """:return: config path tied to the user""" - return Path(self.user_config_dir) - - @property - def site_config_path(self) -> Path: - """:return: config path shared by the users""" - return Path(self.site_config_dir) - - @property - def user_cache_path(self) -> Path: - """:return: cache path tied to the user""" - return Path(self.user_cache_dir) - - @property - def user_state_path(self) -> Path: - """:return: state path tied to the user""" - return Path(self.user_state_dir) - - @property - def user_log_path(self) -> Path: - """:return: log path tied to the user""" - return Path(self.user_log_dir) - - @property - def user_documents_path(self) -> Path: - """:return: documents path tied to the user""" - return Path(self.user_documents_dir) - - @property - def user_runtime_path(self) -> Path: - """:return: runtime path tied to the user""" - return Path(self.user_runtime_dir) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py deleted file mode 100644 index a01337c7764..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/macos.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import os - -from .api import PlatformDirsABC - - -class MacOS(PlatformDirsABC): - """ - Platform directories for the macOS operating system. Follows the guidance from `Apple documentation - <https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html>`_. - Makes use of the `appname <platformdirs.api.PlatformDirsABC.appname>` and - `version <platformdirs.api.PlatformDirsABC.version>`. - """ - - @property - def user_data_dir(self) -> str: - """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support/")) - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, e.g. ``/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version("/Library/Application Support") - - @property - def user_config_dir(self) -> str: - """:return: config directory tied to the user, e.g. ``~/Library/Preferences/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Preferences/")) - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, e.g. ``/Library/Preferences/$appname``""" - return self._append_app_name_and_version("/Library/Preferences") - - @property - def user_cache_dir(self) -> str: - """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) - - @property - def user_documents_dir(self) -> str: - """:return: documents directory tied to the user, e.g. ``~/Documents``""" - return os.path.expanduser("~/Documents") - - @property - def user_runtime_dir(self) -> str: - """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) - - -__all__ = [ - "MacOS", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py deleted file mode 100644 index 9aca5a03054..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/unix.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import os -import sys -from configparser import ConfigParser -from pathlib import Path - -from .api import PlatformDirsABC - -if sys.platform.startswith("linux"): # pragma: no branch # no op check, only to please the type checker - from os import getuid -else: - - def getuid() -> int: - raise RuntimeError("should only be used on Linux") - - -class Unix(PlatformDirsABC): - """ - On Unix/Linux, we follow the - `XDG Basedir Spec <https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html>`_. The spec allows - overriding directories with environment variables. The examples show are the default values, alongside the name of - the environment variable that overrides them. Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>`, - `version <platformdirs.api.PlatformDirsABC.version>`, - `multipath <platformdirs.api.PlatformDirsABC.multipath>`, - `opinion <platformdirs.api.PlatformDirsABC.opinion>`. - """ - - @property - def user_data_dir(self) -> str: - """ - :return: data directory tied to the user, e.g. ``~/.local/share/$appname/$version`` or - ``$XDG_DATA_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_DATA_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.local/share") - return self._append_app_name_and_version(path) - - @property - def site_data_dir(self) -> str: - """ - :return: data directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` is - enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS - path separator), e.g. ``/usr/local/share/$appname/$version`` or ``/usr/share/$appname/$version`` - """ - # XDG default for $XDG_DATA_DIRS; only first, if multipath is False - path = os.environ.get("XDG_DATA_DIRS", "") - if not path.strip(): - path = f"/usr/local/share{os.pathsep}/usr/share" - return self._with_multi_path(path) - - def _with_multi_path(self, path: str) -> str: - path_list = path.split(os.pathsep) - if not self.multipath: - path_list = path_list[0:1] - path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] - return os.pathsep.join(path_list) - - @property - def user_config_dir(self) -> str: - """ - :return: config directory tied to the user, e.g. ``~/.config/$appname/$version`` or - ``$XDG_CONFIG_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_CONFIG_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.config") - return self._append_app_name_and_version(path) - - @property - def site_config_dir(self) -> str: - """ - :return: config directories shared by users (if `multipath <platformdirs.api.PlatformDirsABC.multipath>` - is enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS - path separator), e.g. ``/etc/xdg/$appname/$version`` - """ - # XDG default for $XDG_CONFIG_DIRS only first, if multipath is False - path = os.environ.get("XDG_CONFIG_DIRS", "") - if not path.strip(): - path = "/etc/xdg" - return self._with_multi_path(path) - - @property - def user_cache_dir(self) -> str: - """ - :return: cache directory tied to the user, e.g. ``~/.cache/$appname/$version`` or - ``~/$XDG_CACHE_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_CACHE_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.cache") - return self._append_app_name_and_version(path) - - @property - def user_state_dir(self) -> str: - """ - :return: state directory tied to the user, e.g. ``~/.local/state/$appname/$version`` or - ``$XDG_STATE_HOME/$appname/$version`` - """ - path = os.environ.get("XDG_STATE_HOME", "") - if not path.strip(): - path = os.path.expanduser("~/.local/state") - return self._append_app_name_and_version(path) - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it - """ - path = self.user_state_dir - if self.opinion: - path = os.path.join(path, "log") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user, e.g. ``~/Documents`` - """ - documents_dir = _get_user_dirs_folder("XDG_DOCUMENTS_DIR") - if documents_dir is None: - documents_dir = os.environ.get("XDG_DOCUMENTS_DIR", "").strip() - if not documents_dir: - documents_dir = os.path.expanduser("~/Documents") - - return documents_dir - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or - ``$XDG_RUNTIME_DIR/$appname/$version`` - """ - path = os.environ.get("XDG_RUNTIME_DIR", "") - if not path.strip(): - path = f"/run/user/{getuid()}" - return self._append_app_name_and_version(path) - - @property - def site_data_path(self) -> Path: - """:return: data path shared by users. Only return first item, even if ``multipath`` is set to ``True``""" - return self._first_item_as_path_if_multipath(self.site_data_dir) - - @property - def site_config_path(self) -> Path: - """:return: config path shared by the users. Only return first item, even if ``multipath`` is set to ``True``""" - return self._first_item_as_path_if_multipath(self.site_config_dir) - - def _first_item_as_path_if_multipath(self, directory: str) -> Path: - if self.multipath: - # If multipath is True, the first path is returned. - directory = directory.split(os.pathsep)[0] - return Path(directory) - - -def _get_user_dirs_folder(key: str) -> str | None: - """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/""" - user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs") - if os.path.exists(user_dirs_config_path): - parser = ConfigParser() - - with open(user_dirs_config_path) as stream: - # Add fake section header, so ConfigParser doesn't complain - parser.read_string(f"[top]\n{stream.read()}") - - if key not in parser["top"]: - return None - - path = parser["top"][key].strip('"') - # Handle relative home paths - path = path.replace("$HOME", os.path.expanduser("~")) - return path - - return None - - -__all__ = [ - "Unix", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py deleted file mode 100644 index 9f6eb98e8f0..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/version.py +++ /dev/null @@ -1,4 +0,0 @@ -# file generated by setuptools_scm -# don't change, don't track in version control -__version__ = version = '2.6.2' -__version_tuple__ = version_tuple = (2, 6, 2) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py deleted file mode 100644 index d5c27b34140..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/windows.py +++ /dev/null @@ -1,184 +0,0 @@ -from __future__ import annotations - -import ctypes -import os -import sys -from functools import lru_cache -from typing import Callable - -from .api import PlatformDirsABC - - -class Windows(PlatformDirsABC): - """`MSDN on where to store app data files - <http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120>`_. - Makes use of the - `appname <platformdirs.api.PlatformDirsABC.appname>`, - `appauthor <platformdirs.api.PlatformDirsABC.appauthor>`, - `version <platformdirs.api.PlatformDirsABC.version>`, - `roaming <platformdirs.api.PlatformDirsABC.roaming>`, - `opinion <platformdirs.api.PlatformDirsABC.opinion>`.""" - - @property - def user_data_dir(self) -> str: - """ - :return: data directory tied to the user, e.g. - ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname`` (not roaming) or - ``%USERPROFILE%\\AppData\\Roaming\\$appauthor\\$appname`` (roaming) - """ - const = "CSIDL_APPDATA" if self.roaming else "CSIDL_LOCAL_APPDATA" - path = os.path.normpath(get_win_folder(const)) - return self._append_parts(path) - - def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str: - params = [] - if self.appname: - if self.appauthor is not False: - author = self.appauthor or self.appname - params.append(author) - params.append(self.appname) - if opinion_value is not None and self.opinion: - params.append(opinion_value) - if self.version: - params.append(self.version) - return os.path.join(path, *params) - - @property - def site_data_dir(self) -> str: - """:return: data directory shared by users, e.g. ``C:\\ProgramData\\$appauthor\\$appname``""" - path = os.path.normpath(get_win_folder("CSIDL_COMMON_APPDATA")) - return self._append_parts(path) - - @property - def user_config_dir(self) -> str: - """:return: config directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def site_config_dir(self) -> str: - """:return: config directory shared by the users, same as `site_data_dir`""" - return self.site_data_dir - - @property - def user_cache_dir(self) -> str: - """ - :return: cache directory tied to the user (if opinionated with ``Cache`` folder within ``$appname``) e.g. - ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname\\Cache\\$version`` - """ - path = os.path.normpath(get_win_folder("CSIDL_LOCAL_APPDATA")) - return self._append_parts(path, opinion_value="Cache") - - @property - def user_state_dir(self) -> str: - """:return: state directory tied to the user, same as `user_data_dir`""" - return self.user_data_dir - - @property - def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it - """ - path = self.user_data_dir - if self.opinion: - path = os.path.join(path, "Logs") - return path - - @property - def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents`` - """ - return os.path.normpath(get_win_folder("CSIDL_PERSONAL")) - - @property - def user_runtime_dir(self) -> str: - """ - :return: runtime directory tied to the user, e.g. - ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname`` - """ - path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) - return self._append_parts(path) - - -def get_win_folder_from_env_vars(csidl_name: str) -> str: - """Get folder from environment variables.""" - if csidl_name == "CSIDL_PERSONAL": # does not have an environment name - return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") - - env_var_name = { - "CSIDL_APPDATA": "APPDATA", - "CSIDL_COMMON_APPDATA": "ALLUSERSPROFILE", - "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA", - }.get(csidl_name) - if env_var_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - result = os.environ.get(env_var_name) - if result is None: - raise ValueError(f"Unset environment variable: {env_var_name}") - return result - - -def get_win_folder_from_registry(csidl_name: str) -> str: - """Get folder from the registry. - - This is a fallback technique at best. I'm not sure if using the - registry for this guarantees us the correct answer for all CSIDL_* - names. - """ - shell_folder_name = { - "CSIDL_APPDATA": "AppData", - "CSIDL_COMMON_APPDATA": "Common AppData", - "CSIDL_LOCAL_APPDATA": "Local AppData", - "CSIDL_PERSONAL": "Personal", - }.get(csidl_name) - if shell_folder_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - if sys.platform != "win32": # only needed for mypy type checker to know that this code runs only on Windows - raise NotImplementedError - import winreg - - key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") - directory, _ = winreg.QueryValueEx(key, shell_folder_name) - return str(directory) - - -def get_win_folder_via_ctypes(csidl_name: str) -> str: - """Get folder with ctypes.""" - csidl_const = { - "CSIDL_APPDATA": 26, - "CSIDL_COMMON_APPDATA": 35, - "CSIDL_LOCAL_APPDATA": 28, - "CSIDL_PERSONAL": 5, - }.get(csidl_name) - if csidl_const is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") - - buf = ctypes.create_unicode_buffer(1024) - windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker - windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) - - # Downgrade to short path name if it has highbit chars. - if any(ord(c) > 255 for c in buf): - buf2 = ctypes.create_unicode_buffer(1024) - if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): - buf = buf2 - - return buf.value - - -def _pick_get_win_folder() -> Callable[[str], str]: - if hasattr(ctypes, "windll"): - return get_win_folder_via_ctypes - try: - import winreg # noqa: F401 - except ImportError: - return get_win_folder_from_env_vars - else: - return get_win_folder_from_registry - - -get_win_folder = lru_cache(maxsize=None)(_pick_get_win_folder()) - -__all__ = [ - "Windows", -] diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py b/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py deleted file mode 100644 index 26b723c1fd3..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/zipp.py +++ /dev/null @@ -1,329 +0,0 @@ -import io -import posixpath -import zipfile -import itertools -import contextlib -import sys -import pathlib - -if sys.version_info < (3, 7): - from collections import OrderedDict -else: - OrderedDict = dict - - -__all__ = ['Path'] - - -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = OrderedDict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(zipfile.ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, zipfile.ZipFile): - return cls(_pathlib_compat(source)) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -def _pathlib_compat(path): - """ - For path-like objects, convert to a filename for compatibility - on Python 3.6.1 and earlier. - """ - try: - return path.__fspath__() - except AttributeError: - return str(path) - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = zipfile.ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix - - @property - def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes - - @property - def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *map(_pathlib_compat, other)) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) diff --git a/contrib/python/setuptools/py3/pkg_resources/api_tests.txt b/contrib/python/setuptools/py3/pkg_resources/api_tests.txt new file mode 100644 index 00000000000..d72b85aa375 --- /dev/null +++ b/contrib/python/setuptools/py3/pkg_resources/api_tests.txt @@ -0,0 +1,424 @@ +Pluggable Distributions of Python Software +========================================== + +Distributions +------------- + +A "Distribution" is a collection of files that represent a "Release" of a +"Project" as of a particular point in time, denoted by a +"Version":: + + >>> import sys, pkg_resources + >>> from pkg_resources import Distribution + >>> Distribution(project_name="Foo", version="1.2") + Foo 1.2 + +Distributions have a location, which can be a filename, URL, or really anything +else you care to use:: + + >>> dist = Distribution( + ... location="http://example.com/something", + ... project_name="Bar", version="0.9" + ... ) + + >>> dist + Bar 0.9 (http://example.com/something) + + +Distributions have various introspectable attributes:: + + >>> dist.location + 'http://example.com/something' + + >>> dist.project_name + 'Bar' + + >>> dist.version + '0.9' + + >>> dist.py_version == '{}.{}'.format(*sys.version_info) + True + + >>> print(dist.platform) + None + +Including various computed attributes:: + + >>> from pkg_resources import parse_version + >>> dist.parsed_version == parse_version(dist.version) + True + + >>> dist.key # case-insensitive form of the project name + 'bar' + +Distributions are compared (and hashed) by version first:: + + >>> Distribution(version='1.0') == Distribution(version='1.0') + True + >>> Distribution(version='1.0') == Distribution(version='1.1') + False + >>> Distribution(version='1.0') < Distribution(version='1.1') + True + +but also by project name (case-insensitive), platform, Python version, +location, etc.:: + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="Foo",version="1.0") + True + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="foo",version="1.0") + True + + >>> Distribution(project_name="Foo",version="1.0") == \ + ... Distribution(project_name="Foo",version="1.1") + False + + >>> Distribution(project_name="Foo",py_version="2.3",version="1.0") == \ + ... Distribution(project_name="Foo",py_version="2.4",version="1.0") + False + + >>> Distribution(location="spam",version="1.0") == \ + ... Distribution(location="spam",version="1.0") + True + + >>> Distribution(location="spam",version="1.0") == \ + ... Distribution(location="baz",version="1.0") + False + + + +Hash and compare distribution by prio/plat + +Get version from metadata +provider capabilities +egg_name() +as_requirement() +from_location, from_filename (w/path normalization) + +Releases may have zero or more "Requirements", which indicate +what releases of another project the release requires in order to +function. A Requirement names the other project, expresses some criteria +as to what releases of that project are acceptable, and lists any "Extras" +that the requiring release may need from that project. (An Extra is an +optional feature of a Release, that can only be used if its additional +Requirements are satisfied.) + + + +The Working Set +--------------- + +A collection of active distributions is called a Working Set. Note that a +Working Set can contain any importable distribution, not just pluggable ones. +For example, the Python standard library is an importable distribution that +will usually be part of the Working Set, even though it is not pluggable. +Similarly, when you are doing development work on a project, the files you are +editing are also a Distribution. (And, with a little attention to the +directory names used, and including some additional metadata, such a +"development distribution" can be made pluggable as well.) + + >>> from pkg_resources import WorkingSet + +A working set's entries are the sys.path entries that correspond to the active +distributions. By default, the working set's entries are the items on +``sys.path``:: + + >>> ws = WorkingSet() + >>> ws.entries == sys.path + True + +But you can also create an empty working set explicitly, and add distributions +to it:: + + >>> ws = WorkingSet([]) + >>> ws.add(dist) + >>> ws.entries + ['http://example.com/something'] + >>> dist in ws + True + >>> Distribution('foo',version="") in ws + False + +And you can iterate over its distributions:: + + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +Adding the same distribution more than once is a no-op:: + + >>> ws.add(dist) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +For that matter, adding multiple distributions for the same project also does +nothing, because a working set can only hold one active distribution per +project -- the first one added to it:: + + >>> ws.add( + ... Distribution( + ... 'http://example.com/something', project_name="Bar", + ... version="7.2" + ... ) + ... ) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +You can append a path entry to a working set using ``add_entry()``:: + + >>> ws.entries + ['http://example.com/something'] + >>> ws.add_entry(pkg_resources.__file__) + >>> ws.entries + ['http://example.com/something', '...pkg_resources...'] + +Multiple additions result in multiple entries, even if the entry is already in +the working set (because ``sys.path`` can contain the same entry more than +once):: + + >>> ws.add_entry(pkg_resources.__file__) + >>> ws.entries + ['...example.com...', '...pkg_resources...', '...pkg_resources...'] + +And you can specify the path entry a distribution was found under, using the +optional second parameter to ``add()``:: + + >>> ws = WorkingSet([]) + >>> ws.add(dist,"foo") + >>> ws.entries + ['foo'] + +But even if a distribution is found under multiple path entries, it still only +shows up once when iterating the working set: + + >>> ws.add_entry(ws.entries[0]) + >>> list(ws) + [Bar 0.9 (http://example.com/something)] + +You can ask a WorkingSet to ``find()`` a distribution matching a requirement:: + + >>> from pkg_resources import Requirement + >>> print(ws.find(Requirement.parse("Foo==1.0"))) # no match, return None + None + + >>> ws.find(Requirement.parse("Bar==0.9")) # match, return distribution + Bar 0.9 (http://example.com/something) + +Note that asking for a conflicting version of a distribution already in a +working set triggers a ``pkg_resources.VersionConflict`` error: + + >>> try: + ... ws.find(Requirement.parse("Bar==1.0")) + ... except pkg_resources.VersionConflict as exc: + ... print(str(exc)) + ... else: + ... raise AssertionError("VersionConflict was not raised") + (Bar 0.9 (http://example.com/something), Requirement.parse('Bar==1.0')) + +You can subscribe a callback function to receive notifications whenever a new +distribution is added to a working set. The callback is immediately invoked +once for each existing distribution in the working set, and then is called +again for new distributions added thereafter:: + + >>> def added(dist): print("Added %s" % dist) + >>> ws.subscribe(added) + Added Bar 0.9 + >>> foo12 = Distribution(project_name="Foo", version="1.2", location="f12") + >>> ws.add(foo12) + Added Foo 1.2 + +Note, however, that only the first distribution added for a given project name +will trigger a callback, even during the initial ``subscribe()`` callback:: + + >>> foo14 = Distribution(project_name="Foo", version="1.4", location="f14") + >>> ws.add(foo14) # no callback, because Foo 1.2 is already active + + >>> ws = WorkingSet([]) + >>> ws.add(foo12) + >>> ws.add(foo14) + >>> ws.subscribe(added) + Added Foo 1.2 + +And adding a callback more than once has no effect, either:: + + >>> ws.subscribe(added) # no callbacks + + # and no double-callbacks on subsequent additions, either + >>> just_a_test = Distribution(project_name="JustATest", version="0.99") + >>> ws.add(just_a_test) + Added JustATest 0.99 + + +Finding Plugins +--------------- + +``WorkingSet`` objects can be used to figure out what plugins in an +``Environment`` can be loaded without any resolution errors:: + + >>> from pkg_resources import Environment + + >>> plugins = Environment([]) # normally, a list of plugin directories + >>> plugins.add(foo12) + >>> plugins.add(foo14) + >>> plugins.add(just_a_test) + +In the simplest case, we just get the newest version of each distribution in +the plugin environment:: + + >>> ws = WorkingSet([]) + >>> ws.find_plugins(plugins) + ([JustATest 0.99, Foo 1.4 (f14)], {}) + +But if there's a problem with a version conflict or missing requirements, the +method falls back to older versions, and the error info dict will contain an +exception instance for each unloadable plugin:: + + >>> ws.add(foo12) # this will conflict with Foo 1.4 + >>> ws.find_plugins(plugins) + ([JustATest 0.99, Foo 1.2 (f12)], {Foo 1.4 (f14): VersionConflict(...)}) + +But if you disallow fallbacks, the failed plugin will be skipped instead of +trying older versions:: + + >>> ws.find_plugins(plugins, fallback=False) + ([JustATest 0.99], {Foo 1.4 (f14): VersionConflict(...)}) + + + +Platform Compatibility Rules +---------------------------- + +On the Mac, there are potential compatibility issues for modules compiled +on newer versions of macOS than what the user is running. Additionally, +macOS will soon have two platforms to contend with: Intel and PowerPC. + +Basic equality works as on other platforms:: + + >>> from pkg_resources import compatible_platforms as cp + >>> reqd = 'macosx-10.4-ppc' + >>> cp(reqd, reqd) + True + >>> cp("win32", reqd) + False + +Distributions made on other machine types are not compatible:: + + >>> cp("macosx-10.4-i386", reqd) + False + +Distributions made on earlier versions of the OS are compatible, as +long as they are from the same top-level version. The patchlevel version +number does not matter:: + + >>> cp("macosx-10.4-ppc", reqd) + True + >>> cp("macosx-10.3-ppc", reqd) + True + >>> cp("macosx-10.5-ppc", reqd) + False + >>> cp("macosx-9.5-ppc", reqd) + False + +Backwards compatibility for packages made via earlier versions of +setuptools is provided as well:: + + >>> cp("darwin-8.2.0-Power_Macintosh", reqd) + True + >>> cp("darwin-7.2.0-Power_Macintosh", reqd) + True + >>> cp("darwin-8.2.0-Power_Macintosh", "macosx-10.3-ppc") + False + + +Environment Markers +------------------- + + >>> from pkg_resources import invalid_marker as im, evaluate_marker as em + >>> import os + + >>> print(im("sys_platform")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + sys_platform + ^ + + >>> print(im("sys_platform==")) + Expected a marker variable or quoted string + sys_platform== + ^ + + >>> print(im("sys_platform=='win32'")) + False + + >>> print(im("sys=='x'")) + Expected a marker variable or quoted string + sys=='x' + ^ + + >>> print(im("(extra)")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + (extra) + ^ + + >>> print(im("(extra")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + (extra + ^ + + >>> print(im("os.open('foo')=='y'")) + Expected a marker variable or quoted string + os.open('foo')=='y' + ^ + + >>> print(im("'x'=='y' and os.open('foo')=='y'")) # no short-circuit! + Expected a marker variable or quoted string + 'x'=='y' and os.open('foo')=='y' + ^ + + >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! + Expected a marker variable or quoted string + 'x'=='x' or os.open('foo')=='y' + ^ + + >>> print(im("r'x'=='x'")) + Expected a marker variable or quoted string + r'x'=='x' + ^ + + >>> print(im("'''x'''=='x'")) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + '''x'''=='x' + ^ + + >>> print(im('"""x"""=="x"')) + Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in + """x"""=="x" + ^ + + >>> print(im(r"x\n=='x'")) + Expected a marker variable or quoted string + x\n=='x' + ^ + + >>> print(im("os.open=='y'")) + Expected a marker variable or quoted string + os.open=='y' + ^ + + >>> em("sys_platform=='win32'") == (sys.platform=='win32') + True + + >>> em("python_version >= '2.7'") + True + + >>> em("python_version > '2.6'") + True + + >>> im("implementation_name=='cpython'") + False + + >>> im("platform_python_implementation=='CPython'") + False + + >>> im("implementation_version=='3.5.1'") + False diff --git a/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py b/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py deleted file mode 100644 index daa978ff728..00000000000 --- a/contrib/python/setuptools/py3/pkg_resources/extern/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations -from importlib.machinery import ModuleSpec -import importlib.util -import sys -from types import ModuleType -from typing import Iterable, Sequence - - -class VendorImporter: - """ - A PEP 302 meta path importer for finding optionally-vendored - or otherwise naturally-installed packages from root_name. - """ - - def __init__( - self, - root_name: str, - vendored_names: Iterable[str] = (), - vendor_pkg: str | None = None, - ): - self.root_name = root_name - self.vendored_names = set(vendored_names) - self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') - - @property - def search_path(self): - """ - Search first the vendor package then as a natural package. - """ - yield self.vendor_pkg + '.' - yield '' - - def _module_matches_namespace(self, fullname): - """Figure out if the target module is vendored.""" - root, base, target = fullname.partition(self.root_name + '.') - return not root and any(map(target.startswith, self.vendored_names)) - - def load_module(self, fullname: str): - """ - Iterate over the search path to locate and load fullname. - """ - root, base, target = fullname.partition(self.root_name + '.') - for prefix in self.search_path: - extant = prefix + target - try: - __import__(extant) - except ImportError: - continue - mod = sys.modules[extant] - sys.modules[fullname] = mod - return mod - else: - raise ImportError( - "The '{target}' package is required; " - "normally this is bundled with this package so if you get " - "this warning, consult the packager of your " - "distribution.".format(**locals()) - ) - - def create_module(self, spec: ModuleSpec): - return self.load_module(spec.name) - - def exec_module(self, module: ModuleType): - pass - - def find_spec( - self, - fullname: str, - path: Sequence[str] | None = None, - target: ModuleType | None = None, - ): - """Return a module spec for vendored names.""" - return ( - # This should fix itself next mypy release https://github.com/python/typeshed/pull/11890 - importlib.util.spec_from_loader(fullname, self) # type: ignore[arg-type] - if self._module_matches_namespace(fullname) - else None - ) - - def install(self): - """ - Install this importer into sys.meta_path if not already present. - """ - if self not in sys.meta_path: - sys.meta_path.append(self) - - -# [[[cog -# import cog -# from tools.vendored import yield_top_level -# names = "\n".join(f" {x!r}," for x in yield_top_level('pkg_resources')) -# cog.outl(f"names = (\n{names}\n)") -# ]]] -names = ( - 'backports', - 'importlib_resources', - 'jaraco', - 'more_itertools', - 'packaging', - 'platformdirs', - 'zipp', -) -# [[[end]]] -VendorImporter(__name__, names).install() diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/py.typed b/contrib/python/setuptools/py3/pkg_resources/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/platformdirs/py.typed +++ b/contrib/python/setuptools/py3/pkg_resources/py.typed diff --git a/contrib/python/setuptools/py3/setuptools/__init__.py b/contrib/python/setuptools/py3/setuptools/__init__.py index bf03f37b779..afca08be9c5 100644 --- a/contrib/python/setuptools/py3/setuptools/__init__.py +++ b/contrib/python/setuptools/py3/setuptools/__init__.py @@ -3,8 +3,13 @@ import functools import os import re +import sys from typing import TYPE_CHECKING +sys.path.extend(((vendor_path := os.path.join(os.path.dirname(os.path.dirname(__file__)), 'setuptools', '_vendor')) not in sys.path) * [vendor_path]) # fmt: skip +# workaround for #4476 +sys.modules.pop('backports', None) + import _distutils_hack.override # noqa: F401 import distutils.core from distutils.errors import DistutilsOptionError diff --git a/contrib/python/setuptools/py3/setuptools/_core_metadata.py b/contrib/python/setuptools/py3/setuptools/_core_metadata.py index 45aae7d70be..82ec19fc755 100644 --- a/contrib/python/setuptools/py3/setuptools/_core_metadata.py +++ b/contrib/python/setuptools/py3/setuptools/_core_metadata.py @@ -16,10 +16,10 @@ from tempfile import NamedTemporaryFile from distutils.util import rfc822_escape from . import _normalization, _reqs -from .extern.packaging.markers import Marker -from .extern.packaging.requirements import Requirement -from .extern.packaging.utils import canonicalize_name, canonicalize_version -from .extern.packaging.version import Version +from packaging.markers import Marker +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name, canonicalize_version +from packaging.version import Version from .warnings import SetuptoolsDeprecationWarning diff --git a/contrib/python/setuptools/py3/setuptools/_entry_points.py b/contrib/python/setuptools/py3/setuptools/_entry_points.py index b244e78387b..5de12582beb 100644 --- a/contrib/python/setuptools/py3/setuptools/_entry_points.py +++ b/contrib/python/setuptools/py3/setuptools/_entry_points.py @@ -3,11 +3,11 @@ import operator import itertools from .errors import OptionError -from .extern.jaraco.text import yield_lines -from .extern.jaraco.functools import pass_none +from jaraco.text import yield_lines +from jaraco.functools import pass_none from ._importlib import metadata from ._itertools import ensure_unique -from .extern.more_itertools import consume +from more_itertools import consume def ensure_valid(ep): diff --git a/contrib/python/setuptools/py3/setuptools/_importlib.py b/contrib/python/setuptools/py3/setuptools/_importlib.py index 8e52888d6f7..b2d5b5b84af 100644 --- a/contrib/python/setuptools/py3/setuptools/_importlib.py +++ b/contrib/python/setuptools/py3/setuptools/_importlib.py @@ -1,51 +1,13 @@ import sys -def disable_importlib_metadata_finder(metadata): - """ - Ensure importlib_metadata doesn't provide older, incompatible - Distributions. - - Workaround for #3102. - """ - try: - import importlib_metadata - except ImportError: - return - except AttributeError: - from .warnings import SetuptoolsWarning - - SetuptoolsWarning.emit( - "Incompatibility problem.", - """ - `importlib-metadata` version is incompatible with `setuptools`. - This problem is likely to be solved by installing an updated version of - `importlib-metadata`. - """, - see_url="https://github.com/python/importlib_metadata/issues/396", - ) # Ensure a descriptive message is shown. - raise # This exception can be suppressed by _distutils_hack - - if importlib_metadata is metadata: - return - to_remove = [ - ob - for ob in sys.meta_path - if isinstance(ob, importlib_metadata.MetadataPathFinder) - ] - for item in to_remove: - sys.meta_path.remove(item) - - if sys.version_info < (3, 10): - from setuptools.extern import importlib_metadata as metadata - - disable_importlib_metadata_finder(metadata) + import importlib_metadata as metadata # pragma: no cover else: - import importlib.metadata as metadata + import importlib.metadata as metadata # noqa: F401 if sys.version_info < (3, 9): - from setuptools.extern import importlib_resources as resources + import importlib_resources as resources # pragma: no cover else: import importlib.resources as resources # noqa: F401 diff --git a/contrib/python/setuptools/py3/setuptools/_itertools.py b/contrib/python/setuptools/py3/setuptools/_itertools.py index b8bf6d210ae..d6ca841353c 100644 --- a/contrib/python/setuptools/py3/setuptools/_itertools.py +++ b/contrib/python/setuptools/py3/setuptools/_itertools.py @@ -1,4 +1,4 @@ -from setuptools.extern.more_itertools import consume # noqa: F401 +from more_itertools import consume # noqa: F401 # copied from jaraco.itertools 6.1 diff --git a/contrib/python/setuptools/py3/setuptools/_normalization.py b/contrib/python/setuptools/py3/setuptools/_normalization.py index e858052ccd3..467b643d463 100644 --- a/contrib/python/setuptools/py3/setuptools/_normalization.py +++ b/contrib/python/setuptools/py3/setuptools/_normalization.py @@ -5,7 +5,7 @@ and core metadata import re -from .extern import packaging +import packaging # https://packaging.python.org/en/latest/specifications/core-metadata/#name _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I) @@ -54,7 +54,7 @@ def safe_version(version: str) -> str: >>> safe_version("ubuntu lts") Traceback (most recent call last): ... - setuptools.extern.packaging.version.InvalidVersion: Invalid version: 'ubuntu.lts' + packaging.version.InvalidVersion: Invalid version: 'ubuntu.lts' """ v = version.replace(' ', '.') try: diff --git a/contrib/python/setuptools/py3/setuptools/_reqs.py b/contrib/python/setuptools/py3/setuptools/_reqs.py index 9f83437033c..1b64d9df792 100644 --- a/contrib/python/setuptools/py3/setuptools/_reqs.py +++ b/contrib/python/setuptools/py3/setuptools/_reqs.py @@ -1,8 +1,8 @@ from functools import lru_cache from typing import Callable, Iterable, Iterator, TypeVar, Union, overload -import setuptools.extern.jaraco.text as text -from setuptools.extern.packaging.requirements import Requirement +import jaraco.text as text +from packaging.requirements import Requirement _T = TypeVar("_T") _StrOrIter = Union[str, Iterable[str]] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/backports/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py b/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py deleted file mode 100644 index a7a9a6e7b94..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/backports/tarfile.py +++ /dev/null @@ -1,2900 +0,0 @@ -#!/usr/bin/env python3 -#------------------------------------------------------------------- -# tarfile.py -#------------------------------------------------------------------- -# Copyright (C) 2002 Lars Gustaebel <lars@gustaebel.de> -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -"""Read from and write to tar format archives. -""" - -version = "0.9.0" -__author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" -__credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." - -#--------- -# Imports -#--------- -from builtins import open as bltn_open -import sys -import os -import io -import shutil -import stat -import time -import struct -import copy -import re -import warnings - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -# os.symlink on Windows prior to 6.0 raises NotImplementedError -# OSError (winerror=1314) will be raised if the caller does not hold the -# SeCreateSymbolicLinkPrivilege privilege -symlink_exception = (AttributeError, NotImplementedError, OSError) - -# from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", - "CompressionError", "StreamError", "ExtractError", "HeaderError", - "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", - "tar_filter", "FilterError", "AbsoluteLinkError", - "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] - - -#--------------------------------------------------------- -# tar constants -#--------------------------------------------------------- -NUL = b"\0" # the null character -BLOCKSIZE = 512 # length of processing blocks -RECORDSIZE = BLOCKSIZE * 20 # length of records -GNU_MAGIC = b"ustar \0" # magic gnu tar string -POSIX_MAGIC = b"ustar\x0000" # magic posix tar string - -LENGTH_NAME = 100 # maximum length of a filename -LENGTH_LINK = 100 # maximum length of a linkname -LENGTH_PREFIX = 155 # maximum length of the prefix field - -REGTYPE = b"0" # regular file -AREGTYPE = b"\0" # regular file -LNKTYPE = b"1" # link (inside tarfile) -SYMTYPE = b"2" # symbolic link -CHRTYPE = b"3" # character special device -BLKTYPE = b"4" # block special device -DIRTYPE = b"5" # directory -FIFOTYPE = b"6" # fifo special device -CONTTYPE = b"7" # contiguous file - -GNUTYPE_LONGNAME = b"L" # GNU tar longname -GNUTYPE_LONGLINK = b"K" # GNU tar longlink -GNUTYPE_SPARSE = b"S" # GNU tar sparse file - -XHDTYPE = b"x" # POSIX.1-2001 extended header -XGLTYPE = b"g" # POSIX.1-2001 global header -SOLARIS_XHDTYPE = b"X" # Solaris extended header - -USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format -GNU_FORMAT = 1 # GNU tar format -PAX_FORMAT = 2 # POSIX.1-2001 (pax) format -DEFAULT_FORMAT = PAX_FORMAT - -#--------------------------------------------------------- -# tarfile constants -#--------------------------------------------------------- -# File types that tarfile supports: -SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, - SYMTYPE, DIRTYPE, FIFOTYPE, - CONTTYPE, CHRTYPE, BLKTYPE, - GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# File types that will be treated as a regular file. -REGULAR_TYPES = (REGTYPE, AREGTYPE, - CONTTYPE, GNUTYPE_SPARSE) - -# File types that are part of the GNU tar format. -GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# Fields from a pax header that override a TarInfo attribute. -PAX_FIELDS = ("path", "linkpath", "size", "mtime", - "uid", "gid", "uname", "gname") - -# Fields from a pax header that are affected by hdrcharset. -PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"} - -# Fields in a pax header that are numbers, all other fields -# are treated as strings. -PAX_NUMBER_FIELDS = { - "atime": float, - "ctime": float, - "mtime": float, - "uid": int, - "gid": int, - "size": int -} - -#--------------------------------------------------------- -# initialization -#--------------------------------------------------------- -if os.name == "nt": - ENCODING = "utf-8" -else: - ENCODING = sys.getfilesystemencoding() - -#--------------------------------------------------------- -# Some useful functions -#--------------------------------------------------------- - -def stn(s, length, encoding, errors): - """Convert a string to a null-terminated bytes object. - """ - if s is None: - raise ValueError("metadata cannot contain None") - s = s.encode(encoding, errors) - return s[:length] + (length - len(s)) * NUL - -def nts(s, encoding, errors): - """Convert a null-terminated bytes object to a string. - """ - p = s.find(b"\0") - if p != -1: - s = s[:p] - return s.decode(encoding, errors) - -def nti(s): - """Convert a number field to a python number. - """ - # There are two possible encodings for a number field, see - # itn() below. - if s[0] in (0o200, 0o377): - n = 0 - for i in range(len(s) - 1): - n <<= 8 - n += s[i + 1] - if s[0] == 0o377: - n = -(256 ** (len(s) - 1) - n) - else: - try: - s = nts(s, "ascii", "strict") - n = int(s.strip() or "0", 8) - except ValueError: - raise InvalidHeaderError("invalid header") - return n - -def itn(n, digits=8, format=DEFAULT_FORMAT): - """Convert a python number to a number field. - """ - # POSIX 1003.1-1988 requires numbers to be encoded as a string of - # octal digits followed by a null-byte, this allows values up to - # (8**(digits-1))-1. GNU tar allows storing numbers greater than - # that if necessary. A leading 0o200 or 0o377 byte indicate this - # particular encoding, the following digits-1 bytes are a big-endian - # base-256 representation. This allows values up to (256**(digits-1))-1. - # A 0o200 byte indicates a positive number, a 0o377 byte a negative - # number. - original_n = n - n = int(n) - if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL - elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): - if n >= 0: - s = bytearray([0o200]) - else: - s = bytearray([0o377]) - n = 256 ** digits + n - - for i in range(digits - 1): - s.insert(1, n & 0o377) - n >>= 8 - else: - raise ValueError("overflow in number field") - - return s - -def calc_chksums(buf): - """Calculate the checksum for a member's header by summing up all - characters except for the chksum field which is treated as if - it was filled with spaces. According to the GNU tar sources, - some tars (Sun and NeXT) calculate chksum with signed char, - which will be different if there are chars in the buffer with - the high bit set. So we calculate two checksums, unsigned and - signed. - """ - unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf)) - signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf)) - return unsigned_chksum, signed_chksum - -def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): - """Copy length bytes from fileobj src to fileobj dst. - If length is None, copy the entire content. - """ - bufsize = bufsize or 16 * 1024 - if length == 0: - return - if length is None: - shutil.copyfileobj(src, dst, bufsize) - return - - blocks, remainder = divmod(length, bufsize) - for b in range(blocks): - buf = src.read(bufsize) - if len(buf) < bufsize: - raise exception("unexpected end of data") - dst.write(buf) - - if remainder != 0: - buf = src.read(remainder) - if len(buf) < remainder: - raise exception("unexpected end of data") - dst.write(buf) - return - -def _safe_print(s): - encoding = getattr(sys.stdout, 'encoding', None) - if encoding is not None: - s = s.encode(encoding, 'backslashreplace').decode(encoding) - print(s, end=' ') - - -class TarError(Exception): - """Base exception.""" - pass -class ExtractError(TarError): - """General exception for extract errors.""" - pass -class ReadError(TarError): - """Exception for unreadable tar archives.""" - pass -class CompressionError(TarError): - """Exception for unavailable compression methods.""" - pass -class StreamError(TarError): - """Exception for unsupported operations on stream-like TarFiles.""" - pass -class HeaderError(TarError): - """Base exception for header errors.""" - pass -class EmptyHeaderError(HeaderError): - """Exception for empty headers.""" - pass -class TruncatedHeaderError(HeaderError): - """Exception for truncated headers.""" - pass -class EOFHeaderError(HeaderError): - """Exception for end of file headers.""" - pass -class InvalidHeaderError(HeaderError): - """Exception for invalid headers.""" - pass -class SubsequentHeaderError(HeaderError): - """Exception for missing and invalid extended headers.""" - pass - -#--------------------------- -# internal stream interface -#--------------------------- -class _LowLevelFile: - """Low-level file object. Supports reading and writing. - It is used instead of a regular file object for streaming - access. - """ - - def __init__(self, name, mode): - mode = { - "r": os.O_RDONLY, - "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - }[mode] - if hasattr(os, "O_BINARY"): - mode |= os.O_BINARY - self.fd = os.open(name, mode, 0o666) - - def close(self): - os.close(self.fd) - - def read(self, size): - return os.read(self.fd, size) - - def write(self, s): - os.write(self.fd, s) - -class _Stream: - """Class that serves as an adapter between TarFile and - a stream-like object. The stream-like object only - needs to have a read() or write() method that works with bytes, - and the method is accessed blockwise. - Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin.buffer, - sys.stdout.buffer, a socket, a tape device etc. - - _Stream is intended to be used only internally. - """ - - def __init__(self, name, mode, comptype, fileobj, bufsize, - compresslevel): - """Construct a _Stream object. - """ - self._extfileobj = True - if fileobj is None: - fileobj = _LowLevelFile(name, mode) - self._extfileobj = False - - if comptype == '*': - # Enable transparent compression detection for the - # stream interface - fileobj = _StreamProxy(fileobj) - comptype = fileobj.getcomptype() - - self.name = name or "" - self.mode = mode - self.comptype = comptype - self.fileobj = fileobj - self.bufsize = bufsize - self.buf = b"" - self.pos = 0 - self.closed = False - - try: - if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") from None - self.zlib = zlib - self.crc = zlib.crc32(b"") - if mode == "r": - self.exception = zlib.error - self._init_read_gz() - else: - self._init_write_gz(compresslevel) - - elif comptype == "bz2": - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = bz2.BZ2Decompressor() - self.exception = OSError - else: - self.cmp = bz2.BZ2Compressor(compresslevel) - - elif comptype == "xz": - try: - import lzma - except ImportError: - raise CompressionError("lzma module is not available") from None - if mode == "r": - self.dbuf = b"" - self.cmp = lzma.LZMADecompressor() - self.exception = lzma.LZMAError - else: - self.cmp = lzma.LZMACompressor() - - elif comptype != "tar": - raise CompressionError("unknown compression type %r" % comptype) - - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - def __del__(self): - if hasattr(self, "closed") and not self.closed: - self.close() - - def _init_write_gz(self, compresslevel): - """Initialize for writing with gzip compression. - """ - self.cmp = self.zlib.compressobj(compresslevel, - self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) - timestamp = struct.pack("<L", int(time.time())) - self.__write(b"\037\213\010\010" + timestamp + b"\002\377") - if self.name.endswith(".gz"): - self.name = self.name[:-3] - # Honor "directory components removed" from RFC1952 - self.name = os.path.basename(self.name) - # RFC1952 says we must use ISO-8859-1 for the FNAME field. - self.__write(self.name.encode("iso-8859-1", "replace") + NUL) - - def write(self, s): - """Write string s to the stream. - """ - if self.comptype == "gz": - self.crc = self.zlib.crc32(s, self.crc) - self.pos += len(s) - if self.comptype != "tar": - s = self.cmp.compress(s) - self.__write(s) - - def __write(self, s): - """Write string s to the stream if a whole new block - is ready to be written. - """ - self.buf += s - while len(self.buf) > self.bufsize: - self.fileobj.write(self.buf[:self.bufsize]) - self.buf = self.buf[self.bufsize:] - - def close(self): - """Close the _Stream object. No operation should be - done on it afterwards. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode == "w" and self.comptype != "tar": - self.buf += self.cmp.flush() - - if self.mode == "w" and self.buf: - self.fileobj.write(self.buf) - self.buf = b"" - if self.comptype == "gz": - self.fileobj.write(struct.pack("<L", self.crc)) - self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def _init_read_gz(self): - """Initialize for reading a gzip compressed fileobj. - """ - self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" - - # taken from gzip.GzipFile with some alterations - if self.__read(2) != b"\037\213": - raise ReadError("not a gzip file") - if self.__read(1) != b"\010": - raise CompressionError("unsupported compression method") - - flag = ord(self.__read(1)) - self.__read(6) - - if flag & 4: - xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) - if flag & 8: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 16: - while True: - s = self.__read(1) - if not s or s == NUL: - break - if flag & 2: - self.__read(2) - - def tell(self): - """Return the stream's file pointer position. - """ - return self.pos - - def seek(self, pos=0): - """Set the stream's file pointer to pos. Negative seeking - is forbidden. - """ - if pos - self.pos >= 0: - blocks, remainder = divmod(pos - self.pos, self.bufsize) - for i in range(blocks): - self.read(self.bufsize) - self.read(remainder) - else: - raise StreamError("seeking backwards is not allowed") - return self.pos - - def read(self, size): - """Return the next size number of bytes from the stream.""" - assert size is not None - buf = self._read(size) - self.pos += len(buf) - return buf - - def _read(self, size): - """Return size bytes from the stream. - """ - if self.comptype == "tar": - return self.__read(size) - - c = len(self.dbuf) - t = [self.dbuf] - while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" - else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] - - def __read(self, size): - """Return size bytes from stream. If internal buffer is empty, - read another block from the stream. - """ - c = len(self.buf) - t = [self.buf] - while c < size: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - t.append(buf) - c += len(buf) - t = b"".join(t) - self.buf = t[size:] - return t[:size] -# class _Stream - -class _StreamProxy(object): - """Small proxy class that enables transparent compression - detection for the Stream interface (mode 'r|*'). - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - self.buf = self.fileobj.read(BLOCKSIZE) - - def read(self, size): - self.read = self.fileobj.read - return self.buf - - def getcomptype(self): - if self.buf.startswith(b"\x1f\x8b\x08"): - return "gz" - elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY": - return "bz2" - elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): - return "xz" - else: - return "tar" - - def close(self): - self.fileobj.close() -# class StreamProxy - -#------------------------ -# Extraction file object -#------------------------ -class _FileInFile(object): - """A thin wrapper around an existing file object that - provides a part of its data as an individual file - object. - """ - - def __init__(self, fileobj, offset, size, name, blockinfo=None): - self.fileobj = fileobj - self.offset = offset - self.size = size - self.position = 0 - self.name = name - self.closed = False - - if blockinfo is None: - blockinfo = [(0, size)] - - # Construct a map with data and zero blocks. - self.map_index = 0 - self.map = [] - lastpos = 0 - realpos = self.offset - for offset, size in blockinfo: - if offset > lastpos: - self.map.append((False, lastpos, offset, None)) - self.map.append((True, offset, offset + size, realpos)) - realpos += size - lastpos = offset + size - if lastpos < self.size: - self.map.append((False, lastpos, self.size, None)) - - def flush(self): - pass - - def readable(self): - return True - - def writable(self): - return False - - def seekable(self): - return self.fileobj.seekable() - - def tell(self): - """Return the current file position. - """ - return self.position - - def seek(self, position, whence=io.SEEK_SET): - """Seek to a position in the file. - """ - if whence == io.SEEK_SET: - self.position = min(max(position, 0), self.size) - elif whence == io.SEEK_CUR: - if position < 0: - self.position = max(self.position + position, 0) - else: - self.position = min(self.position + position, self.size) - elif whence == io.SEEK_END: - self.position = max(min(self.size + position, self.size), 0) - else: - raise ValueError("Invalid argument") - return self.position - - def read(self, size=None): - """Read data from the file. - """ - if size is None: - size = self.size - self.position - else: - size = min(size, self.size - self.position) - - buf = b"" - while size > 0: - while True: - data, start, stop, offset = self.map[self.map_index] - if start <= self.position < stop: - break - else: - self.map_index += 1 - if self.map_index == len(self.map): - self.map_index = 0 - length = min(size, stop - self.position) - if data: - self.fileobj.seek(offset + (self.position - start)) - b = self.fileobj.read(length) - if len(b) != length: - raise ReadError("unexpected end of data") - buf += b - else: - buf += NUL * length - size -= length - self.position += length - return buf - - def readinto(self, b): - buf = self.read(len(b)) - b[:len(buf)] = buf - return len(buf) - - def close(self): - self.closed = True -#class _FileInFile - -class ExFileObject(io.BufferedReader): - - def __init__(self, tarfile, tarinfo): - fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data, - tarinfo.size, tarinfo.name, tarinfo.sparse) - super().__init__(fileobj) -#class ExFileObject - - -#----------------------------- -# extraction filters (PEP 706) -#----------------------------- - -class FilterError(TarError): - pass - -class AbsolutePathError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'member {tarinfo.name!r} has an absolute path') - -class OutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, ' - + 'which is outside the destination') - -class SpecialFileError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a special file') - -class AbsoluteLinkError(FilterError): - def __init__(self, tarinfo): - self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a link to an absolute path') - -class LinkOutsideDestinationError(FilterError): - def __init__(self, tarinfo, path): - self.tarinfo = tarinfo - self._path = path - super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' - + 'which is outside the destination') - -def _get_filtered_attrs(member, dest_path, for_data=True): - new_attrs = {} - name = member.name - dest_path = os.path.realpath(dest_path) - # Strip leading / (tar's directory separator) from filenames. - # Include os.sep (target OS directory separator) as well. - if name.startswith(('/', os.sep)): - name = new_attrs['name'] = member.path.lstrip('/' + os.sep) - if os.path.isabs(name): - # Path is absolute even after stripping. - # For example, 'C:/foo' on Windows. - raise AbsolutePathError(member) - # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise OutsideDestinationError(member, target_path) - # Limit permissions (no high bits, and go-w) - mode = member.mode - if mode is not None: - # Strip high bits & group/other write bits - mode = mode & 0o755 - if for_data: - # For data, handle permissions & file types - if member.isreg() or member.islnk(): - if not mode & 0o100: - # Clear executable bits if not executable by user - mode &= ~0o111 - # Ensure owner can read & write - mode |= 0o600 - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Reject special files - raise SpecialFileError(member) - if mode != member.mode: - new_attrs['mode'] = mode - if for_data: - # Ignore ownership for 'data' - if member.uid is not None: - new_attrs['uid'] = None - if member.gid is not None: - new_attrs['gid'] = None - if member.uname is not None: - new_attrs['uname'] = None - if member.gname is not None: - new_attrs['gname'] = None - # Check link destination for 'data' - if member.islnk() or member.issym(): - if os.path.isabs(member.linkname): - raise AbsoluteLinkError(member) - if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) - else: - target_path = os.path.join(dest_path, - member.linkname) - target_path = os.path.realpath(target_path) - if os.path.commonpath([target_path, dest_path]) != dest_path: - raise LinkOutsideDestinationError(member, target_path) - return new_attrs - -def fully_trusted_filter(member, dest_path): - return member - -def tar_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, False) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -def data_filter(member, dest_path): - new_attrs = _get_filtered_attrs(member, dest_path, True) - if new_attrs: - return member.replace(**new_attrs, deep=False) - return member - -_NAMED_FILTERS = { - "fully_trusted": fully_trusted_filter, - "tar": tar_filter, - "data": data_filter, -} - -#------------------ -# Exported Classes -#------------------ - -# Sentinel for replace() defaults, meaning "don't change the attribute" -_KEEP = object() - -class TarInfo(object): - """Informational class which holds the details about an - archive member given by a tar header block. - TarInfo objects are returned by TarFile.getmember(), - TarFile.getmembers() and TarFile.gettarinfo() and are - usually created internally. - """ - - __slots__ = dict( - name = 'Name of the archive member.', - mode = 'Permission bits.', - uid = 'User ID of the user who originally stored this member.', - gid = 'Group ID of the user who originally stored this member.', - size = 'Size in bytes.', - mtime = 'Time of last modification.', - chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), - linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), - uname = 'User name.', - gname = 'Group name.', - devmajor = 'Device major number.', - devminor = 'Device minor number.', - offset = 'The tar header starts here.', - offset_data = "The file's data starts here.", - pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), - sparse = 'Sparse member information.', - tarfile = None, - _sparse_structs = None, - _link_target = None, - ) - - def __init__(self, name=""): - """Construct a TarInfo object. name is the optional name - of the member. - """ - self.name = name # member name - self.mode = 0o644 # file permissions - self.uid = 0 # user id - self.gid = 0 # group id - self.size = 0 # file size - self.mtime = 0 # modification time - self.chksum = 0 # header checksum - self.type = REGTYPE # member type - self.linkname = "" # link name - self.uname = "" # user name - self.gname = "" # group name - self.devmajor = 0 # device major number - self.devminor = 0 # device minor number - - self.offset = 0 # the tar header starts here - self.offset_data = 0 # the file's data starts here - - self.sparse = None # sparse member information - self.pax_headers = {} # pax header information - - @property - def path(self): - 'In pax headers, "name" is called "path".' - return self.name - - @path.setter - def path(self, name): - self.name = name - - @property - def linkpath(self): - 'In pax headers, "linkname" is called "linkpath".' - return self.linkname - - @linkpath.setter - def linkpath(self, linkname): - self.linkname = linkname - - def __repr__(self): - return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) - - def replace(self, *, - name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, - uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, - deep=True, _KEEP=_KEEP): - """Return a deep copy of self with the given attributes replaced. - """ - if deep: - result = copy.deepcopy(self) - else: - result = copy.copy(self) - if name is not _KEEP: - result.name = name - if mtime is not _KEEP: - result.mtime = mtime - if mode is not _KEEP: - result.mode = mode - if linkname is not _KEEP: - result.linkname = linkname - if uid is not _KEEP: - result.uid = uid - if gid is not _KEEP: - result.gid = gid - if uname is not _KEEP: - result.uname = uname - if gname is not _KEEP: - result.gname = gname - return result - - def get_info(self): - """Return the TarInfo's attributes as a dictionary. - """ - if self.mode is None: - mode = None - else: - mode = self.mode & 0o7777 - info = { - "name": self.name, - "mode": mode, - "uid": self.uid, - "gid": self.gid, - "size": self.size, - "mtime": self.mtime, - "chksum": self.chksum, - "type": self.type, - "linkname": self.linkname, - "uname": self.uname, - "gname": self.gname, - "devmajor": self.devmajor, - "devminor": self.devminor - } - - if info["type"] == DIRTYPE and not info["name"].endswith("/"): - info["name"] += "/" - - return info - - def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): - """Return a tar header as a string of 512 byte blocks. - """ - info = self.get_info() - for name, value in info.items(): - if value is None: - raise ValueError("%s may not be None" % name) - - if format == USTAR_FORMAT: - return self.create_ustar_header(info, encoding, errors) - elif format == GNU_FORMAT: - return self.create_gnu_header(info, encoding, errors) - elif format == PAX_FORMAT: - return self.create_pax_header(info, encoding) - else: - raise ValueError("invalid format") - - def create_ustar_header(self, info, encoding, errors): - """Return the object as a ustar header block. - """ - info["magic"] = POSIX_MAGIC - - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - raise ValueError("linkname is too long") - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - info["prefix"], info["name"] = self._posix_split_name(info["name"], encoding, errors) - - return self._create_header(info, USTAR_FORMAT, encoding, errors) - - def create_gnu_header(self, info, encoding, errors): - """Return the object as a GNU header block sequence. - """ - info["magic"] = GNU_MAGIC - - buf = b"" - if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: - buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) - - if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: - buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) - - return buf + self._create_header(info, GNU_FORMAT, encoding, errors) - - def create_pax_header(self, info, encoding): - """Return the object as a ustar header block. If it cannot be - represented this way, prepend a pax extended header sequence - with supplement information. - """ - info["magic"] = POSIX_MAGIC - pax_headers = self.pax_headers.copy() - - # Test string fields for values that exceed the field length or cannot - # be represented in ASCII encoding. - for name, hname, length in ( - ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), - ("uname", "uname", 32), ("gname", "gname", 32)): - - if hname in pax_headers: - # The pax header has priority. - continue - - # Try to encode the string as ASCII. - try: - info[name].encode("ascii", "strict") - except UnicodeEncodeError: - pax_headers[hname] = info[name] - continue - - if len(info[name]) > length: - pax_headers[hname] = info[name] - - # Test number fields for values that exceed the field limit or values - # that like to be stored as float. - for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): - needs_pax = False - - val = info[name] - val_is_float = isinstance(val, float) - val_int = round(val) if val_is_float else val - if not 0 <= val_int < 8 ** (digits - 1): - # Avoid overflow. - info[name] = 0 - needs_pax = True - elif val_is_float: - # Put rounded value in ustar header, and full - # precision value in pax header. - info[name] = val_int - needs_pax = True - - # The existing pax header has priority. - if needs_pax and name not in pax_headers: - pax_headers[name] = str(val) - - # Create a pax extended header if necessary. - if pax_headers: - buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) - else: - buf = b"" - - return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") - - @classmethod - def create_pax_global_header(cls, pax_headers): - """Return the object as a pax global header block sequence. - """ - return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") - - def _posix_split_name(self, name, encoding, errors): - """Split a name longer than 100 chars into a prefix - and a name part. - """ - components = name.split("/") - for i in range(1, len(components)): - prefix = "/".join(components[:i]) - name = "/".join(components[i:]) - if len(prefix.encode(encoding, errors)) <= LENGTH_PREFIX and \ - len(name.encode(encoding, errors)) <= LENGTH_NAME: - break - else: - raise ValueError("name is too long") - - return prefix, name - - @staticmethod - def _create_header(info, format, encoding, errors): - """Return a header block. info is a dictionary with file - information, format must be one of the *_FORMAT constants. - """ - has_device_fields = info.get("type") in (CHRTYPE, BLKTYPE) - if has_device_fields: - devmajor = itn(info.get("devmajor", 0), 8, format) - devminor = itn(info.get("devminor", 0), 8, format) - else: - devmajor = stn("", 8, encoding, errors) - devminor = stn("", 8, encoding, errors) - - # None values in metadata should cause ValueError. - # itn()/stn() do this for all fields except type. - filetype = info.get("type", REGTYPE) - if filetype is None: - raise ValueError("TarInfo.type must not be None") - - parts = [ - stn(info.get("name", ""), 100, encoding, errors), - itn(info.get("mode", 0) & 0o7777, 8, format), - itn(info.get("uid", 0), 8, format), - itn(info.get("gid", 0), 8, format), - itn(info.get("size", 0), 12, format), - itn(info.get("mtime", 0), 12, format), - b" ", # checksum field - filetype, - stn(info.get("linkname", ""), 100, encoding, errors), - info.get("magic", POSIX_MAGIC), - stn(info.get("uname", ""), 32, encoding, errors), - stn(info.get("gname", ""), 32, encoding, errors), - devmajor, - devminor, - stn(info.get("prefix", ""), 155, encoding, errors) - ] - - buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) - chksum = calc_chksums(buf[-BLOCKSIZE:])[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - - @staticmethod - def _create_payload(payload): - """Return the string payload filled with zero bytes - up to the next 512 byte border. - """ - blocks, remainder = divmod(len(payload), BLOCKSIZE) - if remainder > 0: - payload += (BLOCKSIZE - remainder) * NUL - return payload - - @classmethod - def _create_gnu_long_header(cls, name, type, encoding, errors): - """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence - for name. - """ - name = name.encode(encoding, errors) + NUL - - info = {} - info["name"] = "././@LongLink" - info["type"] = type - info["size"] = len(name) - info["magic"] = GNU_MAGIC - - # create extended header + name blocks. - return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ - cls._create_payload(name) - - @classmethod - def _create_pax_generic_header(cls, pax_headers, type, encoding): - """Return a POSIX.1-2008 extended or global header sequence - that contains a list of keyword, value pairs. The values - must be strings. - """ - # Check if one of the fields contains surrogate characters and thereby - # forces hdrcharset=BINARY, see _proc_pax() for more information. - binary = False - for keyword, value in pax_headers.items(): - try: - value.encode("utf-8", "strict") - except UnicodeEncodeError: - binary = True - break - - records = b"" - if binary: - # Put the hdrcharset field at the beginning of the header. - records += b"21 hdrcharset=BINARY\n" - - for keyword, value in pax_headers.items(): - keyword = keyword.encode("utf-8") - if binary: - # Try to restore the original byte representation of `value'. - # Needless to say, that the encoding must match the string. - value = value.encode(encoding, "surrogateescape") - else: - value = value.encode("utf-8") - - l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' - n = p = 0 - while True: - n = l + len(str(p)) - if n == p: - break - p = n - records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" - - # We use a hardcoded "././@PaxHeader" name like star does - # instead of the one that POSIX recommends. - info = {} - info["name"] = "././@PaxHeader" - info["type"] = type - info["size"] = len(records) - info["magic"] = POSIX_MAGIC - - # Create pax header + record blocks. - return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ - cls._create_payload(records) - - @classmethod - def frombuf(cls, buf, encoding, errors): - """Construct a TarInfo object from a 512 byte bytes object. - """ - if len(buf) == 0: - raise EmptyHeaderError("empty header") - if len(buf) != BLOCKSIZE: - raise TruncatedHeaderError("truncated header") - if buf.count(NUL) == BLOCKSIZE: - raise EOFHeaderError("end of file header") - - chksum = nti(buf[148:156]) - if chksum not in calc_chksums(buf): - raise InvalidHeaderError("bad checksum") - - obj = cls() - obj.name = nts(buf[0:100], encoding, errors) - obj.mode = nti(buf[100:108]) - obj.uid = nti(buf[108:116]) - obj.gid = nti(buf[116:124]) - obj.size = nti(buf[124:136]) - obj.mtime = nti(buf[136:148]) - obj.chksum = chksum - obj.type = buf[156:157] - obj.linkname = nts(buf[157:257], encoding, errors) - obj.uname = nts(buf[265:297], encoding, errors) - obj.gname = nts(buf[297:329], encoding, errors) - obj.devmajor = nti(buf[329:337]) - obj.devminor = nti(buf[337:345]) - prefix = nts(buf[345:500], encoding, errors) - - # Old V7 tar format represents a directory as a regular - # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): - obj.type = DIRTYPE - - # The old GNU sparse format occupies some of the unused - # space in the buffer for up to 4 sparse structures. - # Save them for later processing in _proc_sparse(). - if obj.type == GNUTYPE_SPARSE: - pos = 386 - structs = [] - for i in range(4): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[482]) - origsize = nti(buf[483:495]) - obj._sparse_structs = (structs, isextended, origsize) - - # Remove redundant slashes from directories. - if obj.isdir(): - obj.name = obj.name.rstrip("/") - - # Reconstruct a ustar longname. - if prefix and obj.type not in GNU_TYPES: - obj.name = prefix + "/" + obj.name - return obj - - @classmethod - def fromtarfile(cls, tarfile): - """Return the next TarInfo object from TarFile object - tarfile. - """ - buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) - obj.offset = tarfile.fileobj.tell() - BLOCKSIZE - return obj._proc_member(tarfile) - - #-------------------------------------------------------------------------- - # The following are methods that are called depending on the type of a - # member. The entry point is _proc_member() which can be overridden in a - # subclass to add custom _proc_*() methods. A _proc_*() method MUST - # implement the following - # operations: - # 1. Set self.offset_data to the position where the data blocks begin, - # if there is data that follows. - # 2. Set tarfile.offset to the position where the next member's header will - # begin. - # 3. Return self or another valid TarInfo object. - def _proc_member(self, tarfile): - """Choose the right processing method depending on - the type and call it. - """ - if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): - return self._proc_gnulong(tarfile) - elif self.type == GNUTYPE_SPARSE: - return self._proc_sparse(tarfile) - elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): - return self._proc_pax(tarfile) - else: - return self._proc_builtin(tarfile) - - def _proc_builtin(self, tarfile): - """Process a builtin type or an unknown type which - will be treated as a regular file. - """ - self.offset_data = tarfile.fileobj.tell() - offset = self.offset_data - if self.isreg() or self.type not in SUPPORTED_TYPES: - # Skip the following data blocks. - offset += self._block(self.size) - tarfile.offset = offset - - # Patch the TarInfo object with saved global - # header information. - self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if self.isdir(): - self.name = self.name.rstrip("/") - - return self - - def _proc_gnulong(self, tarfile): - """Process the blocks that hold a GNU longname - or longlink member. - """ - buf = tarfile.fileobj.read(self._block(self.size)) - - # Fetch the next header and process it. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Patch the TarInfo object from the next header with - # the longname information. - next.offset = self.offset - if self.type == GNUTYPE_LONGNAME: - next.name = nts(buf, tarfile.encoding, tarfile.errors) - elif self.type == GNUTYPE_LONGLINK: - next.linkname = nts(buf, tarfile.encoding, tarfile.errors) - - # Remove redundant slashes from directories. This is to be consistent - # with frombuf(). - if next.isdir(): - next.name = next.name.removesuffix("/") - - return next - - def _proc_sparse(self, tarfile): - """Process a GNU sparse header plus extra headers. - """ - # We already collected some sparse structures in frombuf(). - structs, isextended, origsize = self._sparse_structs - del self._sparse_structs - - # Collect sparse structures from extended header blocks. - while isextended: - buf = tarfile.fileobj.read(BLOCKSIZE) - pos = 0 - for i in range(21): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset and numbytes: - structs.append((offset, numbytes)) - pos += 24 - isextended = bool(buf[504]) - self.sparse = structs - - self.offset_data = tarfile.fileobj.tell() - tarfile.offset = self.offset_data + self._block(self.size) - self.size = origsize - return self - - def _proc_pax(self, tarfile): - """Process an extended or global header as described in - POSIX.1-2008. - """ - # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) - - # A pax header stores supplemental information for either - # the following file (extended) or all following files - # (global). - if self.type == XGLTYPE: - pax_headers = tarfile.pax_headers - else: - pax_headers = tarfile.pax_headers.copy() - - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - - # Parse pax header information. A record looks like that: - # "%d %s=%s\n" % (length, keyword, value). length is the size - # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") - pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: - raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] - - # Normally, we could just use "utf-8" as the encoding and "strict" - # as the error handler, but we better not take the risk. For - # example, GNU tar <= 1.23 is known to store filenames it cannot - # translate to UTF-8 as raw strings (unfortunately without a - # hdrcharset=BINARY header). - # We first try the strict standard encoding, and if that fails we - # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", - tarfile.errors) - if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, - tarfile.errors) - else: - value = self._decode_pax_field(value, "utf-8", "utf-8", - tarfile.errors) - - pax_headers[keyword] = value - pos += length - - # Fetch the next header. - try: - next = self.fromtarfile(tarfile) - except HeaderError as e: - raise SubsequentHeaderError(str(e)) from None - - # Process GNU sparse information. - if "GNU.sparse.map" in pax_headers: - # GNU extended sparse format version 0.1. - self._proc_gnusparse_01(next, pax_headers) - - elif "GNU.sparse.size" in pax_headers: - # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) - - elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": - # GNU extended sparse format version 1.0. - self._proc_gnusparse_10(next, pax_headers, tarfile) - - if self.type in (XHDTYPE, SOLARIS_XHDTYPE): - # Patch the TarInfo object with the extended header info. - next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset - - if "size" in pax_headers: - # If the extended header replaces the size field, - # we need to recalculate the offset where the next - # header starts. - offset = next.offset_data - if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) - tarfile.offset = offset - - return next - - def _proc_gnusparse_00(self, next, pax_headers, buf): - """Process a GNU tar extended sparse header, version 0.0. - """ - offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) - numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) - next.sparse = list(zip(offsets, numbytes)) - - def _proc_gnusparse_01(self, next, pax_headers): - """Process a GNU tar extended sparse header, version 0.1. - """ - sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _proc_gnusparse_10(self, next, pax_headers, tarfile): - """Process a GNU tar extended sparse header, version 1.0. - """ - fields = None - sparse = [] - buf = tarfile.fileobj.read(BLOCKSIZE) - fields, buf = buf.split(b"\n", 1) - fields = int(fields) - while len(sparse) < fields * 2: - if b"\n" not in buf: - buf += tarfile.fileobj.read(BLOCKSIZE) - number, buf = buf.split(b"\n", 1) - sparse.append(int(number)) - next.offset_data = tarfile.fileobj.tell() - next.sparse = list(zip(sparse[::2], sparse[1::2])) - - def _apply_pax_info(self, pax_headers, encoding, errors): - """Replace fields with supplemental information from a previous - pax extended or global header. - """ - for keyword, value in pax_headers.items(): - if keyword == "GNU.sparse.name": - setattr(self, "path", value) - elif keyword == "GNU.sparse.size": - setattr(self, "size", int(value)) - elif keyword == "GNU.sparse.realsize": - setattr(self, "size", int(value)) - elif keyword in PAX_FIELDS: - if keyword in PAX_NUMBER_FIELDS: - try: - value = PAX_NUMBER_FIELDS[keyword](value) - except ValueError: - value = 0 - if keyword == "path": - value = value.rstrip("/") - setattr(self, keyword, value) - - self.pax_headers = pax_headers.copy() - - def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): - """Decode a single field from a pax record. - """ - try: - return value.decode(encoding, "strict") - except UnicodeDecodeError: - return value.decode(fallback_encoding, fallback_errors) - - def _block(self, count): - """Round up a byte count by BLOCKSIZE and return it, - e.g. _block(834) => 1024. - """ - blocks, remainder = divmod(count, BLOCKSIZE) - if remainder: - blocks += 1 - return blocks * BLOCKSIZE - - def isreg(self): - 'Return True if the Tarinfo object is a regular file.' - return self.type in REGULAR_TYPES - - def isfile(self): - 'Return True if the Tarinfo object is a regular file.' - return self.isreg() - - def isdir(self): - 'Return True if it is a directory.' - return self.type == DIRTYPE - - def issym(self): - 'Return True if it is a symbolic link.' - return self.type == SYMTYPE - - def islnk(self): - 'Return True if it is a hard link.' - return self.type == LNKTYPE - - def ischr(self): - 'Return True if it is a character device.' - return self.type == CHRTYPE - - def isblk(self): - 'Return True if it is a block device.' - return self.type == BLKTYPE - - def isfifo(self): - 'Return True if it is a FIFO.' - return self.type == FIFOTYPE - - def issparse(self): - return self.sparse is not None - - def isdev(self): - 'Return True if it is one of character device, block device or FIFO.' - return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) -# class TarInfo - -class TarFile(object): - """The TarFile Class provides an interface to tar archives. - """ - - debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) - - dereference = False # If true, add content of linked file to the - # tar file, else the link. - - ignore_zeros = False # If true, skips empty or invalid blocks and - # continues processing. - - errorlevel = 1 # If 0, fatal errors only appear in debug - # messages (if debug >= 0). If > 0, errors - # are passed to the caller as exceptions. - - format = DEFAULT_FORMAT # The format to use when creating an archive. - - encoding = ENCODING # Encoding for 8-bit character strings. - - errors = None # Error handler for unicode conversion. - - tarinfo = TarInfo # The default TarInfo class to use. - - fileobject = ExFileObject # The file-object for extractfile(). - - extraction_filter = None # The default filter for extraction. - - def __init__(self, name=None, mode="r", fileobj=None, format=None, - tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, - errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to - read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' - defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. - """ - modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} - if mode not in modes: - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - self.mode = mode - self._mode = modes[mode] - - if not fileobj: - if self.mode == "a" and not os.path.exists(name): - # Create nonexistent files in append mode. - self.mode = "w" - self._mode = "wb" - fileobj = bltn_open(name, self._mode) - self._extfileobj = False - else: - if (name is None and hasattr(fileobj, "name") and - isinstance(fileobj.name, (str, bytes))): - name = fileobj.name - if hasattr(fileobj, "mode"): - self._mode = fileobj.mode - self._extfileobj = True - self.name = os.path.abspath(name) if name else None - self.fileobj = fileobj - - # Init attributes. - if format is not None: - self.format = format - if tarinfo is not None: - self.tarinfo = tarinfo - if dereference is not None: - self.dereference = dereference - if ignore_zeros is not None: - self.ignore_zeros = ignore_zeros - if encoding is not None: - self.encoding = encoding - self.errors = errors - - if pax_headers is not None and self.format == PAX_FORMAT: - self.pax_headers = pax_headers - else: - self.pax_headers = {} - - if debug is not None: - self.debug = debug - if errorlevel is not None: - self.errorlevel = errorlevel - - # Init datastructures. - self.copybufsize = copybufsize - self.closed = False - self.members = [] # list of members as TarInfo objects - self._loaded = False # flag if all members have been read - self.offset = self.fileobj.tell() - # current position in the archive file - self.inodes = {} # dictionary caching the inodes of - # archive members already added - - try: - if self.mode == "r": - self.firstmember = None - self.firstmember = self.next() - - if self.mode == "a": - # Move to the end of the archive, - # before the first empty block. - while True: - self.fileobj.seek(self.offset) - try: - tarinfo = self.tarinfo.fromtarfile(self) - self.members.append(tarinfo) - except EOFHeaderError: - self.fileobj.seek(self.offset) - break - except HeaderError as e: - raise ReadError(str(e)) from None - - if self.mode in ("a", "w", "x"): - self._loaded = True - - if self.pax_headers: - buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) - self.fileobj.write(buf) - self.offset += len(buf) - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - #-------------------------------------------------------------------------- - # Below are the classmethods which act as alternate constructors to the - # TarFile class. The open() method is the only one that is needed for - # public use; it is the "super"-constructor and is able to select an - # adequate "sub"-constructor for a particular compression using the mapping - # from OPEN_METH. - # - # This concept allows one to subclass TarFile without losing the comfort of - # the super-constructor. A sub-constructor is registered and made available - # by adding it to the mapping in OPEN_METH. - - @classmethod - def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): - r"""Open a tar archive for reading, writing or appending. Return - an appropriate TarFile class. - - mode: - 'r' or 'r:\*' open for reading with transparent compression - 'r:' open for reading exclusively uncompressed - 'r:gz' open for reading with gzip compression - 'r:bz2' open for reading with bzip2 compression - 'r:xz' open for reading with lzma compression - 'a' or 'a:' open for appending, creating the file if necessary - 'w' or 'w:' open for writing without compression - 'w:gz' open for writing with gzip compression - 'w:bz2' open for writing with bzip2 compression - 'w:xz' open for writing with lzma compression - - 'x' or 'x:' create a tarfile exclusively without compression, raise - an exception if the file is already created - 'x:gz' create a gzip compressed tarfile, raise an exception - if the file is already created - 'x:bz2' create a bzip2 compressed tarfile, raise an exception - if the file is already created - 'x:xz' create an lzma compressed tarfile, raise an exception - if the file is already created - - 'r|\*' open a stream of tar blocks with transparent compression - 'r|' open an uncompressed stream of tar blocks for reading - 'r|gz' open a gzip compressed stream of tar blocks - 'r|bz2' open a bzip2 compressed stream of tar blocks - 'r|xz' open an lzma compressed stream of tar blocks - 'w|' open an uncompressed stream for writing - 'w|gz' open a gzip compressed stream for writing - 'w|bz2' open a bzip2 compressed stream for writing - 'w|xz' open an lzma compressed stream for writing - """ - - if not name and not fileobj: - raise ValueError("nothing to open") - - if mode in ("r", "r:*"): - # Find out which *open() is appropriate for opening the file. - def not_compressed(comptype): - return cls.OPEN_METH[comptype] == 'taropen' - error_msgs = [] - for comptype in sorted(cls.OPEN_METH, key=not_compressed): - func = getattr(cls, cls.OPEN_METH[comptype]) - if fileobj is not None: - saved_pos = fileobj.tell() - try: - return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError) as e: - error_msgs.append(f'- method {comptype}: {e!r}') - if fileobj is not None: - fileobj.seek(saved_pos) - continue - error_msgs_summary = '\n'.join(error_msgs) - raise ReadError(f"file could not be opened successfully:\n{error_msgs_summary}") - - elif ":" in mode: - filemode, comptype = mode.split(":", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - # Select the *open() function according to - # given compression. - if comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - else: - raise CompressionError("unknown compression type %r" % comptype) - return func(name, filemode, fileobj, **kwargs) - - elif "|" in mode: - filemode, comptype = mode.split("|", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - if filemode not in ("r", "w"): - raise ValueError("mode must be 'r' or 'w'") - - compresslevel = kwargs.pop("compresslevel", 9) - stream = _Stream(name, filemode, comptype, fileobj, bufsize, - compresslevel) - try: - t = cls(name, filemode, stream, **kwargs) - except: - stream.close() - raise - t._extfileobj = False - return t - - elif mode in ("a", "w", "x"): - return cls.taropen(name, mode, fileobj, **kwargs) - - raise ValueError("undiscernible mode") - - @classmethod - def taropen(cls, name, mode="r", fileobj=None, **kwargs): - """Open uncompressed tar archive name for reading or writing. - """ - if mode not in ("r", "a", "w", "x"): - raise ValueError("mode must be 'r', 'a', 'w' or 'x'") - return cls(name, mode, fileobj, **kwargs) - - @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open gzip compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from gzip import GzipFile - except ImportError: - raise CompressionError("gzip module is not available") from None - - try: - fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) - except OSError as e: - if fileobj is not None and mode == 'r': - raise ReadError("not a gzip file") from e - raise - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except OSError as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a gzip file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open bzip2 compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from bz2 import BZ2File - except ImportError: - raise CompressionError("bz2 module is not available") from None - - fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (OSError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not a bzip2 file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - @classmethod - def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): - """Open lzma compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if mode not in ("r", "w", "x"): - raise ValueError("mode must be 'r', 'w' or 'x'") - - try: - from lzma import LZMAFile, LZMAError - except ImportError: - raise CompressionError("lzma module is not available") from None - - fileobj = LZMAFile(fileobj or name, mode, preset=preset) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (LZMAError, EOFError) as e: - fileobj.close() - if mode == 'r': - raise ReadError("not an lzma file") from e - raise - except: - fileobj.close() - raise - t._extfileobj = False - return t - - # All *open() methods are registered here. - OPEN_METH = { - "tar": "taropen", # uncompressed tar - "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar - } - - #-------------------------------------------------------------------------- - # The public methods which TarFile provides: - - def close(self): - """Close the TarFile. In write-mode, two finishing zero blocks are - appended to the archive. - """ - if self.closed: - return - - self.closed = True - try: - if self.mode in ("a", "w", "x"): - self.fileobj.write(NUL * (BLOCKSIZE * 2)) - self.offset += (BLOCKSIZE * 2) - # fill up the end with zero-blocks - # (like option -b20 for tar does) - blocks, remainder = divmod(self.offset, RECORDSIZE) - if remainder > 0: - self.fileobj.write(NUL * (RECORDSIZE - remainder)) - finally: - if not self._extfileobj: - self.fileobj.close() - - def getmember(self, name): - """Return a TarInfo object for member ``name``. If ``name`` can not be - found in the archive, KeyError is raised. If a member occurs more - than once in the archive, its last occurrence is assumed to be the - most up-to-date version. - """ - tarinfo = self._getmember(name.rstrip('/')) - if tarinfo is None: - raise KeyError("filename %r not found" % name) - return tarinfo - - def getmembers(self): - """Return the members of the archive as a list of TarInfo objects. The - list has the same order as the members in the archive. - """ - self._check() - if not self._loaded: # if we want to obtain a list of - self._load() # all members, we first have to - # scan the whole archive. - return self.members - - def getnames(self): - """Return the members of the archive as a list of their names. It has - the same order as the list returned by getmembers(). - """ - return [tarinfo.name for tarinfo in self.getmembers()] - - def gettarinfo(self, name=None, arcname=None, fileobj=None): - """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by ``name``, or - specified as a file object ``fileobj`` with a file descriptor. If - given, ``arcname`` specifies an alternative name for the file in the - archive, otherwise, the name is taken from the 'name' attribute of - 'fileobj', or the 'name' argument. The name should be a text - string. - """ - self._check("awx") - - # When fileobj is given, replace name by - # fileobj's real name. - if fileobj is not None: - name = fileobj.name - - # Building the name of the member in the archive. - # Backward slashes are converted to forward slashes, - # Absolute paths are turned to relative paths. - if arcname is None: - arcname = name - drv, arcname = os.path.splitdrive(arcname) - arcname = arcname.replace(os.sep, "/") - arcname = arcname.lstrip("/") - - # Now, fill the TarInfo object with - # information specific for the file. - tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed - - # Use os.stat or os.lstat, depending on if symlinks shall be resolved. - if fileobj is None: - if not self.dereference: - statres = os.lstat(name) - else: - statres = os.stat(name) - else: - statres = os.fstat(fileobj.fileno()) - linkname = "" - - stmd = statres.st_mode - if stat.S_ISREG(stmd): - inode = (statres.st_ino, statres.st_dev) - if not self.dereference and statres.st_nlink > 1 and \ - inode in self.inodes and arcname != self.inodes[inode]: - # Is it a hardlink to an already - # archived file? - type = LNKTYPE - linkname = self.inodes[inode] - else: - # The inode is added only if its valid. - # For win32 it is always 0. - type = REGTYPE - if inode[0]: - self.inodes[inode] = arcname - elif stat.S_ISDIR(stmd): - type = DIRTYPE - elif stat.S_ISFIFO(stmd): - type = FIFOTYPE - elif stat.S_ISLNK(stmd): - type = SYMTYPE - linkname = os.readlink(name) - elif stat.S_ISCHR(stmd): - type = CHRTYPE - elif stat.S_ISBLK(stmd): - type = BLKTYPE - else: - return None - - # Fill the TarInfo object with all - # information we can get. - tarinfo.name = arcname - tarinfo.mode = stmd - tarinfo.uid = statres.st_uid - tarinfo.gid = statres.st_gid - if type == REGTYPE: - tarinfo.size = statres.st_size - else: - tarinfo.size = 0 - tarinfo.mtime = statres.st_mtime - tarinfo.type = type - tarinfo.linkname = linkname - if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if type in (CHRTYPE, BLKTYPE): - if hasattr(os, "major") and hasattr(os, "minor"): - tarinfo.devmajor = os.major(statres.st_rdev) - tarinfo.devminor = os.minor(statres.st_rdev) - return tarinfo - - def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If ``verbose`` is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. ``members`` is optional and must be a subset of the - list returned by getmembers(). - """ - self._check() - - if members is None: - members = self - for tarinfo in members: - if verbose: - if tarinfo.mode is None: - _safe_print("??????????") - else: - _safe_print(stat.filemode(tarinfo.mode)) - _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid)) - if tarinfo.ischr() or tarinfo.isblk(): - _safe_print("%10s" % - ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) - else: - _safe_print("%10d" % tarinfo.size) - if tarinfo.mtime is None: - _safe_print("????-??-?? ??:??:??") - else: - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) - - _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) - - if verbose: - if tarinfo.issym(): - _safe_print("-> " + tarinfo.linkname) - if tarinfo.islnk(): - _safe_print("link to " + tarinfo.linkname) - print() - - def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file ``name`` to the archive. ``name`` may be any type of file - (directory, fifo, symbolic link, etc.). If given, ``arcname`` - specifies an alternative name for the file in the archive. - Directories are added recursively by default. This can be avoided by - setting ``recursive`` to False. ``filter`` is a function - that expects a TarInfo object argument and returns the changed - TarInfo object, if it returns None the TarInfo object will be - excluded from the archive. - """ - self._check("awx") - - if arcname is None: - arcname = name - - # Skip if somebody tries to archive the archive... - if self.name is not None and os.path.abspath(name) == self.name: - self._dbg(2, "tarfile: Skipped %r" % name) - return - - self._dbg(1, name) - - # Create a TarInfo object from the file. - tarinfo = self.gettarinfo(name, arcname) - - if tarinfo is None: - self._dbg(1, "tarfile: Unsupported type %r" % name) - return - - # Change or exclude the TarInfo object. - if filter is not None: - tarinfo = filter(tarinfo) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Append the tar header and data to the archive. - if tarinfo.isreg(): - with bltn_open(name, "rb") as f: - self.addfile(tarinfo, f) - - elif tarinfo.isdir(): - self.addfile(tarinfo) - if recursive: - for f in sorted(os.listdir(name)): - self.add(os.path.join(name, f), os.path.join(arcname, f), - recursive, filter=filter) - - else: - self.addfile(tarinfo) - - def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object ``tarinfo`` to the archive. If ``fileobj`` is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). - """ - self._check("awx") - - tarinfo = copy.copy(tarinfo) - - buf = tarinfo.tobuf(self.format, self.encoding, self.errors) - self.fileobj.write(buf) - self.offset += len(buf) - bufsize=self.copybufsize - # If there's data to follow, append it. - if fileobj is not None: - copyfileobj(fileobj, self.fileobj, tarinfo.size, bufsize=bufsize) - blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) - if remainder > 0: - self.fileobj.write(NUL * (BLOCKSIZE - remainder)) - blocks += 1 - self.offset += blocks * BLOCKSIZE - - self.members.append(tarinfo) - - def _get_filter_function(self, filter): - if filter is None: - filter = self.extraction_filter - if filter is None: - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter - if isinstance(filter, str): - raise TypeError( - 'String names are not supported for ' - + 'TarFile.extraction_filter. Use a function such as ' - + 'tarfile.data_filter directly.') - return filter - if callable(filter): - return filter - try: - return _NAMED_FILTERS[filter] - except KeyError: - raise ValueError(f"filter {filter!r} not found") from None - - def extractall(self, path=".", members=None, *, numeric_owner=False, - filter=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only - the numbers for user/group names are used and not the names. - - The `filter` function will be called on each member just - before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - directories = [] - - filter_function = self._get_filter_function(filter) - if members is None: - members = self - - for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is None: - continue - if tarinfo.isdir(): - # For directories, delay setting attributes until later, - # since permissions can interfere with extraction and - # extracting contents can reset mtime. - directories.append(tarinfo) - self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) - - # Reverse sort directories. - directories.sort(key=lambda a: a.name, reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, - filter=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` - is True, only the numbers for user/group names are used and not - the names. - - The `filter` function will be called before extraction. - It can return a changed TarInfo or None to skip the member. - String names of common filters are accepted. - """ - filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) - if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) - - def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - unfiltered = tarinfo - try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo - - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" - self._check("r") - - try: - self._extract_member(tarinfo, os.path.join(path, tarinfo.name), - set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: - self._handle_fatal_error(e) - except ExtractError as e: - self._handle_nonfatal_error(e) - - def _handle_nonfatal_error(self, e): - """Handle non-fatal error (ExtractError) according to errorlevel""" - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def _handle_fatal_error(self, e): - """Handle "fatal" error according to self.errorlevel""" - if self.errorlevel > 0: - raise - elif isinstance(e, OSError): - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) - else: - self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) - - def extractfile(self, member): - """Extract a member from the archive as a file object. ``member`` may be - a filename or a TarInfo object. If ``member`` is a regular file or - a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If ``member`` does not appear - in the archive, KeyError is raised. - """ - self._check("r") - - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member - - if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES: - # Members with unknown types are treated as regular files. - return self.fileobject(self, tarinfo) - - elif tarinfo.islnk() or tarinfo.issym(): - if isinstance(self.fileobj, _Stream): - # A small but ugly workaround for the case that someone tries - # to extract a (sym)link as a file-object from a non-seekable - # stream of tar blocks. - raise StreamError("cannot extract (sym)link as file object") - else: - # A (sym)link's file object is its target's file object. - return self.extractfile(self._find_link_target(tarinfo)) - else: - # If there's no data associated with the member (directory, chrdev, - # blkdev, etc.), return None instead of a file object. - return None - - def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical - file called targetpath. - """ - # Fetch the TarInfo object for the given name - # and build the destination pathname, replacing - # forward slashes to platform specific separators. - targetpath = targetpath.rstrip("/") - targetpath = targetpath.replace("/", os.sep) - - # Create all upper directories. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - # Create directories that are not part of the archive with - # default permissions. - os.makedirs(upperdirs) - - if tarinfo.islnk() or tarinfo.issym(): - self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) - else: - self._dbg(1, tarinfo.name) - - if tarinfo.isreg(): - self.makefile(tarinfo, targetpath) - elif tarinfo.isdir(): - self.makedir(tarinfo, targetpath) - elif tarinfo.isfifo(): - self.makefifo(tarinfo, targetpath) - elif tarinfo.ischr() or tarinfo.isblk(): - self.makedev(tarinfo, targetpath) - elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) - elif tarinfo.type not in SUPPORTED_TYPES: - self.makeunknown(tarinfo, targetpath) - else: - self.makefile(tarinfo, targetpath) - - if set_attrs: - self.chown(tarinfo, targetpath, numeric_owner) - if not tarinfo.issym(): - self.chmod(tarinfo, targetpath) - self.utime(tarinfo, targetpath) - - #-------------------------------------------------------------------------- - # Below are the different file methods. They are called via - # _extract_member() when extract() is called. They can be replaced in a - # subclass to implement other functionality. - - def makedir(self, tarinfo, targetpath): - """Make a directory called targetpath. - """ - try: - if tarinfo.mode is None: - # Use the system's default mode - os.mkdir(targetpath) - else: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) - except FileExistsError: - if not os.path.isdir(targetpath): - raise - - def makefile(self, tarinfo, targetpath): - """Make a file called targetpath. - """ - source = self.fileobj - source.seek(tarinfo.offset_data) - bufsize = self.copybufsize - with bltn_open(targetpath, "wb") as target: - if tarinfo.sparse is not None: - for offset, size in tarinfo.sparse: - target.seek(offset) - copyfileobj(source, target, size, ReadError, bufsize) - target.seek(tarinfo.size) - target.truncate() - else: - copyfileobj(source, target, tarinfo.size, ReadError, bufsize) - - def makeunknown(self, tarinfo, targetpath): - """Make a file from a TarInfo object with an unknown type - at targetpath. - """ - self.makefile(tarinfo, targetpath) - self._dbg(1, "tarfile: Unknown file type %r, " \ - "extracted as regular file." % tarinfo.type) - - def makefifo(self, tarinfo, targetpath): - """Make a fifo called targetpath. - """ - if hasattr(os, "mkfifo"): - os.mkfifo(targetpath) - else: - raise ExtractError("fifo not supported by system") - - def makedev(self, tarinfo, targetpath): - """Make a character or block device called targetpath. - """ - if not hasattr(os, "mknod") or not hasattr(os, "makedev"): - raise ExtractError("special devices not supported by system") - - mode = tarinfo.mode - if mode is None: - # Use mknod's default - mode = 0o600 - if tarinfo.isblk(): - mode |= stat.S_IFBLK - else: - mode |= stat.S_IFCHR - - os.mknod(targetpath, mode, - os.makedev(tarinfo.devmajor, tarinfo.devminor)) - - def makelink(self, tarinfo, targetpath): - """Make a (symbolic) link called targetpath. If it cannot be created - (platform limitation), we try to make a copy of the referenced file - instead of a link. - """ - try: - # For systems that support symbolic and hard links. - if tarinfo.issym(): - if os.path.lexists(targetpath): - # Avoid FileExistsError on following os.symlink. - os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) - else: - if os.path.exists(tarinfo._link_target): - os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except symlink_exception: - try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None - - def chown(self, tarinfo, targetpath, numeric_owner): - """Set owner of targetpath according to tarinfo. If numeric_owner - is True, use .gid/.uid instead of .gname/.uname. If numeric_owner - is False, fall back to .gid/.uid when the search based on name - fails. - """ - if hasattr(os, "geteuid") and os.geteuid() == 0: - # We have to be root to do so. - g = tarinfo.gid - u = tarinfo.uid - if not numeric_owner: - try: - if grp and tarinfo.gname: - g = grp.getgrnam(tarinfo.gname)[2] - except KeyError: - pass - try: - if pwd and tarinfo.uname: - u = pwd.getpwnam(tarinfo.uname)[2] - except KeyError: - pass - if g is None: - g = -1 - if u is None: - u = -1 - try: - if tarinfo.issym() and hasattr(os, "lchown"): - os.lchown(targetpath, u, g) - else: - os.chown(targetpath, u, g) - except OSError as e: - raise ExtractError("could not change owner") from e - - def chmod(self, tarinfo, targetpath): - """Set file permissions of targetpath according to tarinfo. - """ - if tarinfo.mode is None: - return - try: - os.chmod(targetpath, tarinfo.mode) - except OSError as e: - raise ExtractError("could not change mode") from e - - def utime(self, tarinfo, targetpath): - """Set modification time of targetpath according to tarinfo. - """ - mtime = tarinfo.mtime - if mtime is None: - return - if not hasattr(os, 'utime'): - return - try: - os.utime(targetpath, (mtime, mtime)) - except OSError as e: - raise ExtractError("could not change modification time") from e - - #-------------------------------------------------------------------------- - def next(self): - """Return the next member of the archive as a TarInfo object, when - TarFile is opened for reading. Return None if there is no more - available. - """ - self._check("ra") - if self.firstmember is not None: - m = self.firstmember - self.firstmember = None - return m - - # Advance the file pointer. - if self.offset != self.fileobj.tell(): - if self.offset == 0: - return None - self.fileobj.seek(self.offset - 1) - if not self.fileobj.read(1): - raise ReadError("unexpected end of data") - - # Read the next block. - tarinfo = None - while True: - try: - tarinfo = self.tarinfo.fromtarfile(self) - except EOFHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - except InvalidHeaderError as e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - elif self.offset == 0: - raise ReadError(str(e)) from None - except EmptyHeaderError: - if self.offset == 0: - raise ReadError("empty file") from None - except TruncatedHeaderError as e: - if self.offset == 0: - raise ReadError(str(e)) from None - except SubsequentHeaderError as e: - raise ReadError(str(e)) from None - except Exception as e: - try: - import zlib - if isinstance(e, zlib.error): - raise ReadError(f'zlib error: {e}') from None - else: - raise e - except ImportError: - raise e - break - - if tarinfo is not None: - self.members.append(tarinfo) - else: - self._loaded = True - - return tarinfo - - #-------------------------------------------------------------------------- - # Little helper methods: - - def _getmember(self, name, tarinfo=None, normalize=False): - """Find an archive member by name from bottom to top. - If tarinfo is given, it is used as the starting point. - """ - # Ensure that all members have been loaded. - members = self.getmembers() - - # Limit the member search list up to tarinfo. - skipping = False - if tarinfo is not None: - try: - index = members.index(tarinfo) - except ValueError: - # The given starting point might be a (modified) copy. - # We'll later skip members until we find an equivalent. - skipping = True - else: - # Happy fast path - members = members[:index] - - if normalize: - name = os.path.normpath(name) - - for member in reversed(members): - if skipping: - if tarinfo.offset == member.offset: - skipping = False - continue - if normalize: - member_name = os.path.normpath(member.name) - else: - member_name = member.name - - if name == member_name: - return member - - if skipping: - # Starting point was not found - raise ValueError(tarinfo) - - def _load(self): - """Read through the entire archive file and look for readable - members. - """ - while self.next() is not None: - pass - self._loaded = True - - def _check(self, mode=None): - """Check if TarFile is still open, and if the operation's mode - corresponds to TarFile's mode. - """ - if self.closed: - raise OSError("%s is closed" % self.__class__.__name__) - if mode is not None and self.mode not in mode: - raise OSError("bad operation for mode %r" % self.mode) - - def _find_link_target(self, tarinfo): - """Find the target member of a symlink or hardlink member in the - archive. - """ - if tarinfo.issym(): - # Always search the entire archive. - linkname = "/".join(filter(None, (os.path.dirname(tarinfo.name), tarinfo.linkname))) - limit = None - else: - # Search the archive before the link, because a hard link is - # just a reference to an already archived file. - linkname = tarinfo.linkname - limit = tarinfo - - member = self._getmember(linkname, tarinfo=limit, normalize=True) - if member is None: - raise KeyError("linkname %r not found" % linkname) - return member - - def __iter__(self): - """Provide an iterator object. - """ - if self._loaded: - yield from self.members - return - - # Yield items using TarFile's next() method. - # When all members have been read, set TarFile as _loaded. - index = 0 - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will have already exhausted the next() method. - if self.firstmember is not None: - tarinfo = self.next() - index += 1 - yield tarinfo - - while True: - if index < len(self.members): - tarinfo = self.members[index] - elif not self._loaded: - tarinfo = self.next() - if not tarinfo: - self._loaded = True - return - else: - return - index += 1 - yield tarinfo - - def _dbg(self, level, msg): - """Write debugging output to sys.stderr. - """ - if level <= self.debug: - print(msg, file=sys.stderr) - - def __enter__(self): - self._check() - return self - - def __exit__(self, type, value, traceback): - if type is None: - self.close() - else: - # An exception occurred. We must not call close() because - # it would try to write end-of-archive blocks and padding. - if not self._extfileobj: - self.fileobj.close() - self.closed = True - -#-------------------- -# exported functions -#-------------------- - -def is_tarfile(name): - """Return True if name points to a tar archive that we - are able to handle, else return False. - - 'name' should be a string, file, or file-like object. - """ - try: - if hasattr(name, "read"): - pos = name.tell() - t = open(fileobj=name) - name.seek(pos) - else: - t = open(name) - t.close() - return True - except TarError: - return False - -open = TarFile.open - - -def main(): - import argparse - - description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='Verbose output') - parser.add_argument('--filter', metavar='<filtername>', - choices=_NAMED_FILTERS, - help='Filter for extraction') - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='<tarfile>', - help='Show listing of a tarfile') - group.add_argument('-e', '--extract', nargs='+', - metavar=('<tarfile>', '<output_dir>'), - help='Extract tarfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('<name>', '<file>'), - help='Create tarfile from sources') - group.add_argument('-t', '--test', metavar='<tarfile>', - help='Test if a tarfile is valid') - - args = parser.parse_args() - - if args.filter and args.extract is None: - parser.exit(1, '--filter is only valid for extraction\n') - - if args.test is not None: - src = args.test - if is_tarfile(src): - with open(src, 'r') as tar: - tar.getmembers() - print(tar.getmembers(), file=sys.stderr) - if args.verbose: - print('{!r} is a tar archive.'.format(src)) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.list is not None: - src = args.list - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.list(verbose=args.verbose) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.extract is not None: - if len(args.extract) == 1: - src = args.extract[0] - curdir = os.curdir - elif len(args.extract) == 2: - src, curdir = args.extract - else: - parser.exit(1, parser.format_help()) - - if is_tarfile(src): - with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir, filter=args.filter) - if args.verbose: - if curdir == '.': - msg = '{!r} file is extracted.'.format(src) - else: - msg = ('{!r} file is extracted ' - 'into {!r} directory.').format(src, curdir) - print(msg) - else: - parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) - - elif args.create is not None: - tar_name = args.create.pop(0) - _, ext = os.path.splitext(tar_name) - compressions = { - # gz - '.gz': 'gz', - '.tgz': 'gz', - # xz - '.xz': 'xz', - '.txz': 'xz', - # bz2 - '.bz2': 'bz2', - '.tbz': 'bz2', - '.tbz2': 'bz2', - '.tb2': 'bz2', - } - tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' - tar_files = args.create - - with TarFile.open(tar_name, tar_mode) as tf: - for file_name in tar_files: - tf.add(file_name) - - if args.verbose: - print('{!r} file created.'.format(tar_name)) - -if __name__ == '__main__': - main() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py deleted file mode 100644 index 88642143755..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/__init__.py +++ /dev/null @@ -1,904 +0,0 @@ -import os -import re -import abc -import csv -import sys -from .. import zipp -import email -import pathlib -import operator -import textwrap -import warnings -import functools -import itertools -import posixpath -import collections - -from . import _adapters, _meta, _py39compat -from ._collections import FreezableDefaultDict, Pair -from ._compat import ( - NullFinder, - install, - pypy_partial, -) -from ._functools import method_cache, pass_none -from ._itertools import always_iterable, unique_everseen -from ._meta import PackageMetadata, SimplePath - -from contextlib import suppress -from importlib import import_module -from importlib.abc import MetaPathFinder -from itertools import starmap -from typing import List, Mapping, Optional - - -__all__ = [ - 'Distribution', - 'DistributionFinder', - 'PackageMetadata', - 'PackageNotFoundError', - 'distribution', - 'distributions', - 'entry_points', - 'files', - 'metadata', - 'packages_distributions', - 'requires', - 'version', -] - - -class PackageNotFoundError(ModuleNotFoundError): - """The package was not found.""" - - def __str__(self): - return f"No package metadata was found for {self.name}" - - @property - def name(self): - (name,) = self.args - return name - - -class Sectioned: - """ - A simple entry point config parser for performance - - >>> for item in Sectioned.read(Sectioned._sample): - ... print(item) - Pair(name='sec1', value='# comments ignored') - Pair(name='sec1', value='a = 1') - Pair(name='sec1', value='b = 2') - Pair(name='sec2', value='a = 2') - - >>> res = Sectioned.section_pairs(Sectioned._sample) - >>> item = next(res) - >>> item.name - 'sec1' - >>> item.value - Pair(name='a', value='1') - >>> item = next(res) - >>> item.value - Pair(name='b', value='2') - >>> item = next(res) - >>> item.name - 'sec2' - >>> item.value - Pair(name='a', value='2') - >>> list(res) - [] - """ - - _sample = textwrap.dedent( - """ - [sec1] - # comments ignored - a = 1 - b = 2 - - [sec2] - a = 2 - """ - ).lstrip() - - @classmethod - def section_pairs(cls, text): - return ( - section._replace(value=Pair.parse(section.value)) - for section in cls.read(text, filter_=cls.valid) - if section.name is not None - ) - - @staticmethod - def read(text, filter_=None): - lines = filter(filter_, map(str.strip, text.splitlines())) - name = None - for value in lines: - section_match = value.startswith('[') and value.endswith(']') - if section_match: - name = value.strip('[]') - continue - yield Pair(name, value) - - @staticmethod - def valid(line): - return line and not line.startswith('#') - - -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=pypy_partial(2), - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): - """An entry point as defined by Python packaging conventions. - - See `the packaging docs on entry points - <https://packaging.python.org/specifications/entry-points/>`_ - for more information. - - >>> ep = EntryPoint( - ... name=None, group=None, value='package.module:attr [extra1, extra2]') - >>> ep.module - 'package.module' - >>> ep.attr - 'attr' - >>> ep.extras - ['extra1', 'extra2'] - """ - - pattern = re.compile( - r'(?P<module>[\w.]+)\s*' - r'(:\s*(?P<attr>[\w.]+)\s*)?' - r'((?P<extras>\[.*\])\s*)?$' - ) - """ - A regular expression describing the syntax for an entry point, - which might look like: - - - module - - package.module - - package.module:attribute - - package.module:object.attribute - - package.module:attr [extra1, extra2] - - Other combinations are possible as well. - - The expression is lenient about whitespace around the ':', - following the attr, and following any extras. - """ - - name: str - value: str - group: str - - dist: Optional['Distribution'] = None - - def __init__(self, name, value, group): - vars(self).update(name=name, value=value, group=group) - - def load(self): - """Load the entry point from its definition. If only a module - is indicated by the value, return that module. Otherwise, - return the named object. - """ - match = self.pattern.match(self.value) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) - return functools.reduce(getattr, attrs, module) - - @property - def module(self): - match = self.pattern.match(self.value) - return match.group('module') - - @property - def attr(self): - match = self.pattern.match(self.value) - return match.group('attr') - - @property - def extras(self): - match = self.pattern.match(self.value) - return re.findall(r'\w+', match.group('extras') or '') - - def _for(self, dist): - vars(self).update(dist=dist) - return self - - def matches(self, **params): - """ - EntryPoint matches the given parameters. - - >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]') - >>> ep.matches(group='foo') - True - >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]') - True - >>> ep.matches(group='foo', name='other') - False - >>> ep.matches() - True - >>> ep.matches(extras=['extra1', 'extra2']) - True - >>> ep.matches(module='bing') - True - >>> ep.matches(attr='bong') - True - """ - attrs = (getattr(self, param) for param in params) - return all(map(operator.eq, params.values(), attrs)) - - def _key(self): - return self.name, self.value, self.group - - def __lt__(self, other): - return self._key() < other._key() - - def __eq__(self, other): - return self._key() == other._key() - - def __setattr__(self, name, value): - raise AttributeError("EntryPoint objects are immutable.") - - def __repr__(self): - return ( - f'EntryPoint(name={self.name!r}, value={self.value!r}, ' - f'group={self.group!r})' - ) - - def __hash__(self): - return hash(self._key()) - - -class EntryPoints(tuple): - """ - An immutable collection of selectable EntryPoint objects. - """ - - __slots__ = () - - def __getitem__(self, name): # -> EntryPoint: - """ - Get the EntryPoint in self matching name. - """ - try: - return next(iter(self.select(name=name))) - except StopIteration: - raise KeyError(name) - - def select(self, **params): - """ - Select entry points from self that match the - given parameters (typically group and/or name). - """ - return EntryPoints(ep for ep in self if _py39compat.ep_matches(ep, **params)) - - @property - def names(self): - """ - Return the set of all names of all entry points. - """ - return {ep.name for ep in self} - - @property - def groups(self): - """ - Return the set of all groups of all entry points. - """ - return {ep.group for ep in self} - - @classmethod - def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in cls._from_text(text)) - - @staticmethod - def _from_text(text): - return ( - EntryPoint(name=item.value.name, value=item.value.value, group=item.name) - for item in Sectioned.section_pairs(text or '') - ) - - -class PackagePath(pathlib.PurePosixPath): - """A reference to a path in a package""" - - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() - - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() - - def locate(self): - """Return a path-like object for this path""" - return self.dist.locate_file(self) - - -class FileHash: - def __init__(self, spec): - self.mode, _, self.value = spec.partition('=') - - def __repr__(self): - return f'<FileHash mode: {self.mode} value: {self.value}>' - - -class Distribution(metaclass=abc.ABCMeta): - """A Python distribution package.""" - - @abc.abstractmethod - def read_text(self, filename): - """Attempt to load metadata file given by the name. - - :param filename: The name of the file in the distribution info. - :return: The text if found, otherwise None. - """ - - @abc.abstractmethod - def locate_file(self, path): - """ - Given a path to a file in this distribution, return a path - to it. - """ - - @classmethod - def from_name(cls, name: str): - """Return the Distribution for the given package name. - - :param name: The name of the distribution package to search for. - :return: The Distribution instance (or subclass thereof) for the named - package, if found. - :raises PackageNotFoundError: When the named package's distribution - metadata cannot be found. - :raises ValueError: When an invalid value is supplied for name. - """ - if not name: - raise ValueError("A distribution name is required.") - try: - return next(cls.discover(name=name)) - except StopIteration: - raise PackageNotFoundError(name) - - @classmethod - def discover(cls, **kwargs): - """Return an iterable of Distribution objects for all packages. - - Pass a ``context`` or pass keyword arguments for constructing - a context. - - :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. - """ - context = kwargs.pop('context', None) - if context and kwargs: - raise ValueError("cannot accept context and kwargs") - context = context or DistributionFinder.Context(**kwargs) - return itertools.chain.from_iterable( - resolver(context) for resolver in cls._discover_resolvers() - ) - - @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path - - :param path: a string or path-like object - :return: a concrete Distribution instance for the path - """ - return PathDistribution(pathlib.Path(path)) - - @staticmethod - def _discover_resolvers(): - """Search the meta_path for resolvers.""" - declared = ( - getattr(finder, 'find_distributions', None) for finder in sys.meta_path - ) - return filter(None, declared) - - @property - def metadata(self) -> _meta.PackageMetadata: - """Return the parsed metadata for this Distribution. - - The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. - """ - text = ( - self.read_text('METADATA') - or self.read_text('PKG-INFO') - # This last clause is here to support old egg-info files. Its - # effect is to just end up using the PathDistribution's self._path - # (which points to the egg-info file) attribute unchanged. - or self.read_text('') - ) - return _adapters.Message(email.message_from_string(text)) - - @property - def name(self): - """Return the 'Name' metadata for the distribution package.""" - return self.metadata['Name'] - - @property - def _normalized_name(self): - """Return a normalized version of the name.""" - return Prepared.normalize(self.name) - - @property - def version(self): - """Return the 'Version' metadata for the distribution package.""" - return self.metadata['Version'] - - @property - def entry_points(self): - return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) - - @property - def files(self): - """Files in this distribution. - - :return: List of PackagePath for this distribution or None - - Result is `None` if the metadata file that enumerates files - (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is - missing. - Result may be empty if the metadata exists but is empty. - """ - - def make_file(name, hash=None, size_str=None): - result = PackagePath(name) - result.hash = FileHash(hash) if hash else None - result.size = int(size_str) if size_str else None - result.dist = self - return result - - @pass_none - def make_files(lines): - return list(starmap(make_file, csv.reader(lines))) - - return make_files(self._read_files_distinfo() or self._read_files_egginfo()) - - def _read_files_distinfo(self): - """ - Read the lines of RECORD - """ - text = self.read_text('RECORD') - return text and text.splitlines() - - def _read_files_egginfo(self): - """ - SOURCES.txt might contain literal commas, so wrap each line - in quotes. - """ - text = self.read_text('SOURCES.txt') - return text and map('"{}"'.format, text.splitlines()) - - @property - def requires(self): - """Generated requirements specified for this Distribution""" - reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() - return reqs and list(reqs) - - def _read_dist_info_reqs(self): - return self.metadata.get_all('Requires-Dist') - - def _read_egg_info_reqs(self): - source = self.read_text('requires.txt') - return pass_none(self._deps_from_requires_text)(source) - - @classmethod - def _deps_from_requires_text(cls, source): - return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) - - @staticmethod - def _convert_egg_info_reqs_to_simple_reqs(sections): - """ - Historically, setuptools would solicit and store 'extra' - requirements, including those with environment markers, - in separate sections. More modern tools expect each - dependency to be defined separately, with any relevant - extras and environment markers attached directly to that - requirement. This method converts the former to the - latter. See _test_deps_from_requires_text for an example. - """ - - def make_condition(name): - return name and f'extra == "{name}"' - - def quoted_marker(section): - section = section or '' - extra, sep, markers = section.partition(':') - if extra and markers: - markers = f'({markers})' - conditions = list(filter(None, [markers, make_condition(extra)])) - return '; ' + ' and '.join(conditions) if conditions else '' - - def url_req_space(req): - """ - PEP 508 requires a space between the url_spec and the quoted_marker. - Ref python/importlib_metadata#357. - """ - # '@' is uniquely indicative of a url_req. - return ' ' * ('@' in req) - - for section in sections: - space = url_req_space(section.value) - yield section.value + space + quoted_marker(section.name) - - -class DistributionFinder(MetaPathFinder): - """ - A MetaPathFinder capable of discovering installed distributions. - """ - - class Context: - """ - Keyword arguments presented by the caller to - ``distributions()`` or ``Distribution.discover()`` - to narrow the scope of a search for distributions - in all DistributionFinders. - - Each DistributionFinder may expect any parameters - and should attempt to honor the canonical - parameters defined below when appropriate. - """ - - name = None - """ - Specific name for which a distribution finder should match. - A name of ``None`` matches all distributions. - """ - - def __init__(self, **kwargs): - vars(self).update(kwargs) - - @property - def path(self): - """ - The sequence of directory path that a distribution finder - should search. - - Typically refers to Python installed package paths such as - "site-packages" directories and defaults to ``sys.path``. - """ - return vars(self).get('path', sys.path) - - @abc.abstractmethod - def find_distributions(self, context=Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching the ``context``, - a DistributionFinder.Context instance. - """ - - -class FastPath: - """ - Micro-optimized class for searching a path for - children. - - >>> FastPath('').children() - ['...'] - """ - - @functools.lru_cache() # type: ignore - def __new__(cls, root): - return super().__new__(cls) - - def __init__(self, root): - self.root = root - - def joinpath(self, child): - return pathlib.Path(self.root, child) - - def children(self): - with suppress(Exception): - return os.listdir(self.root or '.') - with suppress(Exception): - return self.zip_children() - return [] - - def zip_children(self): - zip_path = zipp.Path(self.root) - names = zip_path.root.namelist() - self.joinpath = zip_path.joinpath - - return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - - def search(self, name): - return self.lookup(self.mtime).search(name) - - @property - def mtime(self): - with suppress(OSError): - return os.stat(self.root).st_mtime - self.lookup.cache_clear() - - @method_cache - def lookup(self, mtime): - return Lookup(self) - - -class Lookup: - def __init__(self, path: FastPath): - base = os.path.basename(path.root).lower() - base_is_egg = base.endswith(".egg") - self.infos = FreezableDefaultDict(list) - self.eggs = FreezableDefaultDict(list) - - for child in path.children(): - low = child.lower() - if low.endswith((".dist-info", ".egg-info")): - # rpartition is faster than splitext and suitable for this purpose. - name = low.rpartition(".")[0].partition("-")[0] - normalized = Prepared.normalize(name) - self.infos[normalized].append(path.joinpath(child)) - elif base_is_egg and low == "egg-info": - name = base.rpartition(".")[0].partition("-")[0] - legacy_normalized = Prepared.legacy_normalize(name) - self.eggs[legacy_normalized].append(path.joinpath(child)) - - self.infos.freeze() - self.eggs.freeze() - - def search(self, prepared): - infos = ( - self.infos[prepared.normalized] - if prepared - else itertools.chain.from_iterable(self.infos.values()) - ) - eggs = ( - self.eggs[prepared.legacy_normalized] - if prepared - else itertools.chain.from_iterable(self.eggs.values()) - ) - return itertools.chain(infos, eggs) - - -class Prepared: - """ - A prepared search for metadata on a possibly-named package. - """ - - normalized = None - legacy_normalized = None - - def __init__(self, name): - self.name = name - if name is None: - return - self.normalized = self.normalize(name) - self.legacy_normalized = self.legacy_normalize(name) - - @staticmethod - def normalize(name): - """ - PEP 503 normalization plus dashes as underscores. - """ - return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') - - @staticmethod - def legacy_normalize(name): - """ - Normalize the package name as found in the convention in - older packaging tools versions and specs. - """ - return name.lower().replace('-', '_') - - def __bool__(self): - return bool(self.name) - - -@install -class MetadataPathFinder(NullFinder, DistributionFinder): - """A degenerate finder for distribution packages on the file system. - - This finder supplies only a find_distributions() method for versions - of Python that do not have a PathFinder find_distributions(). - """ - - def find_distributions(self, context=DistributionFinder.Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching ``context.name`` - (or all names if ``None`` indicated) along the paths in the list - of directories ``context.path``. - """ - found = self._search_paths(context.name, context.path) - return map(PathDistribution, found) - - @classmethod - def _search_paths(cls, name, paths): - """Find metadata directories in paths heuristically.""" - prepared = Prepared(name) - return itertools.chain.from_iterable( - path.search(prepared) for path in map(FastPath, paths) - ) - - def invalidate_caches(cls): - FastPath.__new__.cache_clear() - - -class PathDistribution(Distribution): - def __init__(self, path: SimplePath): - """Construct a distribution. - - :param path: SimplePath indicating the metadata directory. - """ - self._path = path - - def read_text(self, filename): - with suppress( - FileNotFoundError, - IsADirectoryError, - KeyError, - NotADirectoryError, - PermissionError, - ): - return self._path.joinpath(filename).read_text(encoding='utf-8') - - read_text.__doc__ = Distribution.read_text.__doc__ - - def locate_file(self, path): - return self._path.parent / path - - @property - def _normalized_name(self): - """ - Performance optimization: where possible, resolve the - normalized name from the file system path. - """ - stem = os.path.basename(str(self._path)) - return ( - pass_none(Prepared.normalize)(self._name_from_stem(stem)) - or super()._normalized_name - ) - - @staticmethod - def _name_from_stem(stem): - """ - >>> PathDistribution._name_from_stem('foo-3.0.egg-info') - 'foo' - >>> PathDistribution._name_from_stem('CherryPy-3.0.dist-info') - 'CherryPy' - >>> PathDistribution._name_from_stem('face.egg-info') - 'face' - >>> PathDistribution._name_from_stem('foo.bar') - """ - filename, ext = os.path.splitext(stem) - if ext not in ('.dist-info', '.egg-info'): - return - name, sep, rest = filename.partition('-') - return name - - -def distribution(distribution_name): - """Get the ``Distribution`` instance for the named package. - - :param distribution_name: The name of the distribution package as a string. - :return: A ``Distribution`` instance (or subclass thereof). - """ - return Distribution.from_name(distribution_name) - - -def distributions(**kwargs): - """Get all ``Distribution`` instances in the current environment. - - :return: An iterable of ``Distribution`` instances. - """ - return Distribution.discover(**kwargs) - - -def metadata(distribution_name) -> _meta.PackageMetadata: - """Get the metadata for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: A PackageMetadata containing the parsed metadata. - """ - return Distribution.from_name(distribution_name).metadata - - -def version(distribution_name): - """Get the version string for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: The version string for the package as defined in the package's - "Version" metadata key. - """ - return distribution(distribution_name).version - - -_unique = functools.partial( - unique_everseen, - key=_py39compat.normalized_name, -) -""" -Wrapper for ``distributions`` to return unique distributions by name. -""" - - -def entry_points(**params) -> EntryPoints: - """Return EntryPoint objects for all installed packages. - - Pass selection parameters (group or name) to filter the - result to entry points matching those properties (see - EntryPoints.select()). - - :return: EntryPoints for all installed packages. - """ - eps = itertools.chain.from_iterable( - dist.entry_points for dist in _unique(distributions()) - ) - return EntryPoints(eps).select(**params) - - -def files(distribution_name): - """Return a list of files for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: List of files composing the distribution. - """ - return distribution(distribution_name).files - - -def requires(distribution_name): - """ - Return a list of requirements for the named package. - - :return: An iterator of requirements, suitable for - packaging.requirement.Requirement. - """ - return distribution(distribution_name).requires - - -def packages_distributions() -> Mapping[str, List[str]]: - """ - Return a mapping of top-level packages to their - distributions. - - >>> import collections.abc - >>> pkgs = packages_distributions() - >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) - True - """ - pkg_to_dist = collections.defaultdict(list) - for dist in distributions(): - for pkg in _top_level_declared(dist) or _top_level_inferred(dist): - pkg_to_dist[pkg].append(dist.metadata['Name']) - return dict(pkg_to_dist) - - -def _top_level_declared(dist): - return (dist.read_text('top_level.txt') or '').split() - - -def _top_level_inferred(dist): - return { - f.parts[0] if len(f.parts) > 1 else f.with_suffix('').name - for f in always_iterable(dist.files) - if f.suffix == ".py" - } diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py deleted file mode 100644 index e33cba5e44d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_adapters.py +++ /dev/null @@ -1,90 +0,0 @@ -import functools -import warnings -import re -import textwrap -import email.message - -from ._text import FoldedCase -from ._compat import pypy_partial - - -# Do not remove prior to 2024-01-01 or Python 3.14 -_warn = functools.partial( - warnings.warn, - "Implicit None on return values is deprecated and will raise KeyErrors.", - DeprecationWarning, - stacklevel=pypy_partial(2), -) - - -class Message(email.message.Message): - multiple_use_keys = set( - map( - FoldedCase, - [ - 'Classifier', - 'Obsoletes-Dist', - 'Platform', - 'Project-URL', - 'Provides-Dist', - 'Provides-Extra', - 'Requires-Dist', - 'Requires-External', - 'Supported-Platform', - 'Dynamic', - ], - ) - ) - """ - Keys that may be indicated multiple times per PEP 566. - """ - - def __new__(cls, orig: email.message.Message): - res = super().__new__(cls) - vars(res).update(vars(orig)) - return res - - def __init__(self, *args, **kwargs): - self._headers = self._repair_headers() - - # suppress spurious error from mypy - def __iter__(self): - return super().__iter__() - - def __getitem__(self, item): - """ - Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. - """ - res = super().__getitem__(item) - if res is None: - _warn() - return res - - def _repair_headers(self): - def redent(value): - "Correct for RFC822 indentation" - if not value or '\n' not in value: - return value - return textwrap.dedent(' ' * 8 + value) - - headers = [(key, redent(value)) for key, value in vars(self)['_headers']] - if self._payload: - headers.append(('Description', self.get_payload())) - return headers - - @property - def json(self): - """ - Convert PackageMetadata to a JSON-compatible format - per PEP 0566. - """ - - def transform(key): - value = self.get_all(key) if key in self.multiple_use_keys else self[key] - if key == 'Keywords': - value = re.split(r'\s+', value) - tk = key.lower().replace('-', '_') - return tk, value - - return dict(map(transform, map(FoldedCase, self))) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py deleted file mode 100644 index cf0954e1a30..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_collections.py +++ /dev/null @@ -1,30 +0,0 @@ -import collections - - -# from jaraco.collections 3.3 -class FreezableDefaultDict(collections.defaultdict): - """ - Often it is desirable to prevent the mutation of - a default dict after its initial construction, such - as to prevent mutation during iteration. - - >>> dd = FreezableDefaultDict(list) - >>> dd[0].append('1') - >>> dd.freeze() - >>> dd[1] - [] - >>> len(dd) - 1 - """ - - def __missing__(self, key): - return getattr(self, '_frozen', super().__missing__)(key) - - def freeze(self): - self._frozen = lambda key: self.default_factory() - - -class Pair(collections.namedtuple('Pair', 'name value')): - @classmethod - def parse(cls, text): - return cls(*map(str.strip, text.split("=", 1))) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py deleted file mode 100644 index 84f9eea4f3c..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_compat.py +++ /dev/null @@ -1,72 +0,0 @@ -import sys -import platform - - -__all__ = ['install', 'NullFinder', 'Protocol'] - - -try: - from typing import Protocol -except ImportError: # pragma: no cover - # Python 3.7 compatibility - from ..typing_extensions import Protocol # type: ignore - - -def install(cls): - """ - Class decorator for installation on sys.meta_path. - - Adds the backport DistributionFinder to sys.meta_path and - attempts to disable the finder functionality of the stdlib - DistributionFinder. - """ - sys.meta_path.append(cls()) - disable_stdlib_finder() - return cls - - -def disable_stdlib_finder(): - """ - Give the backport primacy for discovering path-based distributions - by monkey-patching the stdlib O_O. - - See #91 for more background for rationale on this sketchy - behavior. - """ - - def matches(finder): - return getattr( - finder, '__module__', None - ) == '_frozen_importlib_external' and hasattr(finder, 'find_distributions') - - for finder in filter(matches, sys.meta_path): # pragma: nocover - del finder.find_distributions - - -class NullFinder: - """ - A "Finder" (aka "MetaClassFinder") that never finds any modules, - but may find distributions. - """ - - @staticmethod - def find_spec(*args, **kwargs): - return None - - # In Python 2, the import system requires finders - # to have a find_module() method, but this usage - # is deprecated in Python 3 in favor of find_spec(). - # For the purposes of this finder (i.e. being present - # on sys.meta_path but having no other import - # system functionality), the two methods are identical. - find_module = find_spec - - -def pypy_partial(val): - """ - Adjust for variable stacklevel on partial under PyPy. - - Workaround for #327. - """ - is_pypy = platform.python_implementation() == 'PyPy' - return val + is_pypy diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py deleted file mode 100644 index 71f66bd03cb..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_functools.py +++ /dev/null @@ -1,104 +0,0 @@ -import types -import functools - - -# from jaraco.functools 3.3 -def method_cache(method, cache_wrapper=None): - """ - Wrap lru_cache to support storing the cache data in the object instances. - - Abstracts the common paradigm where the method explicitly saves an - underscore-prefixed protected property on first call and returns that - subsequently. - - >>> class MyClass: - ... calls = 0 - ... - ... @method_cache - ... def method(self, value): - ... self.calls += 1 - ... return value - - >>> a = MyClass() - >>> a.method(3) - 3 - >>> for x in range(75): - ... res = a.method(x) - >>> a.calls - 75 - - Note that the apparent behavior will be exactly like that of lru_cache - except that the cache is stored on each instance, so values in one - instance will not flush values from another, and when an instance is - deleted, so are the cached values for that instance. - - >>> b = MyClass() - >>> for x in range(35): - ... res = b.method(x) - >>> b.calls - 35 - >>> a.method(0) - 0 - >>> a.calls - 75 - - Note that if method had been decorated with ``functools.lru_cache()``, - a.calls would have been 76 (due to the cached value of 0 having been - flushed by the 'b' instance). - - Clear the cache with ``.cache_clear()`` - - >>> a.method.cache_clear() - - Same for a method that hasn't yet been called. - - >>> c = MyClass() - >>> c.method.cache_clear() - - Another cache wrapper may be supplied: - - >>> cache = functools.lru_cache(maxsize=2) - >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) - >>> a = MyClass() - >>> a.method2() - 3 - - Caution - do not subsequently wrap the method with another decorator, such - as ``@property``, which changes the semantics of the function. - - See also - http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ - for another implementation and additional justification. - """ - cache_wrapper = cache_wrapper or functools.lru_cache() - - def wrapper(self, *args, **kwargs): - # it's the first call, replace the method with a cached, bound method - bound_method = types.MethodType(method, self) - cached_method = cache_wrapper(bound_method) - setattr(self, method.__name__, cached_method) - return cached_method(*args, **kwargs) - - # Support cache clear even before cache has been created. - wrapper.cache_clear = lambda: None - - return wrapper - - -# From jaraco.functools 3.3 -def pass_none(func): - """ - Wrap func so it's not called if its first param is None - - >>> print_text = pass_none(print) - >>> print_text('text') - text - >>> print_text(None) - """ - - @functools.wraps(func) - def wrapper(param, *args, **kwargs): - if param is not None: - return func(param, *args, **kwargs) - - return wrapper diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py deleted file mode 100644 index d4ca9b9140e..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_itertools.py +++ /dev/null @@ -1,73 +0,0 @@ -from itertools import filterfalse - - -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - -# copied from more_itertools 8.8 -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py deleted file mode 100644 index 259b15ba194..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_meta.py +++ /dev/null @@ -1,49 +0,0 @@ -from ._compat import Protocol -from typing import Any, Dict, Iterator, List, TypeVar, Union - - -_T = TypeVar("_T") - - -class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover - - def __contains__(self, item: str) -> bool: - ... # pragma: no cover - - def __getitem__(self, key: str) -> str: - ... # pragma: no cover - - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover - - def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: - """ - Return all values associated with a possibly multi-valued key. - """ - - @property - def json(self) -> Dict[str, Union[str, List[str]]]: - """ - A JSON-compatible form of the metadata. - """ - - -class SimplePath(Protocol[_T]): - """ - A minimal subset of pathlib.Path required by PathDistribution. - """ - - def joinpath(self) -> _T: - ... # pragma: no cover - - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover - - @property - def parent(self) -> _T: - ... # pragma: no cover - - def read_text(self) -> str: - ... # pragma: no cover diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py deleted file mode 100644 index cde4558fbbe..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_py39compat.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Compatibility layer with Python 3.8/3.9 -""" -from typing import TYPE_CHECKING, Any, Optional - -if TYPE_CHECKING: # pragma: no cover - # Prevent circular imports on runtime. - from . import Distribution, EntryPoint -else: - Distribution = EntryPoint = Any - - -def normalized_name(dist: Distribution) -> Optional[str]: - """ - Honor name normalization for distributions that don't provide ``_normalized_name``. - """ - try: - return dist._normalized_name - except AttributeError: - from . import Prepared # -> delay to prevent circular imports. - - return Prepared.normalize(getattr(dist, "name", None) or dist.metadata['Name']) - - -def ep_matches(ep: EntryPoint, **params) -> bool: - """ - Workaround for ``EntryPoint`` objects without the ``matches`` method. - """ - try: - return ep.matches(**params) - except AttributeError: - from . import EntryPoint # -> delay to prevent circular imports. - - # Reconstruct the EntryPoint object to make sure it is compatible. - return EntryPoint(ep.name, ep.value, ep.group).matches(**params) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py deleted file mode 100644 index c88cfbb2349..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/_text.py +++ /dev/null @@ -1,99 +0,0 @@ -import re - -from ._functools import method_cache - - -# from jaraco.text 3.5 -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use in_: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super().lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super().lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py deleted file mode 100644 index 34e3a9950cc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Read resources contained within a package.""" - -from ._common import ( - as_file, - files, - Package, -) - -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - -from .abc import ResourceReader - - -__all__ = [ - 'Package', - 'Resource', - 'ResourceReader', - 'as_file', - 'contents', - 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', -] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py deleted file mode 100644 index ea363d86a56..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_adapters.py +++ /dev/null @@ -1,170 +0,0 @@ -from contextlib import suppress -from io import TextIOWrapper - -from . import abc - - -class SpecLoaderAdapter: - """ - Adapt a package spec to adapt the underlying loader. - """ - - def __init__(self, spec, adapter=lambda spec: spec.loader): - self.spec = spec - self.loader = adapter(spec) - - def __getattr__(self, name): - return getattr(self.spec, name) - - -class TraversableResourcesLoader: - """ - Adapt a loader to provide TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - def get_resource_reader(self, name): - return CompatibilityFiles(self.spec)._native() - - -def _io_wrapper(file, mode='r', *args, **kwargs): - if mode == 'r': - return TextIOWrapper(file, *args, **kwargs) - elif mode == 'rb': - return file - raise ValueError( - "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode) - ) - - -class CompatibilityFiles: - """ - Adapter for an existing or non-existent resource reader - to provide a compatibility .files(). - """ - - class SpecPath(abc.Traversable): - """ - Path tied to a module spec. - Can be read and exposes the resource reader children. - """ - - def __init__(self, spec, reader): - self._spec = spec - self._reader = reader - - def iterdir(self): - if not self._reader: - return iter(()) - return iter( - CompatibilityFiles.ChildPath(self._reader, path) - for path in self._reader.contents() - ) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - if not self._reader: - return CompatibilityFiles.OrphanPath(other) - return CompatibilityFiles.ChildPath(self._reader, other) - - @property - def name(self): - return self._spec.name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) - - class ChildPath(abc.Traversable): - """ - Path tied to a resource reader child. - Can be read but doesn't expose any meaningful children. - """ - - def __init__(self, reader, name): - self._reader = reader - self._name = name - - def iterdir(self): - return iter(()) - - def is_file(self): - return self._reader.is_resource(self.name) - - def is_dir(self): - return not self.is_file() - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(self.name, other) - - @property - def name(self): - return self._name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper( - self._reader.open_resource(self.name), mode, *args, **kwargs - ) - - class OrphanPath(abc.Traversable): - """ - Orphan path, not tied to a module spec or resource reader. - Can't be read and doesn't expose any meaningful children. - """ - - def __init__(self, *path_parts): - if len(path_parts) < 1: - raise ValueError('Need at least one path part to construct a path') - self._path = path_parts - - def iterdir(self): - return iter(()) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(*self._path, other) - - @property - def name(self): - return self._path[-1] - - def open(self, mode='r', *args, **kwargs): - raise FileNotFoundError("Can't open orphan path") - - def __init__(self, spec): - self.spec = spec - - @property - def _reader(self): - with suppress(AttributeError): - return self.spec.loader.get_resource_reader(self.spec.name) - - def _native(self): - """ - Return the native reader if it supports files(). - """ - reader = self._reader - return reader if hasattr(reader, 'files') else self - - def __getattr__(self, attr): - return getattr(self._reader, attr) - - def files(self): - return CompatibilityFiles.SpecPath(self.spec, self._reader) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - """ - return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py deleted file mode 100644 index 3c6de1cfb2e..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_common.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -import pathlib -import tempfile -import functools -import contextlib -import types -import importlib -import inspect -import warnings -import itertools - -from typing import Union, Optional, cast -from .abc import ResourceReader, Traversable - -from ._compat import wrap_spec - -Package = Union[types.ModuleType, str] -Anchor = Package - - -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor -def files(anchor: Optional[Anchor] = None) -> Traversable: - """ - Get a Traversable resource for an anchor. - """ - return from_package(resolve(anchor)) - - -def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: - """ - Return the package's loader if it's a ResourceReader. - """ - # We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore - if reader is None: - return None - return reader(spec.name) # type: ignore - - -@functools.singledispatch -def resolve(cand: Optional[Anchor]) -> types.ModuleType: - return cast(types.ModuleType, cand) - - -@resolve.register -def _(cand: str) -> types.ModuleType: - return importlib.import_module(cand) - - -@resolve.register -def _(cand: None) -> types.ModuleType: - return resolve(_infer_caller().f_globals['__name__']) - - -def _infer_caller(): - """ - Walk the stack and find the frame of the first caller not in this module. - """ - - def is_this_file(frame_info): - return frame_info.filename == __file__ - - def is_wrapper(frame_info): - return frame_info.function == 'wrapper' - - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) - # also exclude 'wrapper' due to singledispatch in the call stack - callers = itertools.filterfalse(is_wrapper, not_this_file) - return next(callers).frame - - -def from_package(package: types.ModuleType): - """ - Return a Traversable object for the given package. - - """ - spec = wrap_spec(package) - reader = spec.loader.get_resource_reader(spec.name) - return reader.files() - - -@contextlib.contextmanager -def _tempfile( - reader, - suffix='', - # gh-93353: Keep a reference to call os.remove() in late Python - # finalization. - *, - _os_remove=os.remove, -): - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on Windows - # properly. - fd, raw_path = tempfile.mkstemp(suffix=suffix) - try: - try: - os.write(fd, reader()) - finally: - os.close(fd) - del reader - yield pathlib.Path(raw_path) - finally: - try: - _os_remove(raw_path) - except FileNotFoundError: - pass - - -def _temp_file(path): - return _tempfile(path.read_bytes, suffix=path.name) - - -def _is_present_dir(path: Traversable) -> bool: - """ - Some Traversables implement ``is_dir()`` to raise an - exception (i.e. ``FileNotFoundError``) when the - directory doesn't exist. This function wraps that call - to always return a boolean and only return True - if there's a dir and it exists. - """ - with contextlib.suppress(FileNotFoundError): - return path.is_dir() - return False - - -@functools.singledispatch -def as_file(path): - """ - Given a Traversable object, return that object as a - path on the local file system in a context manager. - """ - return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) - - -@as_file.register(pathlib.Path) -@contextlib.contextmanager -def _(path): - """ - Degenerate behavior for pathlib.Path objects. - """ - yield path - - -@contextlib.contextmanager -def _temp_path(dir: tempfile.TemporaryDirectory): - """ - Wrap tempfile.TemporyDirectory to return a pathlib object. - """ - with dir as result: - yield pathlib.Path(result) - - -@contextlib.contextmanager -def _temp_dir(path): - """ - Given a traversable dir, recursively replicate the whole tree - to the file system in a context manager. - """ - assert path.is_dir() - with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: - yield _write_contents(temp_dir, path) - - -def _write_contents(target, source): - child = target.joinpath(source.name) - if source.is_dir(): - child.mkdir() - for item in source.iterdir(): - _write_contents(child, item) - else: - child.write_bytes(source.read_bytes()) - return child diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py deleted file mode 100644 index 8b5b1d280f3..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_compat.py +++ /dev/null @@ -1,108 +0,0 @@ -# flake8: noqa - -import abc -import os -import sys -import pathlib -from contextlib import suppress -from typing import Union - - -if sys.version_info >= (3, 10): - from zipfile import Path as ZipPath # type: ignore -else: - from ..zipp import Path as ZipPath # type: ignore - - -try: - from typing import runtime_checkable # type: ignore -except ImportError: - - def runtime_checkable(cls): # type: ignore - return cls - - -try: - from typing import Protocol # type: ignore -except ImportError: - Protocol = abc.ABC # type: ignore - - -class TraversableResourcesLoader: - """ - Adapt loaders to provide TraversableResources and other - compatibility. - - Used primarily for Python 3.9 and earlier where the native - loaders do not yet implement TraversableResources. - """ - - def __init__(self, spec): - self.spec = spec - - @property - def path(self): - return self.spec.origin - - def get_resource_reader(self, name): - from . import readers, _adapters - - def _zip_reader(spec): - with suppress(AttributeError): - return readers.ZipReader(spec.loader, spec.name) - - def _namespace_reader(spec): - with suppress(AttributeError, ValueError): - return readers.NamespaceReader(spec.submodule_search_locations) - - def _available_reader(spec): - with suppress(AttributeError): - return spec.loader.get_resource_reader(spec.name) - - def _native_reader(spec): - reader = _available_reader(spec) - return reader if hasattr(reader, 'files') else None - - def _file_reader(spec): - try: - path = pathlib.Path(self.path) - except TypeError: - return None - if path.exists(): - return readers.FileReader(self) - - return ( - # native reader if it supplies 'files' - _native_reader(self.spec) - or - # local ZipReader if a zip module - _zip_reader(self.spec) - or - # local NamespaceReader if a namespace module - _namespace_reader(self.spec) - or - # local FileReader - _file_reader(self.spec) - # fallback - adapt the spec ResourceReader to TraversableReader - or _adapters.CompatibilityFiles(self.spec) - ) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - - Supersedes _adapters.wrap_spec to use TraversableResourcesLoader - from above for older Python compatibility (<3.10). - """ - from . import _adapters - - return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) - - -if sys.version_info >= (3, 9): - StrPath = Union[str, os.PathLike[str]] -else: - # PathLike is only subscriptable at runtime in 3.9+ - StrPath = Union[str, "os.PathLike[str]"] diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py deleted file mode 100644 index cce05582ffc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_itertools.py +++ /dev/null @@ -1,35 +0,0 @@ -from itertools import filterfalse - -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py deleted file mode 100644 index 23b6aeafe4f..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/abc.py +++ /dev/null @@ -1,170 +0,0 @@ -import abc -import io -import itertools -import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional - -from ._compat import runtime_checkable, Protocol, StrPath - - -__all__ = ["ResourceReader", "Traversable", "TraversableResources"] - - -class ResourceReader(metaclass=abc.ABCMeta): - """Abstract base class for loaders to provide resource reading support.""" - - @abc.abstractmethod - def open_resource(self, resource: Text) -> BinaryIO: - """Return an opened, file-like object for binary reading. - - The 'resource' argument is expected to represent only a file name. - If the resource cannot be found, FileNotFoundError is raised. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def resource_path(self, resource: Text) -> Text: - """Return the file system path to the specified resource. - - The 'resource' argument is expected to represent only a file name. - If the resource does not exist on the file system, raise - FileNotFoundError. - """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. - raise FileNotFoundError - - @abc.abstractmethod - def is_resource(self, path: Text) -> bool: - """Return True if the named 'path' is a resource. - - Files are resources, directories are not. - """ - raise FileNotFoundError - - @abc.abstractmethod - def contents(self) -> Iterable[str]: - """Return an iterable of entries in `package`.""" - raise FileNotFoundError - - -class TraversalError(Exception): - pass - - -@runtime_checkable -class Traversable(Protocol): - """ - An object with a subset of pathlib.Path methods suitable for - traversing directories and opening files. - - Any exceptions that occur when accessing the backing resource - may propagate unaltered. - """ - - @abc.abstractmethod - def iterdir(self) -> Iterator["Traversable"]: - """ - Yield Traversable objects in self - """ - - def read_bytes(self) -> bytes: - """ - Read contents of self as bytes - """ - with self.open('rb') as strm: - return strm.read() - - def read_text(self, encoding: Optional[str] = None) -> str: - """ - Read contents of self as text - """ - with self.open(encoding=encoding) as strm: - return strm.read() - - @abc.abstractmethod - def is_dir(self) -> bool: - """ - Return True if self is a directory - """ - - @abc.abstractmethod - def is_file(self) -> bool: - """ - Return True if self is a file - """ - - def joinpath(self, *descendants: StrPath) -> "Traversable": - """ - Return Traversable resolved with any descendants applied. - - Each descendant should be a path segment relative to self - and each may contain multiple levels separated by - ``posixpath.sep`` (``/``). - """ - if not descendants: - return self - names = itertools.chain.from_iterable( - path.parts for path in map(pathlib.PurePosixPath, descendants) - ) - target = next(names) - matches = ( - traversable for traversable in self.iterdir() if traversable.name == target - ) - try: - match = next(matches) - except StopIteration: - raise TraversalError( - "Target not found during traversal.", target, list(names) - ) - return match.joinpath(*names) - - def __truediv__(self, child: StrPath) -> "Traversable": - """ - Return Traversable child in self - """ - return self.joinpath(child) - - @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): - """ - mode may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as pathlib.Path.open). - - When opening as text, accepts encoding parameters such as those - accepted by io.TextIOWrapper. - """ - - @property - @abc.abstractmethod - def name(self) -> str: - """ - The base name of this object without any parent references. - """ - - -class TraversableResources(ResourceReader): - """ - The required interface for providing traversable - resources. - """ - - @abc.abstractmethod - def files(self) -> "Traversable": - """Return a Traversable object for the loaded package.""" - - def open_resource(self, resource: StrPath) -> io.BufferedReader: - return self.files().joinpath(resource).open('rb') - - def resource_path(self, resource: Any) -> NoReturn: - raise FileNotFoundError(resource) - - def is_resource(self, path: StrPath) -> bool: - return self.files().joinpath(path).is_file() - - def contents(self) -> Iterator[str]: - return (item.name for item in self.files().iterdir()) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py deleted file mode 100644 index ab34db74091..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/readers.py +++ /dev/null @@ -1,120 +0,0 @@ -import collections -import pathlib -import operator - -from . import abc - -from ._itertools import unique_everseen -from ._compat import ZipPath - - -def remove_duplicates(items): - return iter(collections.OrderedDict.fromkeys(items)) - - -class FileReader(abc.TraversableResources): - def __init__(self, loader): - self.path = pathlib.Path(loader.path).parent - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path - - -class ZipReader(abc.TraversableResources): - def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' - self.archive = loader.archive - - def open_resource(self, resource): - try: - return super().open_resource(resource) - except KeyError as exc: - raise FileNotFoundError(exc.args[0]) - - def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. - target = self.files().joinpath(path) - return target.is_file() and target.exists() - - def files(self): - return ZipPath(self.archive, self.prefix) - - -class MultiplexedPath(abc.Traversable): - """ - Given a series of Traversable objects, implement a merged - version of the interface across all objects. Useful for - namespace packages which may be multihomed at a single - name. - """ - - def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) - if not self._paths: - message = 'MultiplexedPath must contain at least one path' - raise FileNotFoundError(message) - if not all(path.is_dir() for path in self._paths): - raise NotADirectoryError('MultiplexedPath only supports directories') - - def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) - - def read_bytes(self): - raise FileNotFoundError(f'{self} is not a file') - - def read_text(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - def is_dir(self): - return True - - def is_file(self): - return False - - def joinpath(self, *descendants): - try: - return super().joinpath(*descendants) - except abc.TraversalError: - # One of the paths did not resolve (a directory does not exist). - # Just return something that will not exist. - return self._paths[0].joinpath(*descendants) - - def open(self, *args, **kwargs): - raise FileNotFoundError(f'{self} is not a file') - - @property - def name(self): - return self._paths[0].name - - def __repr__(self): - paths = ', '.join(f"'{path}'" for path in self._paths) - return f'MultiplexedPath({paths})' - - -class NamespaceReader(abc.TraversableResources): - def __init__(self, namespace_path): - if 'NamespacePath' not in str(namespace_path): - raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) - - def resource_path(self, resource): - """ - Return the file system path to prevent - `resources.path()` from creating a temporary - copy. - """ - return str(self.path.joinpath(resource)) - - def files(self): - return self.path diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py b/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py deleted file mode 100644 index 7770c922c84..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_resources/simple.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Interface adapters for low-level readers. -""" - -import abc -import io -import itertools -from typing import BinaryIO, List - -from .abc import Traversable, TraversableResources - - -class SimpleReader(abc.ABC): - """ - The minimum, low-level interface required from a resource - provider. - """ - - @property - @abc.abstractmethod - def package(self) -> str: - """ - The name of the package for which this reader loads resources. - """ - - @abc.abstractmethod - def children(self) -> List['SimpleReader']: - """ - Obtain an iterable of SimpleReader for available - child containers (e.g. directories). - """ - - @abc.abstractmethod - def resources(self) -> List[str]: - """ - Obtain available named resources for this virtual package. - """ - - @abc.abstractmethod - def open_binary(self, resource: str) -> BinaryIO: - """ - Obtain a File-like for a named resource. - """ - - @property - def name(self): - return self.package.split('.')[-1] - - -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader: SimpleReader): - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - -class ResourceHandle(Traversable): - """ - Handle to a named resource in a ResourceReader. - """ - - def __init__(self, parent: ResourceContainer, name: str): - self.parent = parent - self.name = name # type: ignore - - def is_file(self): - return True - - def is_dir(self): - return False - - def open(self, mode='r', *args, **kwargs): - stream = self.parent.reader.open_binary(self.name) - if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) - return stream - - def joinpath(self, name): - raise RuntimeError("Cannot traverse into a resource") - - -class TraversableReader(TraversableResources, SimpleReader): - """ - A TraversableResources based on SimpleReader. Resource providers - may derive from this class to provide the TraversableResources - interface by supplying the SimpleReader interface. - """ - - def files(self): - return ResourceContainer(self) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/__init__.py +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/functools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py deleted file mode 100644 index a0306d5ff5c..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/jaraco/text/__init__.py +++ /dev/null @@ -1,599 +0,0 @@ -import re -import itertools -import textwrap -import functools - -try: - from importlib.resources import files # type: ignore -except ImportError: # pragma: nocover - from setuptools.extern.importlib_resources import files # type: ignore - -from setuptools.extern.jaraco.functools import compose, method_cache -from setuptools.extern.jaraco.context import ExceptionTrap - - -def substitution(old, new): - """ - Return a function that will perform a substitution on a string - """ - return lambda s: s.replace(old, new) - - -def multi_substitution(*substitutions): - """ - Take a sequence of pairs specifying substitutions, and create - a function that performs those substitutions. - - >>> multi_substitution(('foo', 'bar'), ('bar', 'baz'))('foo') - 'baz' - """ - substitutions = itertools.starmap(substitution, substitutions) - # compose function applies last function first, so reverse the - # substitutions to get the expected order. - substitutions = reversed(tuple(substitutions)) - return compose(*substitutions) - - -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use ``in_``: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super().lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super().lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) - - -# Python 3.8 compatibility -_unicode_trap = ExceptionTrap(UnicodeDecodeError) - - -@_unicode_trap.passes -def is_decodable(value): - r""" - Return True if the supplied value is decodable (using the default - encoding). - - >>> is_decodable(b'\xff') - False - >>> is_decodable(b'\x32') - True - """ - value.decode() - - -def is_binary(value): - r""" - Return True if the value appears to be binary (that is, it's a byte - string and isn't decodable). - - >>> is_binary(b'\xff') - True - >>> is_binary('\xff') - False - """ - return isinstance(value, bytes) and not is_decodable(value) - - -def trim(s): - r""" - Trim something like a docstring to remove the whitespace that - is common due to indentation and formatting. - - >>> trim("\n\tfoo = bar\n\t\tbar = baz\n") - 'foo = bar\n\tbar = baz' - """ - return textwrap.dedent(s).strip() - - -def wrap(s): - """ - Wrap lines of text, retaining existing newlines as - paragraph markers. - - >>> print(wrap(lorem_ipsum)) - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do - eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad - minim veniam, quis nostrud exercitation ullamco laboris nisi ut - aliquip ex ea commodo consequat. Duis aute irure dolor in - reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla - pariatur. Excepteur sint occaecat cupidatat non proident, sunt in - culpa qui officia deserunt mollit anim id est laborum. - <BLANKLINE> - Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam - varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus - magna felis sollicitudin mauris. Integer in mauris eu nibh euismod - gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis - risus a elit. Etiam tempor. Ut ullamcorper, ligula eu tempor congue, - eros est euismod turpis, id tincidunt sapien risus a quam. Maecenas - fermentum consequat mi. Donec fermentum. Pellentesque malesuada nulla - a mi. Duis sapien sem, aliquet nec, commodo eget, consequat quis, - neque. Aliquam faucibus, elit ut dictum aliquet, felis nisl adipiscing - sapien, sed malesuada diam lacus eget erat. Cras mollis scelerisque - nunc. Nullam arcu. Aliquam consequat. Curabitur augue lorem, dapibus - quis, laoreet et, pretium ac, nisi. Aenean magna nisl, mollis quis, - molestie eu, feugiat in, orci. In hac habitasse platea dictumst. - """ - paragraphs = s.splitlines() - wrapped = ('\n'.join(textwrap.wrap(para)) for para in paragraphs) - return '\n\n'.join(wrapped) - - -def unwrap(s): - r""" - Given a multi-line string, return an unwrapped version. - - >>> wrapped = wrap(lorem_ipsum) - >>> wrapped.count('\n') - 20 - >>> unwrapped = unwrap(wrapped) - >>> unwrapped.count('\n') - 1 - >>> print(unwrapped) - Lorem ipsum dolor sit amet, consectetur adipiscing ... - Curabitur pretium tincidunt lacus. Nulla gravida orci ... - - """ - paragraphs = re.split(r'\n\n+', s) - cleaned = (para.replace('\n', ' ') for para in paragraphs) - return '\n'.join(cleaned) - - - - -class Splitter(object): - """object that will split a string with the given arguments for each call - - >>> s = Splitter(',') - >>> s('hello, world, this is your, master calling') - ['hello', ' world', ' this is your', ' master calling'] - """ - - def __init__(self, *args): - self.args = args - - def __call__(self, s): - return s.split(*self.args) - - -def indent(string, prefix=' ' * 4): - """ - >>> indent('foo') - ' foo' - """ - return prefix + string - - -class WordSet(tuple): - """ - Given an identifier, return the words that identifier represents, - whether in camel case, underscore-separated, etc. - - >>> WordSet.parse("camelCase") - ('camel', 'Case') - - >>> WordSet.parse("under_sep") - ('under', 'sep') - - Acronyms should be retained - - >>> WordSet.parse("firstSNL") - ('first', 'SNL') - - >>> WordSet.parse("you_and_I") - ('you', 'and', 'I') - - >>> WordSet.parse("A simple test") - ('A', 'simple', 'test') - - Multiple caps should not interfere with the first cap of another word. - - >>> WordSet.parse("myABCClass") - ('my', 'ABC', 'Class') - - The result is a WordSet, so you can get the form you need. - - >>> WordSet.parse("myABCClass").underscore_separated() - 'my_ABC_Class' - - >>> WordSet.parse('a-command').camel_case() - 'ACommand' - - >>> WordSet.parse('someIdentifier').lowered().space_separated() - 'some identifier' - - Slices of the result should return another WordSet. - - >>> WordSet.parse('taken-out-of-context')[1:].underscore_separated() - 'out_of_context' - - >>> WordSet.from_class_name(WordSet()).lowered().space_separated() - 'word set' - - >>> example = WordSet.parse('figured it out') - >>> example.headless_camel_case() - 'figuredItOut' - >>> example.dash_separated() - 'figured-it-out' - - """ - - _pattern = re.compile('([A-Z]?[a-z]+)|([A-Z]+(?![a-z]))') - - def capitalized(self): - return WordSet(word.capitalize() for word in self) - - def lowered(self): - return WordSet(word.lower() for word in self) - - def camel_case(self): - return ''.join(self.capitalized()) - - def headless_camel_case(self): - words = iter(self) - first = next(words).lower() - new_words = itertools.chain((first,), WordSet(words).camel_case()) - return ''.join(new_words) - - def underscore_separated(self): - return '_'.join(self) - - def dash_separated(self): - return '-'.join(self) - - def space_separated(self): - return ' '.join(self) - - def trim_right(self, item): - """ - Remove the item from the end of the set. - - >>> WordSet.parse('foo bar').trim_right('foo') - ('foo', 'bar') - >>> WordSet.parse('foo bar').trim_right('bar') - ('foo',) - >>> WordSet.parse('').trim_right('bar') - () - """ - return self[:-1] if self and self[-1] == item else self - - def trim_left(self, item): - """ - Remove the item from the beginning of the set. - - >>> WordSet.parse('foo bar').trim_left('foo') - ('bar',) - >>> WordSet.parse('foo bar').trim_left('bar') - ('foo', 'bar') - >>> WordSet.parse('').trim_left('bar') - () - """ - return self[1:] if self and self[0] == item else self - - def trim(self, item): - """ - >>> WordSet.parse('foo bar').trim('foo') - ('bar',) - """ - return self.trim_left(item).trim_right(item) - - def __getitem__(self, item): - result = super(WordSet, self).__getitem__(item) - if isinstance(item, slice): - result = WordSet(result) - return result - - @classmethod - def parse(cls, identifier): - matches = cls._pattern.finditer(identifier) - return WordSet(match.group(0) for match in matches) - - @classmethod - def from_class_name(cls, subject): - return cls.parse(subject.__class__.__name__) - - -# for backward compatibility -words = WordSet.parse - - -def simple_html_strip(s): - r""" - Remove HTML from the string `s`. - - >>> str(simple_html_strip('')) - '' - - >>> print(simple_html_strip('A <bold>stormy</bold> day in paradise')) - A stormy day in paradise - - >>> print(simple_html_strip('Somebody <!-- do not --> tell the truth.')) - Somebody tell the truth. - - >>> print(simple_html_strip('What about<br/>\nmultiple lines?')) - What about - multiple lines? - """ - html_stripper = re.compile('(<!--.*?-->)|(<[^>]*>)|([^<]+)', re.DOTALL) - texts = (match.group(3) or '' for match in html_stripper.finditer(s)) - return ''.join(texts) - - -class SeparatedValues(str): - """ - A string separated by a separator. Overrides __iter__ for getting - the values. - - >>> list(SeparatedValues('a,b,c')) - ['a', 'b', 'c'] - - Whitespace is stripped and empty values are discarded. - - >>> list(SeparatedValues(' a, b , c, ')) - ['a', 'b', 'c'] - """ - - separator = ',' - - def __iter__(self): - parts = self.split(self.separator) - return filter(None, (part.strip() for part in parts)) - - -class Stripper: - r""" - Given a series of lines, find the common prefix and strip it from them. - - >>> lines = [ - ... 'abcdefg\n', - ... 'abc\n', - ... 'abcde\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix - 'abc' - >>> list(res.lines) - ['defg\n', '\n', 'de\n'] - - If no prefix is common, nothing should be stripped. - - >>> lines = [ - ... 'abcd\n', - ... '1234\n', - ... ] - >>> res = Stripper.strip_prefix(lines) - >>> res.prefix = '' - >>> list(res.lines) - ['abcd\n', '1234\n'] - """ - - def __init__(self, prefix, lines): - self.prefix = prefix - self.lines = map(self, lines) - - @classmethod - def strip_prefix(cls, lines): - prefix_lines, lines = itertools.tee(lines) - prefix = functools.reduce(cls.common_prefix, prefix_lines) - return cls(prefix, lines) - - def __call__(self, line): - if not self.prefix: - return line - null, prefix, rest = line.partition(self.prefix) - return rest - - @staticmethod - def common_prefix(s1, s2): - """ - Return the common prefix of two lines. - """ - index = min(len(s1), len(s2)) - while s1[:index] != s2[:index]: - index -= 1 - return s1[:index] - - -def remove_prefix(text, prefix): - """ - Remove the prefix from the text if it exists. - - >>> remove_prefix('underwhelming performance', 'underwhelming ') - 'performance' - - >>> remove_prefix('something special', 'sample') - 'something special' - """ - null, prefix, rest = text.rpartition(prefix) - return rest - - -def remove_suffix(text, suffix): - """ - Remove the suffix from the text if it exists. - - >>> remove_suffix('name.git', '.git') - 'name' - - >>> remove_suffix('something special', 'sample') - 'something special' - """ - rest, suffix, null = text.partition(suffix) - return rest - - -def normalize_newlines(text): - r""" - Replace alternate newlines with the canonical newline. - - >>> normalize_newlines('Lorem Ipsum\u2029') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\r\n') - 'Lorem Ipsum\n' - >>> normalize_newlines('Lorem Ipsum\x85') - 'Lorem Ipsum\n' - """ - newlines = ['\r\n', '\r', '\n', '\u0085', '\u2028', '\u2029'] - pattern = '|'.join(newlines) - return re.sub(pattern, '\n', text) - - -def _nonblank(str): - return str and not str.startswith('#') - - -@functools.singledispatch -def yield_lines(iterable): - r""" - Yield valid lines of a string or iterable. - - >>> list(yield_lines('')) - [] - >>> list(yield_lines(['foo', 'bar'])) - ['foo', 'bar'] - >>> list(yield_lines('foo\nbar')) - ['foo', 'bar'] - >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) - ['foo', 'baz #comment'] - >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) - ['foo', 'bar', 'baz', 'bing'] - """ - return itertools.chain.from_iterable(map(yield_lines, iterable)) - - -@yield_lines.register(str) -def _(text): - return filter(_nonblank, map(str.strip, text.splitlines())) - - -def drop_comment(line): - """ - Drop comments. - - >>> drop_comment('foo # bar') - 'foo' - - A hash without a space may be in a URL. - - >>> drop_comment('http://example.com/foo#bar') - 'http://example.com/foo#bar' - """ - return line.partition(' #')[0] - - -def join_continuation(lines): - r""" - Join lines continued by a trailing backslash. - - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar', 'baz'])) - ['foobar', 'baz'] - >>> list(join_continuation(['foo \\', 'bar \\', 'baz'])) - ['foobarbaz'] - - Not sure why, but... - The character preceeding the backslash is also elided. - - >>> list(join_continuation(['goo\\', 'dly'])) - ['godly'] - - A terrible idea, but... - If no line is available to continue, suppress the lines. - - >>> list(join_continuation(['foo', 'bar\\', 'baz\\'])) - ['foo'] - """ - lines = iter(lines) - for item in lines: - while item.endswith('\\'): - try: - item = item[:-2].strip() + next(lines) - except StopIteration: - return - yield item diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py deleted file mode 100644 index 19a169fc301..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .more import * # noqa -from .recipes import * # noqa - -__version__ = '8.8.0' diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi deleted file mode 100644 index 96f6e36c7f4..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/__init__.pyi +++ /dev/null @@ -1,2 +0,0 @@ -from .more import * -from .recipes import * diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py deleted file mode 100644 index e6fca4d47f6..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.py +++ /dev/null @@ -1,3824 +0,0 @@ -import warnings - -from collections import Counter, defaultdict, deque, abc -from collections.abc import Sequence -from functools import partial, reduce, wraps -from heapq import merge, heapify, heapreplace, heappop -from itertools import ( - chain, - compress, - count, - cycle, - dropwhile, - groupby, - islice, - repeat, - starmap, - takewhile, - tee, - zip_longest, -) -from math import exp, factorial, floor, log -from queue import Empty, Queue -from random import random, randrange, uniform -from operator import itemgetter, mul, sub, gt, lt -from sys import hexversion, maxsize -from time import monotonic - -from .recipes import ( - consume, - flatten, - pairwise, - powerset, - take, - unique_everseen, -) - -__all__ = [ - 'AbortThread', - 'adjacent', - 'always_iterable', - 'always_reversible', - 'bucket', - 'callback_iter', - 'chunked', - 'circular_shifts', - 'collapse', - 'collate', - 'consecutive_groups', - 'consumer', - 'countable', - 'count_cycle', - 'mark_ends', - 'difference', - 'distinct_combinations', - 'distinct_permutations', - 'distribute', - 'divide', - 'exactly_n', - 'filter_except', - 'first', - 'groupby_transform', - 'ilen', - 'interleave_longest', - 'interleave', - 'intersperse', - 'islice_extended', - 'iterate', - 'ichunked', - 'is_sorted', - 'last', - 'locate', - 'lstrip', - 'make_decorator', - 'map_except', - 'map_reduce', - 'nth_or_last', - 'nth_permutation', - 'nth_product', - 'numeric_range', - 'one', - 'only', - 'padded', - 'partitions', - 'set_partitions', - 'peekable', - 'repeat_last', - 'replace', - 'rlocate', - 'rstrip', - 'run_length', - 'sample', - 'seekable', - 'SequenceView', - 'side_effect', - 'sliced', - 'sort_together', - 'split_at', - 'split_after', - 'split_before', - 'split_when', - 'split_into', - 'spy', - 'stagger', - 'strip', - 'substrings', - 'substrings_indexes', - 'time_limited', - 'unique_to_each', - 'unzip', - 'windowed', - 'with_iter', - 'UnequalIterablesError', - 'zip_equal', - 'zip_offset', - 'windowed_complete', - 'all_unique', - 'value_chain', - 'product_index', - 'combination_index', - 'permutation_index', -] - -_marker = object() - - -def chunked(iterable, n, strict=False): - """Break *iterable* into lists of length *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) - [[1, 2, 3], [4, 5, 6]] - - By the default, the last yielded list will have fewer than *n* elements - if the length of *iterable* is not divisible by *n*: - - >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) - [[1, 2, 3], [4, 5, 6], [7, 8]] - - To use a fill-in value instead, see the :func:`grouper` recipe. - - If the length of *iterable* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - list is yielded. - - """ - iterator = iter(partial(take, n, iter(iterable)), []) - if strict: - - def ret(): - for chunk in iterator: - if len(chunk) != n: - raise ValueError('iterable is not divisible by n.') - yield chunk - - return iter(ret()) - else: - return iterator - - -def first(iterable, default=_marker): - """Return the first item of *iterable*, or *default* if *iterable* is - empty. - - >>> first([0, 1, 2, 3]) - 0 - >>> first([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - - :func:`first` is useful when you have a generator of expensive-to-retrieve - values and want any arbitrary one. It is marginally shorter than - ``next(iter(iterable), default)``. - - """ - try: - return next(iter(iterable)) - except StopIteration as e: - if default is _marker: - raise ValueError( - 'first() was called on an empty iterable, and no ' - 'default value was provided.' - ) from e - return default - - -def last(iterable, default=_marker): - """Return the last item of *iterable*, or *default* if *iterable* is - empty. - - >>> last([0, 1, 2, 3]) - 3 - >>> last([], 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - try: - if isinstance(iterable, Sequence): - return iterable[-1] - # Work around https://bugs.python.org/issue38525 - elif hasattr(iterable, '__reversed__') and (hexversion != 0x030800F0): - return next(reversed(iterable)) - else: - return deque(iterable, maxlen=1)[-1] - except (IndexError, TypeError, StopIteration): - if default is _marker: - raise ValueError( - 'last() was called on an empty iterable, and no default was ' - 'provided.' - ) - return default - - -def nth_or_last(iterable, n, default=_marker): - """Return the nth or the last item of *iterable*, - or *default* if *iterable* is empty. - - >>> nth_or_last([0, 1, 2, 3], 2) - 2 - >>> nth_or_last([0, 1], 2) - 1 - >>> nth_or_last([], 0, 'some default') - 'some default' - - If *default* is not provided and there are no items in the iterable, - raise ``ValueError``. - """ - return last(islice(iterable, n + 1), default=default) - - -class peekable: - """Wrap an iterator to allow lookahead and prepending elements. - - Call :meth:`peek` on the result to get the value that will be returned - by :func:`next`. This won't advance the iterator: - - >>> p = peekable(['a', 'b']) - >>> p.peek() - 'a' - >>> next(p) - 'a' - - Pass :meth:`peek` a default value to return that instead of raising - ``StopIteration`` when the iterator is exhausted. - - >>> p = peekable([]) - >>> p.peek('hi') - 'hi' - - peekables also offer a :meth:`prepend` method, which "inserts" items - at the head of the iterable: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> p.peek() - 11 - >>> list(p) - [11, 12, 1, 2, 3] - - peekables can be indexed. Index 0 is the item that will be returned by - :func:`next`, index 1 is the item after that, and so on: - The values up to the given index will be cached. - - >>> p = peekable(['a', 'b', 'c', 'd']) - >>> p[0] - 'a' - >>> p[1] - 'b' - >>> next(p) - 'a' - - Negative indexes are supported, but be aware that they will cache the - remaining items in the source iterator, which may require significant - storage. - - To check whether a peekable is exhausted, check its truth value: - - >>> p = peekable(['a', 'b']) - >>> if p: # peekable has items - ... list(p) - ['a', 'b'] - >>> if not p: # peekable is exhausted - ... list(p) - [] - - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self._cache = deque() - - def __iter__(self): - return self - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - """Return the item that will be next returned from ``next()``. - - Return ``default`` if there are no items left. If ``default`` is not - provided, raise ``StopIteration``. - - """ - if not self._cache: - try: - self._cache.append(next(self._it)) - except StopIteration: - if default is _marker: - raise - return default - return self._cache[0] - - def prepend(self, *items): - """Stack up items to be the next ones returned from ``next()`` or - ``self.peek()``. The items will be returned in - first in, first out order:: - - >>> p = peekable([1, 2, 3]) - >>> p.prepend(10, 11, 12) - >>> next(p) - 10 - >>> list(p) - [11, 12, 1, 2, 3] - - It is possible, by prepending items, to "resurrect" a peekable that - previously raised ``StopIteration``. - - >>> p = peekable([]) - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - >>> p.prepend(1) - >>> next(p) - 1 - >>> next(p) - Traceback (most recent call last): - ... - StopIteration - - """ - self._cache.extendleft(reversed(items)) - - def __next__(self): - if self._cache: - return self._cache.popleft() - - return next(self._it) - - def _get_slice(self, index): - # Normalize the slice's arguments - step = 1 if (index.step is None) else index.step - if step > 0: - start = 0 if (index.start is None) else index.start - stop = maxsize if (index.stop is None) else index.stop - elif step < 0: - start = -1 if (index.start is None) else index.start - stop = (-maxsize - 1) if (index.stop is None) else index.stop - else: - raise ValueError('slice step cannot be zero') - - # If either the start or stop index is negative, we'll need to cache - # the rest of the iterable in order to slice from the right side. - if (start < 0) or (stop < 0): - self._cache.extend(self._it) - # Otherwise we'll need to find the rightmost index and cache to that - # point. - else: - n = min(max(start, stop) + 1, maxsize) - cache_len = len(self._cache) - if n >= cache_len: - self._cache.extend(islice(self._it, n - cache_len)) - - return list(self._cache)[index] - - def __getitem__(self, index): - if isinstance(index, slice): - return self._get_slice(index) - - cache_len = len(self._cache) - if index < 0: - self._cache.extend(self._it) - elif index >= cache_len: - self._cache.extend(islice(self._it, index + 1 - cache_len)) - - return self._cache[index] - - -def collate(*iterables, **kwargs): - """Return a sorted merge of the items from each of several already-sorted - *iterables*. - - >>> list(collate('ACDZ', 'AZ', 'JKL')) - ['A', 'A', 'C', 'D', 'J', 'K', 'L', 'Z', 'Z'] - - Works lazily, keeping only the next value from each iterable in memory. Use - :func:`collate` to, for example, perform a n-way mergesort of items that - don't fit in memory. - - If a *key* function is specified, the iterables will be sorted according - to its result: - - >>> key = lambda s: int(s) # Sort by numeric value, not by string - >>> list(collate(['1', '10'], ['2', '11'], key=key)) - ['1', '2', '10', '11'] - - - If the *iterables* are sorted in descending order, set *reverse* to - ``True``: - - >>> list(collate([5, 3, 1], [4, 2, 0], reverse=True)) - [5, 4, 3, 2, 1, 0] - - If the elements of the passed-in iterables are out of order, you might get - unexpected results. - - On Python 3.5+, this function is an alias for :func:`heapq.merge`. - - """ - warnings.warn( - "collate is no longer part of more_itertools, use heapq.merge", - DeprecationWarning, - ) - return merge(*iterables, **kwargs) - - -def consumer(func): - """Decorator that automatically advances a PEP-342-style "reverse iterator" - to its first yield point so you don't have to call ``next()`` on it - manually. - - >>> @consumer - ... def tally(): - ... i = 0 - ... while True: - ... print('Thing number %s is %s.' % (i, (yield))) - ... i += 1 - ... - >>> t = tally() - >>> t.send('red') - Thing number 0 is red. - >>> t.send('fish') - Thing number 1 is fish. - - Without the decorator, you would have to call ``next(t)`` before - ``t.send()`` could be used. - - """ - - @wraps(func) - def wrapper(*args, **kwargs): - gen = func(*args, **kwargs) - next(gen) - return gen - - return wrapper - - -def ilen(iterable): - """Return the number of items in *iterable*. - - >>> ilen(x for x in range(1000000) if x % 3 == 0) - 333334 - - This consumes the iterable, so handle with care. - - """ - # This approach was selected because benchmarks showed it's likely the - # fastest of the known implementations at the time of writing. - # See GitHub tracker: #236, #230. - counter = count() - deque(zip(iterable, counter), maxlen=0) - return next(counter) - - -def iterate(func, start): - """Return ``start``, ``func(start)``, ``func(func(start))``, ... - - >>> from itertools import islice - >>> list(islice(iterate(lambda x: 2*x, 1), 10)) - [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - - """ - while True: - yield start - start = func(start) - - -def with_iter(context_manager): - """Wrap an iterable in a ``with`` statement, so it closes once exhausted. - - For example, this will close the file when the iterator is exhausted:: - - upper_lines = (line.upper() for line in with_iter(open('foo'))) - - Any context manager which returns an iterable is a candidate for - ``with_iter``. - - """ - with context_manager as iterable: - yield from iterable - - -def one(iterable, too_short=None, too_long=None): - """Return the first item from *iterable*, which is expected to contain only - that item. Raise an exception if *iterable* is empty or has more than one - item. - - :func:`one` is useful for ensuring that an iterable contains only one item. - For example, it can be used to retrieve the result of a database query - that is expected to return a single row. - - If *iterable* is empty, ``ValueError`` will be raised. You may specify a - different exception with the *too_short* keyword: - - >>> it = [] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: too many items in iterable (expected 1)' - >>> too_short = IndexError('too few items') - >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - IndexError: too few items - - Similarly, if *iterable* contains more than one item, ``ValueError`` will - be raised. You may specify a different exception with the *too_long* - keyword: - - >>> it = ['too', 'many'] - >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 'too', - 'many', and perhaps more. - >>> too_long = RuntimeError - >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - RuntimeError - - Note that :func:`one` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check iterable - contents less destructively. - - """ - it = iter(iterable) - - try: - first_value = next(it) - except StopIteration as e: - raise ( - too_short or ValueError('too few items in iterable (expected 1)') - ) from e - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def distinct_permutations(iterable, r=None): - """Yield successive distinct permutations of the elements in *iterable*. - - >>> sorted(distinct_permutations([1, 0, 1])) - [(0, 1, 1), (1, 0, 1), (1, 1, 0)] - - Equivalent to ``set(permutations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - Duplicate permutations arise when there are duplicated elements in the - input iterable. The number of items returned is - `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of - items input, and each `x_i` is the count of a distinct item in the input - sequence. - - If *r* is given, only the *r*-length permutations are yielded. - - >>> sorted(distinct_permutations([1, 0, 1], r=2)) - [(0, 1), (1, 0), (1, 1)] - >>> sorted(distinct_permutations(range(3), r=2)) - [(0, 1), (0, 2), (1, 0), (1, 2), (2, 0), (2, 1)] - - """ - # Algorithm: https://w.wiki/Qai - def _full(A): - while True: - # Yield the permutation we have - yield tuple(A) - - # Find the largest index i such that A[i] < A[i + 1] - for i in range(size - 2, -1, -1): - if A[i] < A[i + 1]: - break - # If no such index exists, this permutation is the last one - else: - return - - # Find the largest index j greater than j such that A[i] < A[j] - for j in range(size - 1, i, -1): - if A[i] < A[j]: - break - - # Swap the value of A[i] with that of A[j], then reverse the - # sequence from A[i + 1] to form the new permutation - A[i], A[j] = A[j], A[i] - A[i + 1 :] = A[: i - size : -1] # A[i + 1:][::-1] - - # Algorithm: modified from the above - def _partial(A, r): - # Split A into the first r items and the last r items - head, tail = A[:r], A[r:] - right_head_indexes = range(r - 1, -1, -1) - left_tail_indexes = range(len(tail)) - - while True: - # Yield the permutation we have - yield tuple(head) - - # Starting from the right, find the first index of the head with - # value smaller than the maximum value of the tail - call it i. - pivot = tail[-1] - for i in right_head_indexes: - if head[i] < pivot: - break - pivot = head[i] - else: - return - - # Starting from the left, find the first value of the tail - # with a value greater than head[i] and swap. - for j in left_tail_indexes: - if tail[j] > head[i]: - head[i], tail[j] = tail[j], head[i] - break - # If we didn't find one, start from the right and find the first - # index of the head with a value greater than head[i] and swap. - else: - for j in right_head_indexes: - if head[j] > head[i]: - head[i], head[j] = head[j], head[i] - break - - # Reverse head[i + 1:] and swap it with tail[:r - (i + 1)] - tail += head[: i - r : -1] # head[i + 1:][::-1] - i += 1 - head[i:], tail[:] = tail[: r - i], tail[r - i :] - - items = sorted(iterable) - - size = len(items) - if r is None: - r = size - - if 0 < r <= size: - return _full(items) if (r == size) else _partial(items, r) - - return iter(() if r else ((),)) - - -def intersperse(e, iterable, n=1): - """Intersperse filler element *e* among the items in *iterable*, leaving - *n* items between each filler element. - - >>> list(intersperse('!', [1, 2, 3, 4, 5])) - [1, '!', 2, '!', 3, '!', 4, '!', 5] - - >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) - [1, 2, None, 3, 4, None, 5] - - """ - if n == 0: - raise ValueError('n must be > 0') - elif n == 1: - # interleave(repeat(e), iterable) -> e, x_0, e, e, x_1, e, x_2... - # islice(..., 1, None) -> x_0, e, e, x_1, e, x_2... - return islice(interleave(repeat(e), iterable), 1, None) - else: - # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... - # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... - # flatten(...) -> x_0, x_1, e, x_2, x_3... - filler = repeat([e]) - chunks = chunked(iterable, n) - return flatten(islice(interleave(filler, chunks), 1, None)) - - -def unique_to_each(*iterables): - """Return the elements from each of the input iterables that aren't in the - other input iterables. - - For example, suppose you have a set of packages, each with a set of - dependencies:: - - {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} - - If you remove one package, which dependencies can also be removed? - - If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not - associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for - ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: - - >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) - [['A'], ['C'], ['D']] - - If there are duplicates in one input iterable that aren't in the others - they will be duplicated in the output. Input order is preserved:: - - >>> unique_to_each("mississippi", "missouri") - [['p', 'p'], ['o', 'u', 'r']] - - It is assumed that the elements of each iterable are hashable. - - """ - pool = [list(it) for it in iterables] - counts = Counter(chain.from_iterable(map(set, pool))) - uniques = {element for element in counts if counts[element] == 1} - return [list(filter(uniques.__contains__, it)) for it in pool] - - -def windowed(seq, n, fillvalue=None, step=1): - """Return a sliding window of width *n* over the given iterable. - - >>> all_windows = windowed([1, 2, 3, 4, 5], 3) - >>> list(all_windows) - [(1, 2, 3), (2, 3, 4), (3, 4, 5)] - - When the window is larger than the iterable, *fillvalue* is used in place - of missing values: - - >>> list(windowed([1, 2, 3], 4)) - [(1, 2, 3, None)] - - Each window will advance in increments of *step*: - - >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) - [(1, 2, 3), (3, 4, 5), (5, 6, '!')] - - To slide into the iterable's items, use :func:`chain` to add filler items - to the left: - - >>> iterable = [1, 2, 3, 4] - >>> n = 3 - >>> padding = [None] * (n - 1) - >>> list(windowed(chain(padding, iterable), 3)) - [(None, None, 1), (None, 1, 2), (1, 2, 3), (2, 3, 4)] - """ - if n < 0: - raise ValueError('n must be >= 0') - if n == 0: - yield tuple() - return - if step < 1: - raise ValueError('step must be >= 1') - - window = deque(maxlen=n) - i = n - for _ in map(window.append, seq): - i -= 1 - if not i: - i = step - yield tuple(window) - - size = len(window) - if size < n: - yield tuple(chain(window, repeat(fillvalue, n - size))) - elif 0 < i < min(step, n): - window += (fillvalue,) * i - yield tuple(window) - - -def substrings(iterable): - """Yield all of the substrings of *iterable*. - - >>> [''.join(s) for s in substrings('more')] - ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] - - Note that non-string iterables can also be subdivided. - - >>> list(substrings([0, 1, 2])) - [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] - - """ - # The length-1 substrings - seq = [] - for item in iter(iterable): - seq.append(item) - yield (item,) - seq = tuple(seq) - item_count = len(seq) - - # And the rest - for n in range(2, item_count + 1): - for i in range(item_count - n + 1): - yield seq[i : i + n] - - -def substrings_indexes(seq, reverse=False): - """Yield all substrings and their positions in *seq* - - The items yielded will be a tuple of the form ``(substr, i, j)``, where - ``substr == seq[i:j]``. - - This function only works for iterables that support slicing, such as - ``str`` objects. - - >>> for item in substrings_indexes('more'): - ... print(item) - ('m', 0, 1) - ('o', 1, 2) - ('r', 2, 3) - ('e', 3, 4) - ('mo', 0, 2) - ('or', 1, 3) - ('re', 2, 4) - ('mor', 0, 3) - ('ore', 1, 4) - ('more', 0, 4) - - Set *reverse* to ``True`` to yield the same items in the opposite order. - - - """ - r = range(1, len(seq) + 1) - if reverse: - r = reversed(r) - return ( - (seq[i : i + L], i, i + L) for L in r for i in range(len(seq) - L + 1) - ) - - -class bucket: - """Wrap *iterable* and return an object that buckets it iterable into - child iterables based on a *key* function. - - >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] - >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character - >>> sorted(list(s)) # Get the keys - ['a', 'b', 'c'] - >>> a_iterable = s['a'] - >>> next(a_iterable) - 'a1' - >>> next(a_iterable) - 'a2' - >>> list(s['b']) - ['b1', 'b2', 'b3'] - - The original iterable will be advanced and its items will be cached until - they are used by the child iterables. This may require significant storage. - - By default, attempting to select a bucket to which no items belong will - exhaust the iterable and cache all values. - If you specify a *validator* function, selected buckets will instead be - checked against it. - - >>> from itertools import count - >>> it = count(1, 2) # Infinite sequence of odd numbers - >>> key = lambda x: x % 10 # Bucket by last digit - >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only - >>> s = bucket(it, key=key, validator=validator) - >>> 2 in s - False - >>> list(s[2]) - [] - - """ - - def __init__(self, iterable, key, validator=None): - self._it = iter(iterable) - self._key = key - self._cache = defaultdict(deque) - self._validator = validator or (lambda x: True) - - def __contains__(self, value): - if not self._validator(value): - return False - - try: - item = next(self[value]) - except StopIteration: - return False - else: - self._cache[value].appendleft(item) - - return True - - def _get_values(self, value): - """ - Helper to yield items from the parent iterator that match *value*. - Items that don't match are stored in the local cache as they - are encountered. - """ - while True: - # If we've cached some items that match the target value, emit - # the first one and evict it from the cache. - if self._cache[value]: - yield self._cache[value].popleft() - # Otherwise we need to advance the parent iterator to search for - # a matching item, caching the rest. - else: - while True: - try: - item = next(self._it) - except StopIteration: - return - item_value = self._key(item) - if item_value == value: - yield item - break - elif self._validator(item_value): - self._cache[item_value].append(item) - - def __iter__(self): - for item in self._it: - item_value = self._key(item) - if self._validator(item_value): - self._cache[item_value].append(item) - - yield from self._cache.keys() - - def __getitem__(self, value): - if not self._validator(value): - return iter(()) - - return self._get_values(value) - - -def spy(iterable, n=1): - """Return a 2-tuple with a list containing the first *n* elements of - *iterable*, and an iterator with the same items as *iterable*. - This allows you to "look ahead" at the items in the iterable without - advancing it. - - There is one item in the list by default: - - >>> iterable = 'abcdefg' - >>> head, iterable = spy(iterable) - >>> head - ['a'] - >>> list(iterable) - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - - You may use unpacking to retrieve items instead of lists: - - >>> (head,), iterable = spy('abcdefg') - >>> head - 'a' - >>> (first, second), iterable = spy('abcdefg', 2) - >>> first - 'a' - >>> second - 'b' - - The number of items requested can be larger than the number of items in - the iterable: - - >>> iterable = [1, 2, 3, 4, 5] - >>> head, iterable = spy(iterable, 10) - >>> head - [1, 2, 3, 4, 5] - >>> list(iterable) - [1, 2, 3, 4, 5] - - """ - it = iter(iterable) - head = take(n, it) - - return head.copy(), chain(head, it) - - -def interleave(*iterables): - """Return a new iterable yielding from each iterable in turn, - until the shortest is exhausted. - - >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7] - - For a version that doesn't terminate after the shortest iterable is - exhausted, see :func:`interleave_longest`. - - """ - return chain.from_iterable(zip(*iterables)) - - -def interleave_longest(*iterables): - """Return a new iterable yielding from each iterable in turn, - skipping any that are exhausted. - - >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) - [1, 4, 6, 2, 5, 7, 3, 8] - - This function produces the same output as :func:`roundrobin`, but may - perform better for some inputs (in particular when the number of iterables - is large). - - """ - i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) - return (x for x in i if x is not _marker) - - -def collapse(iterable, base_type=None, levels=None): - """Flatten an iterable with multiple levels of nesting (e.g., a list of - lists of tuples) into non-iterable types. - - >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] - >>> list(collapse(iterable)) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and - will not be collapsed. - - To avoid collapsing other types, specify *base_type*: - - >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] - >>> list(collapse(iterable, base_type=tuple)) - ['ab', ('cd', 'ef'), 'gh', 'ij'] - - Specify *levels* to stop flattening after a certain level: - - >>> iterable = [('a', ['b']), ('c', ['d'])] - >>> list(collapse(iterable)) # Fully flattened - ['a', 'b', 'c', 'd'] - >>> list(collapse(iterable, levels=1)) # Only one level flattened - ['a', ['b'], 'c', ['d']] - - """ - - def walk(node, level): - if ( - ((levels is not None) and (level > levels)) - or isinstance(node, (str, bytes)) - or ((base_type is not None) and isinstance(node, base_type)) - ): - yield node - return - - try: - tree = iter(node) - except TypeError: - yield node - return - else: - for child in tree: - yield from walk(child, level + 1) - - yield from walk(iterable, 0) - - -def side_effect(func, iterable, chunk_size=None, before=None, after=None): - """Invoke *func* on each item in *iterable* (or on each *chunk_size* group - of items) before yielding the item. - - `func` must be a function that takes a single argument. Its return value - will be discarded. - - *before* and *after* are optional functions that take no arguments. They - will be executed before iteration starts and after it ends, respectively. - - `side_effect` can be used for logging, updating progress bars, or anything - that is not functionally "pure." - - Emitting a status message: - - >>> from more_itertools import consume - >>> func = lambda item: print('Received {}'.format(item)) - >>> consume(side_effect(func, range(2))) - Received 0 - Received 1 - - Operating on chunks of items: - - >>> pair_sums = [] - >>> func = lambda chunk: pair_sums.append(sum(chunk)) - >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) - [0, 1, 2, 3, 4, 5] - >>> list(pair_sums) - [1, 5, 9] - - Writing to a file-like object: - - >>> from io import StringIO - >>> from more_itertools import consume - >>> f = StringIO() - >>> func = lambda x: print(x, file=f) - >>> before = lambda: print(u'HEADER', file=f) - >>> after = f.close - >>> it = [u'a', u'b', u'c'] - >>> consume(side_effect(func, it, before=before, after=after)) - >>> f.closed - True - - """ - try: - if before is not None: - before() - - if chunk_size is None: - for item in iterable: - func(item) - yield item - else: - for chunk in chunked(iterable, chunk_size): - func(chunk) - yield from chunk - finally: - if after is not None: - after() - - -def sliced(seq, n, strict=False): - """Yield slices of length *n* from the sequence *seq*. - - >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) - [(1, 2, 3), (4, 5, 6)] - - By the default, the last yielded slice will have fewer than *n* elements - if the length of *seq* is not divisible by *n*: - - >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) - [(1, 2, 3), (4, 5, 6), (7, 8)] - - If the length of *seq* is not divisible by *n* and *strict* is - ``True``, then ``ValueError`` will be raised before the last - slice is yielded. - - This function will only work for iterables that support slicing. - For non-sliceable iterables, see :func:`chunked`. - - """ - iterator = takewhile(len, (seq[i : i + n] for i in count(0, n))) - if strict: - - def ret(): - for _slice in iterator: - if len(_slice) != n: - raise ValueError("seq is not divisible by n.") - yield _slice - - return iter(ret()) - else: - return iterator - - -def split_at(iterable, pred, maxsplit=-1, keep_separator=False): - """Yield lists of items from *iterable*, where each list is delimited by - an item where callable *pred* returns ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b')) - [['a'], ['c', 'd', 'c'], ['a']] - - >>> list(split_at(range(10), lambda n: n % 2 == 1)) - [[0], [2], [4], [6], [8], []] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_at(range(10), lambda n: n % 2 == 1, maxsplit=2)) - [[0], [2], [4, 5, 6, 7, 8, 9]] - - By default, the delimiting items are not included in the output. - The include them, set *keep_separator* to ``True``. - - >>> list(split_at('abcdcba', lambda x: x == 'b', keep_separator=True)) - [['a'], ['b'], ['c', 'd', 'c'], ['b'], ['a']] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item): - yield buf - if keep_separator: - yield [item] - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - else: - buf.append(item) - yield buf - - -def split_before(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends just before - an item for which callable *pred* returns ``True``: - - >>> list(split_before('OneTwo', lambda s: s.isupper())) - [['O', 'n', 'e'], ['T', 'w', 'o']] - - >>> list(split_before(range(10), lambda n: n % 3 == 0)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_before(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]] - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield [item] + list(it) - return - buf = [] - maxsplit -= 1 - buf.append(item) - if buf: - yield buf - - -def split_after(iterable, pred, maxsplit=-1): - """Yield lists of items from *iterable*, where each list ends with an - item where callable *pred* returns ``True``: - - >>> list(split_after('one1two2', lambda s: s.isdigit())) - [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] - - >>> list(split_after(range(10), lambda n: n % 3 == 0)) - [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_after(range(10), lambda n: n % 3 == 0, maxsplit=2)) - [[0], [1, 2, 3], [4, 5, 6, 7, 8, 9]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - buf = [] - it = iter(iterable) - for item in it: - buf.append(item) - if pred(item) and buf: - yield buf - if maxsplit == 1: - yield list(it) - return - buf = [] - maxsplit -= 1 - if buf: - yield buf - - -def split_when(iterable, pred, maxsplit=-1): - """Split *iterable* into pieces based on the output of *pred*. - *pred* should be a function that takes successive pairs of items and - returns ``True`` if the iterable should be split in between them. - - For example, to find runs of increasing numbers, split the iterable when - element ``i`` is larger than element ``i + 1``: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], lambda x, y: x > y)) - [[1, 2, 3, 3], [2, 5], [2, 4], [2]] - - At most *maxsplit* splits are done. If *maxsplit* is not specified or -1, - then there is no limit on the number of splits: - - >>> list(split_when([1, 2, 3, 3, 2, 5, 2, 4, 2], - ... lambda x, y: x > y, maxsplit=2)) - [[1, 2, 3, 3], [2, 5], [2, 4, 2]] - - """ - if maxsplit == 0: - yield list(iterable) - return - - it = iter(iterable) - try: - cur_item = next(it) - except StopIteration: - return - - buf = [cur_item] - for next_item in it: - if pred(cur_item, next_item): - yield buf - if maxsplit == 1: - yield [next_item] + list(it) - return - buf = [] - maxsplit -= 1 - - buf.append(next_item) - cur_item = next_item - - yield buf - - -def split_into(iterable, sizes): - """Yield a list of sequential items from *iterable* of length 'n' for each - integer 'n' in *sizes*. - - >>> list(split_into([1,2,3,4,5,6], [1,2,3])) - [[1], [2, 3], [4, 5, 6]] - - If the sum of *sizes* is smaller than the length of *iterable*, then the - remaining items of *iterable* will not be returned. - - >>> list(split_into([1,2,3,4,5,6], [2,3])) - [[1, 2], [3, 4, 5]] - - If the sum of *sizes* is larger than the length of *iterable*, fewer items - will be returned in the iteration that overruns *iterable* and further - lists will be empty: - - >>> list(split_into([1,2,3,4], [1,2,3,4])) - [[1], [2, 3], [4], []] - - When a ``None`` object is encountered in *sizes*, the returned list will - contain items up to the end of *iterable* the same way that itertools.slice - does: - - >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) - [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] - - :func:`split_into` can be useful for grouping a series of items where the - sizes of the groups are not uniform. An example would be where in a row - from a table, multiple columns represent elements of the same feature - (e.g. a point represented by x,y,z) but, the format is not the same for - all columns. - """ - # convert the iterable argument into an iterator so its contents can - # be consumed by islice in case it is a generator - it = iter(iterable) - - for size in sizes: - if size is None: - yield list(it) - return - else: - yield list(islice(it, size)) - - -def padded(iterable, fillvalue=None, n=None, next_multiple=False): - """Yield the elements from *iterable*, followed by *fillvalue*, such that - at least *n* items are emitted. - - >>> list(padded([1, 2, 3], '?', 5)) - [1, 2, 3, '?', '?'] - - If *next_multiple* is ``True``, *fillvalue* will be emitted until the - number of items emitted is a multiple of *n*:: - - >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) - [1, 2, 3, 4, None, None] - - If *n* is ``None``, *fillvalue* will be emitted indefinitely. - - """ - it = iter(iterable) - if n is None: - yield from chain(it, repeat(fillvalue)) - elif n < 1: - raise ValueError('n must be at least 1') - else: - item_count = 0 - for item in it: - yield item - item_count += 1 - - remaining = (n - item_count) % n if next_multiple else n - item_count - for _ in range(remaining): - yield fillvalue - - -def repeat_last(iterable, default=None): - """After the *iterable* is exhausted, keep yielding its last element. - - >>> list(islice(repeat_last(range(3)), 5)) - [0, 1, 2, 2, 2] - - If the iterable is empty, yield *default* forever:: - - >>> list(islice(repeat_last(range(0), 42), 5)) - [42, 42, 42, 42, 42] - - """ - item = _marker - for item in iterable: - yield item - final = default if item is _marker else item - yield from repeat(final) - - -def distribute(n, iterable): - """Distribute the items from *iterable* among *n* smaller iterables. - - >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 3, 5] - >>> list(group_2) - [2, 4, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 4, 7], [2, 5], [3, 6]] - - If the length of *iterable* is smaller than *n*, then the last returned - iterables will be empty: - - >>> children = distribute(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function uses :func:`itertools.tee` and may require significant - storage. If you need the order items in the smaller iterables to match the - original iterable, see :func:`divide`. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - children = tee(iterable, n) - return [islice(it, index, None, n) for index, it in enumerate(children)] - - -def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): - """Yield tuples whose elements are offset from *iterable*. - The amount by which the `i`-th item in each tuple is offset is given by - the `i`-th item in *offsets*. - - >>> list(stagger([0, 1, 2, 3])) - [(None, 0, 1), (0, 1, 2), (1, 2, 3)] - >>> list(stagger(range(8), offsets=(0, 2, 4))) - [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] - - By default, the sequence will end when the final element of a tuple is the - last item in the iterable. To continue until the first element of a tuple - is the last item in the iterable, set *longest* to ``True``:: - - >>> list(stagger([0, 1, 2, 3], longest=True)) - [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - children = tee(iterable, len(offsets)) - - return zip_offset( - *children, offsets=offsets, longest=longest, fillvalue=fillvalue - ) - - -class UnequalIterablesError(ValueError): - def __init__(self, details=None): - msg = 'Iterables have different lengths' - if details is not None: - msg += (': index 0 has length {}; index {} has length {}').format( - *details - ) - - super().__init__(msg) - - -def _zip_equal_generator(iterables): - for combo in zip_longest(*iterables, fillvalue=_marker): - for val in combo: - if val is _marker: - raise UnequalIterablesError() - yield combo - - -def zip_equal(*iterables): - """``zip`` the input *iterables* together, but raise - ``UnequalIterablesError`` if they aren't all the same length. - - >>> it_1 = range(3) - >>> it_2 = iter('abc') - >>> list(zip_equal(it_1, it_2)) - [(0, 'a'), (1, 'b'), (2, 'c')] - - >>> it_1 = range(3) - >>> it_2 = iter('abcd') - >>> list(zip_equal(it_1, it_2)) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - more_itertools.more.UnequalIterablesError: Iterables have different - lengths - - """ - if hexversion >= 0x30A00A6: - warnings.warn( - ( - 'zip_equal will be removed in a future version of ' - 'more-itertools. Use the builtin zip function with ' - 'strict=True instead.' - ), - DeprecationWarning, - ) - # Check whether the iterables are all the same size. - try: - first_size = len(iterables[0]) - for i, it in enumerate(iterables[1:], 1): - size = len(it) - if size != first_size: - break - else: - # If we didn't break out, we can use the built-in zip. - return zip(*iterables) - - # If we did break out, there was a mismatch. - raise UnequalIterablesError(details=(first_size, i, size)) - # If any one of the iterables didn't have a length, start reading - # them until one runs out. - except TypeError: - return _zip_equal_generator(iterables) - - -def zip_offset(*iterables, offsets, longest=False, fillvalue=None): - """``zip`` the input *iterables* together, but offset the `i`-th iterable - by the `i`-th item in *offsets*. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] - - This can be used as a lightweight alternative to SciPy or pandas to analyze - data sets in which some series have a lead or lag relationship. - - By default, the sequence will end when the shortest iterable is exhausted. - To continue until the longest iterable is exhausted, set *longest* to - ``True``. - - >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) - [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] - - By default, ``None`` will be used to replace offsets beyond the end of the - sequence. Specify *fillvalue* to use some other value. - - """ - if len(iterables) != len(offsets): - raise ValueError("Number of iterables and offsets didn't match") - - staggered = [] - for it, n in zip(iterables, offsets): - if n < 0: - staggered.append(chain(repeat(fillvalue, -n), it)) - elif n > 0: - staggered.append(islice(it, n, None)) - else: - staggered.append(it) - - if longest: - return zip_longest(*staggered, fillvalue=fillvalue) - - return zip(*staggered) - - -def sort_together(iterables, key_list=(0,), key=None, reverse=False): - """Return the input iterables sorted together, with *key_list* as the - priority for sorting. All iterables are trimmed to the length of the - shortest one. - - This can be used like the sorting function in a spreadsheet. If each - iterable represents a column of data, the key list determines which - columns are used for sorting. - - By default, all iterables are sorted using the ``0``-th iterable:: - - >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] - >>> sort_together(iterables) - [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] - - Set a different key list to sort according to another iterable. - Specifying multiple keys dictates how ties are broken:: - - >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] - >>> sort_together(iterables, key_list=(1, 2)) - [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] - - To sort by a function of the elements of the iterable, pass a *key* - function. Its arguments are the elements of the iterables corresponding to - the key list:: - - >>> names = ('a', 'b', 'c') - >>> lengths = (1, 2, 3) - >>> widths = (5, 2, 1) - >>> def area(length, width): - ... return length * width - >>> sort_together([names, lengths, widths], key_list=(1, 2), key=area) - [('c', 'b', 'a'), (3, 2, 1), (1, 2, 5)] - - Set *reverse* to ``True`` to sort in descending order. - - >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) - [(3, 2, 1), ('a', 'b', 'c')] - - """ - if key is None: - # if there is no key function, the key argument to sorted is an - # itemgetter - key_argument = itemgetter(*key_list) - else: - # if there is a key function, call it with the items at the offsets - # specified by the key function as arguments - key_list = list(key_list) - if len(key_list) == 1: - # if key_list contains a single item, pass the item at that offset - # as the only argument to the key function - key_offset = key_list[0] - key_argument = lambda zipped_items: key(zipped_items[key_offset]) - else: - # if key_list contains multiple items, use itemgetter to return a - # tuple of items, which we pass as *args to the key function - get_key_items = itemgetter(*key_list) - key_argument = lambda zipped_items: key( - *get_key_items(zipped_items) - ) - - return list( - zip(*sorted(zip(*iterables), key=key_argument, reverse=reverse)) - ) - - -def unzip(iterable): - """The inverse of :func:`zip`, this function disaggregates the elements - of the zipped *iterable*. - - The ``i``-th iterable contains the ``i``-th element from each element - of the zipped iterable. The first element is used to to determine the - length of the remaining elements. - - >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> letters, numbers = unzip(iterable) - >>> list(letters) - ['a', 'b', 'c', 'd'] - >>> list(numbers) - [1, 2, 3, 4] - - This is similar to using ``zip(*iterable)``, but it avoids reading - *iterable* into memory. Note, however, that this function uses - :func:`itertools.tee` and thus may require significant storage. - - """ - head, iterable = spy(iter(iterable)) - if not head: - # empty iterable, e.g. zip([], [], []) - return () - # spy returns a one-length iterable as head - head = head[0] - iterables = tee(iterable, len(head)) - - def itemgetter(i): - def getter(obj): - try: - return obj[i] - except IndexError: - # basically if we have an iterable like - # iter([(1, 2, 3), (4, 5), (6,)]) - # the second unzipped iterable would fail at the third tuple - # since it would try to access tup[1] - # same with the third unzipped iterable and the second tuple - # to support these "improperly zipped" iterables, - # we create a custom itemgetter - # which just stops the unzipped iterables - # at first length mismatch - raise StopIteration - - return getter - - return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) - - -def divide(n, iterable): - """Divide the elements from *iterable* into *n* parts, maintaining - order. - - >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) - >>> list(group_1) - [1, 2, 3] - >>> list(group_2) - [4, 5, 6] - - If the length of *iterable* is not evenly divisible by *n*, then the - length of the returned iterables will not be identical: - - >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) - >>> [list(c) for c in children] - [[1, 2, 3], [4, 5], [6, 7]] - - If the length of the iterable is smaller than n, then the last returned - iterables will be empty: - - >>> children = divide(5, [1, 2, 3]) - >>> [list(c) for c in children] - [[1], [2], [3], [], []] - - This function will exhaust the iterable before returning and may require - significant storage. If order is not important, see :func:`distribute`, - which does not first pull the iterable into memory. - - """ - if n < 1: - raise ValueError('n must be at least 1') - - try: - iterable[:0] - except TypeError: - seq = tuple(iterable) - else: - seq = iterable - - q, r = divmod(len(seq), n) - - ret = [] - stop = 0 - for i in range(1, n + 1): - start = stop - stop += q + 1 if i <= r else q - ret.append(iter(seq[start:stop])) - - return ret - - -def always_iterable(obj, base_type=(str, bytes)): - """If *obj* is iterable, return an iterator over its items:: - - >>> obj = (1, 2, 3) - >>> list(always_iterable(obj)) - [1, 2, 3] - - If *obj* is not iterable, return a one-item iterable containing *obj*:: - - >>> obj = 1 - >>> list(always_iterable(obj)) - [1] - - If *obj* is ``None``, return an empty iterable: - - >>> obj = None - >>> list(always_iterable(None)) - [] - - By default, binary and text strings are not considered iterable:: - - >>> obj = 'foo' - >>> list(always_iterable(obj)) - ['foo'] - - If *base_type* is set, objects for which ``isinstance(obj, base_type)`` - returns ``True`` won't be considered iterable. - - >>> obj = {'a': 1} - >>> list(always_iterable(obj)) # Iterate over the dict's keys - ['a'] - >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit - [{'a': 1}] - - Set *base_type* to ``None`` to avoid any special handling and treat objects - Python considers iterable as iterable: - - >>> obj = 'foo' - >>> list(always_iterable(obj, base_type=None)) - ['f', 'o', 'o'] - """ - if obj is None: - return iter(()) - - if (base_type is not None) and isinstance(obj, base_type): - return iter((obj,)) - - try: - return iter(obj) - except TypeError: - return iter((obj,)) - - -def adjacent(predicate, iterable, distance=1): - """Return an iterable over `(bool, item)` tuples where the `item` is - drawn from *iterable* and the `bool` indicates whether - that item satisfies the *predicate* or is adjacent to an item that does. - - For example, to find whether items are adjacent to a ``3``:: - - >>> list(adjacent(lambda x: x == 3, range(6))) - [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] - - Set *distance* to change what counts as adjacent. For example, to find - whether items are two places away from a ``3``: - - >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) - [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] - - This is useful for contextualizing the results of a search function. - For example, a code comparison tool might want to identify lines that - have changed, but also surrounding lines to give the viewer of the diff - context. - - The predicate function will only be called once for each item in the - iterable. - - See also :func:`groupby_transform`, which can be used with this function - to group ranges of items with the same `bool` value. - - """ - # Allow distance=0 mainly for testing that it reproduces results with map() - if distance < 0: - raise ValueError('distance must be at least 0') - - i1, i2 = tee(iterable) - padding = [False] * distance - selected = chain(padding, map(predicate, i1), padding) - adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) - return zip(adjacent_to_selected, i2) - - -def groupby_transform(iterable, keyfunc=None, valuefunc=None, reducefunc=None): - """An extension of :func:`itertools.groupby` that can apply transformations - to the grouped data. - - * *keyfunc* is a function computing a key value for each item in *iterable* - * *valuefunc* is a function that transforms the individual items from - *iterable* after grouping - * *reducefunc* is a function that transforms each group of items - - >>> iterable = 'aAAbBBcCC' - >>> keyfunc = lambda k: k.upper() - >>> valuefunc = lambda v: v.lower() - >>> reducefunc = lambda g: ''.join(g) - >>> list(groupby_transform(iterable, keyfunc, valuefunc, reducefunc)) - [('A', 'aaa'), ('B', 'bbb'), ('C', 'ccc')] - - Each optional argument defaults to an identity function if not specified. - - :func:`groupby_transform` is useful when grouping elements of an iterable - using a separate iterable as the key. To do this, :func:`zip` the iterables - and pass a *keyfunc* that extracts the first element and a *valuefunc* - that extracts the second element:: - - >>> from operator import itemgetter - >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] - >>> values = 'abcdefghi' - >>> iterable = zip(keys, values) - >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) - >>> [(k, ''.join(g)) for k, g in grouper] - [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] - - Note that the order of items in the iterable is significant. - Only adjacent items are grouped together, so if you don't want any - duplicate groups, you should sort the iterable by the key function. - - """ - ret = groupby(iterable, keyfunc) - if valuefunc: - ret = ((k, map(valuefunc, g)) for k, g in ret) - if reducefunc: - ret = ((k, reducefunc(g)) for k, g in ret) - - return ret - - -class numeric_range(abc.Sequence, abc.Hashable): - """An extension of the built-in ``range()`` function whose arguments can - be any orderable numeric type. - - With only *stop* specified, *start* defaults to ``0`` and *step* - defaults to ``1``. The output items will match the type of *stop*: - - >>> list(numeric_range(3.5)) - [0.0, 1.0, 2.0, 3.0] - - With only *start* and *stop* specified, *step* defaults to ``1``. The - output items will match the type of *start*: - - >>> from decimal import Decimal - >>> start = Decimal('2.1') - >>> stop = Decimal('5.1') - >>> list(numeric_range(start, stop)) - [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] - - With *start*, *stop*, and *step* specified the output items will match - the type of ``start + step``: - - >>> from fractions import Fraction - >>> start = Fraction(1, 2) # Start at 1/2 - >>> stop = Fraction(5, 2) # End at 5/2 - >>> step = Fraction(1, 2) # Count by 1/2 - >>> list(numeric_range(start, stop, step)) - [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] - - If *step* is zero, ``ValueError`` is raised. Negative steps are supported: - - >>> list(numeric_range(3, -1, -1.0)) - [3.0, 2.0, 1.0, 0.0] - - Be aware of the limitations of floating point numbers; the representation - of the yielded numbers may be surprising. - - ``datetime.datetime`` objects can be used for *start* and *stop*, if *step* - is a ``datetime.timedelta`` object: - - >>> import datetime - >>> start = datetime.datetime(2019, 1, 1) - >>> stop = datetime.datetime(2019, 1, 3) - >>> step = datetime.timedelta(days=1) - >>> items = iter(numeric_range(start, stop, step)) - >>> next(items) - datetime.datetime(2019, 1, 1, 0, 0) - >>> next(items) - datetime.datetime(2019, 1, 2, 0, 0) - - """ - - _EMPTY_HASH = hash(range(0, 0)) - - def __init__(self, *args): - argc = len(args) - if argc == 1: - (self._stop,) = args - self._start = type(self._stop)(0) - self._step = type(self._stop - self._start)(1) - elif argc == 2: - self._start, self._stop = args - self._step = type(self._stop - self._start)(1) - elif argc == 3: - self._start, self._stop, self._step = args - elif argc == 0: - raise TypeError( - 'numeric_range expected at least ' - '1 argument, got {}'.format(argc) - ) - else: - raise TypeError( - 'numeric_range expected at most ' - '3 arguments, got {}'.format(argc) - ) - - self._zero = type(self._step)(0) - if self._step == self._zero: - raise ValueError('numeric_range() arg 3 must not be zero') - self._growing = self._step > self._zero - self._init_len() - - def __bool__(self): - if self._growing: - return self._start < self._stop - else: - return self._start > self._stop - - def __contains__(self, elem): - if self._growing: - if self._start <= elem < self._stop: - return (elem - self._start) % self._step == self._zero - else: - if self._start >= elem > self._stop: - return (self._start - elem) % (-self._step) == self._zero - - return False - - def __eq__(self, other): - if isinstance(other, numeric_range): - empty_self = not bool(self) - empty_other = not bool(other) - if empty_self or empty_other: - return empty_self and empty_other # True if both empty - else: - return ( - self._start == other._start - and self._step == other._step - and self._get_by_index(-1) == other._get_by_index(-1) - ) - else: - return False - - def __getitem__(self, key): - if isinstance(key, int): - return self._get_by_index(key) - elif isinstance(key, slice): - step = self._step if key.step is None else key.step * self._step - - if key.start is None or key.start <= -self._len: - start = self._start - elif key.start >= self._len: - start = self._stop - else: # -self._len < key.start < self._len - start = self._get_by_index(key.start) - - if key.stop is None or key.stop >= self._len: - stop = self._stop - elif key.stop <= -self._len: - stop = self._start - else: # -self._len < key.stop < self._len - stop = self._get_by_index(key.stop) - - return numeric_range(start, stop, step) - else: - raise TypeError( - 'numeric range indices must be ' - 'integers or slices, not {}'.format(type(key).__name__) - ) - - def __hash__(self): - if self: - return hash((self._start, self._get_by_index(-1), self._step)) - else: - return self._EMPTY_HASH - - def __iter__(self): - values = (self._start + (n * self._step) for n in count()) - if self._growing: - return takewhile(partial(gt, self._stop), values) - else: - return takewhile(partial(lt, self._stop), values) - - def __len__(self): - return self._len - - def _init_len(self): - if self._growing: - start = self._start - stop = self._stop - step = self._step - else: - start = self._stop - stop = self._start - step = -self._step - distance = stop - start - if distance <= self._zero: - self._len = 0 - else: # distance > 0 and step > 0: regular euclidean division - q, r = divmod(distance, step) - self._len = int(q) + int(r != self._zero) - - def __reduce__(self): - return numeric_range, (self._start, self._stop, self._step) - - def __repr__(self): - if self._step == 1: - return "numeric_range({}, {})".format( - repr(self._start), repr(self._stop) - ) - else: - return "numeric_range({}, {}, {})".format( - repr(self._start), repr(self._stop), repr(self._step) - ) - - def __reversed__(self): - return iter( - numeric_range( - self._get_by_index(-1), self._start - self._step, -self._step - ) - ) - - def count(self, value): - return int(value in self) - - def index(self, value): - if self._growing: - if self._start <= value < self._stop: - q, r = divmod(value - self._start, self._step) - if r == self._zero: - return int(q) - else: - if self._start >= value > self._stop: - q, r = divmod(self._start - value, -self._step) - if r == self._zero: - return int(q) - - raise ValueError("{} is not in numeric range".format(value)) - - def _get_by_index(self, i): - if i < 0: - i += self._len - if i < 0 or i >= self._len: - raise IndexError("numeric range object index out of range") - return self._start + i * self._step - - -def count_cycle(iterable, n=None): - """Cycle through the items from *iterable* up to *n* times, yielding - the number of completed cycles along with each item. If *n* is omitted the - process repeats indefinitely. - - >>> list(count_cycle('AB', 3)) - [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] - - """ - iterable = tuple(iterable) - if not iterable: - return iter(()) - counter = count() if n is None else range(n) - return ((i, item) for i in counter for item in iterable) - - -def mark_ends(iterable): - """Yield 3-tuples of the form ``(is_first, is_last, item)``. - - >>> list(mark_ends('ABC')) - [(True, False, 'A'), (False, False, 'B'), (False, True, 'C')] - - Use this when looping over an iterable to take special action on its first - and/or last items: - - >>> iterable = ['Header', 100, 200, 'Footer'] - >>> total = 0 - >>> for is_first, is_last, item in mark_ends(iterable): - ... if is_first: - ... continue # Skip the header - ... if is_last: - ... continue # Skip the footer - ... total += item - >>> print(total) - 300 - """ - it = iter(iterable) - - try: - b = next(it) - except StopIteration: - return - - try: - for i in count(): - a = b - b = next(it) - yield i == 0, False, a - - except StopIteration: - yield i == 0, True, a - - -def locate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(locate([0, 1, 1, 0, 1, 0, 0])) - [1, 2, 4] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item. - - >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) - [1, 3] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(locate(iterable, pred=pred, window_size=3)) - [1, 5, 9] - - Use with :func:`seekable` to find indexes and then retrieve the associated - items: - - >>> from itertools import count - >>> from more_itertools import seekable - >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) - >>> it = seekable(source) - >>> pred = lambda x: x > 100 - >>> indexes = locate(it, pred=pred) - >>> i = next(indexes) - >>> it.seek(i) - >>> next(it) - 106 - - """ - if window_size is None: - return compress(count(), map(pred, iterable)) - - if window_size < 1: - raise ValueError('window size must be at least 1') - - it = windowed(iterable, window_size, fillvalue=_marker) - return compress(count(), starmap(pred, it)) - - -def lstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the beginning - for which *pred* returns ``True``. - - For example, to remove a set of items from the start of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(lstrip(iterable, pred)) - [1, 2, None, 3, False, None] - - This function is analogous to to :func:`str.lstrip`, and is essentially - an wrapper for :func:`itertools.dropwhile`. - - """ - return dropwhile(pred, iterable) - - -def rstrip(iterable, pred): - """Yield the items from *iterable*, but strip any from the end - for which *pred* returns ``True``. - - For example, to remove a set of items from the end of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(rstrip(iterable, pred)) - [None, False, None, 1, 2, None, 3] - - This function is analogous to :func:`str.rstrip`. - - """ - cache = [] - cache_append = cache.append - cache_clear = cache.clear - for x in iterable: - if pred(x): - cache_append(x) - else: - yield from cache - cache_clear() - yield x - - -def strip(iterable, pred): - """Yield the items from *iterable*, but strip any from the - beginning and end for which *pred* returns ``True``. - - For example, to remove a set of items from both ends of an iterable: - - >>> iterable = (None, False, None, 1, 2, None, 3, False, None) - >>> pred = lambda x: x in {None, False, ''} - >>> list(strip(iterable, pred)) - [1, 2, None, 3] - - This function is analogous to :func:`str.strip`. - - """ - return rstrip(lstrip(iterable, pred), pred) - - -class islice_extended: - """An extension of :func:`itertools.islice` that supports negative values - for *stop*, *start*, and *step*. - - >>> iterable = iter('abcdefgh') - >>> list(islice_extended(iterable, -4, -1)) - ['e', 'f', 'g'] - - Slices with negative values require some caching of *iterable*, but this - function takes care to minimize the amount of memory required. - - For example, you can use a negative step with an infinite iterator: - - >>> from itertools import count - >>> list(islice_extended(count(), 110, 99, -2)) - [110, 108, 106, 104, 102, 100] - - You can also use slice notation directly: - - >>> iterable = map(str, count()) - >>> it = islice_extended(iterable)[10:20:2] - >>> list(it) - ['10', '12', '14', '16', '18'] - - """ - - def __init__(self, iterable, *args): - it = iter(iterable) - if args: - self._iterable = _islice_helper(it, slice(*args)) - else: - self._iterable = it - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterable) - - def __getitem__(self, key): - if isinstance(key, slice): - return islice_extended(_islice_helper(self._iterable, key)) - - raise TypeError('islice_extended.__getitem__ argument must be a slice') - - -def _islice_helper(it, s): - start = s.start - stop = s.stop - if s.step == 0: - raise ValueError('step argument must be a non-zero integer or None.') - step = s.step or 1 - - if step > 0: - start = 0 if (start is None) else start - - if start < 0: - # Consume all but the last -start items - cache = deque(enumerate(it, 1), maxlen=-start) - len_iter = cache[-1][0] if cache else 0 - - # Adjust start to be positive - i = max(len_iter + start, 0) - - # Adjust stop to be positive - if stop is None: - j = len_iter - elif stop >= 0: - j = min(stop, len_iter) - else: - j = max(len_iter + stop, 0) - - # Slice the cache - n = j - i - if n <= 0: - return - - for index, item in islice(cache, 0, n, step): - yield item - elif (stop is not None) and (stop < 0): - # Advance to the start position - next(islice(it, start, start), None) - - # When stop is negative, we have to carry -stop items while - # iterating - cache = deque(islice(it, -stop), maxlen=-stop) - - for index, item in enumerate(it): - cached_item = cache.popleft() - if index % step == 0: - yield cached_item - cache.append(item) - else: - # When both start and stop are positive we have the normal case - yield from islice(it, start, stop, step) - else: - start = -1 if (start is None) else start - - if (stop is not None) and (stop < 0): - # Consume all but the last items - n = -stop - 1 - cache = deque(enumerate(it, 1), maxlen=n) - len_iter = cache[-1][0] if cache else 0 - - # If start and stop are both negative they are comparable and - # we can just slice. Otherwise we can adjust start to be negative - # and then slice. - if start < 0: - i, j = start, stop - else: - i, j = min(start - len_iter, -1), None - - for index, item in list(cache)[i:j:step]: - yield item - else: - # Advance to the stop position - if stop is not None: - m = stop + 1 - next(islice(it, m, m), None) - - # stop is positive, so if start is negative they are not comparable - # and we need the rest of the items. - if start < 0: - i = start - n = None - # stop is None and start is positive, so we just need items up to - # the start index. - elif stop is None: - i = None - n = start + 1 - # Both stop and start are positive, so they are comparable. - else: - i = None - n = start - stop - if n <= 0: - return - - cache = list(islice(it, n)) - - yield from cache[i::step] - - -def always_reversible(iterable): - """An extension of :func:`reversed` that supports all iterables, not - just those which implement the ``Reversible`` or ``Sequence`` protocols. - - >>> print(*always_reversible(x for x in range(3))) - 2 1 0 - - If the iterable is already reversible, this function returns the - result of :func:`reversed()`. If the iterable is not reversible, - this function will cache the remaining items in the iterable and - yield them in reverse order, which may require significant storage. - """ - try: - return reversed(iterable) - except TypeError: - return reversed(list(iterable)) - - -def consecutive_groups(iterable, ordering=lambda x: x): - """Yield groups of consecutive items using :func:`itertools.groupby`. - The *ordering* function determines whether two items are adjacent by - returning their position. - - By default, the ordering function is the identity function. This is - suitable for finding runs of numbers: - - >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] - >>> for group in consecutive_groups(iterable): - ... print(list(group)) - [1] - [10, 11, 12] - [20] - [30, 31, 32, 33] - [40] - - For finding runs of adjacent letters, try using the :meth:`index` method - of a string of letters: - - >>> from string import ascii_lowercase - >>> iterable = 'abcdfgilmnop' - >>> ordering = ascii_lowercase.index - >>> for group in consecutive_groups(iterable, ordering): - ... print(list(group)) - ['a', 'b', 'c', 'd'] - ['f', 'g'] - ['i'] - ['l', 'm', 'n', 'o', 'p'] - - Each group of consecutive items is an iterator that shares it source with - *iterable*. When an an output group is advanced, the previous group is - no longer available unless its elements are copied (e.g., into a ``list``). - - >>> iterable = [1, 2, 11, 12, 21, 22] - >>> saved_groups = [] - >>> for group in consecutive_groups(iterable): - ... saved_groups.append(list(group)) # Copy group elements - >>> saved_groups - [[1, 2], [11, 12], [21, 22]] - - """ - for k, g in groupby( - enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) - ): - yield map(itemgetter(1), g) - - -def difference(iterable, func=sub, *, initial=None): - """This function is the inverse of :func:`itertools.accumulate`. By default - it will compute the first difference of *iterable* using - :func:`operator.sub`: - - >>> from itertools import accumulate - >>> iterable = accumulate([0, 1, 2, 3, 4]) # produces 0, 1, 3, 6, 10 - >>> list(difference(iterable)) - [0, 1, 2, 3, 4] - - *func* defaults to :func:`operator.sub`, but other functions can be - specified. They will be applied as follows:: - - A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... - - For example, to do progressive division: - - >>> iterable = [1, 2, 6, 24, 120] - >>> func = lambda x, y: x // y - >>> list(difference(iterable, func)) - [1, 2, 3, 4, 5] - - If the *initial* keyword is set, the first element will be skipped when - computing successive differences. - - >>> it = [10, 11, 13, 16] # from accumulate([1, 2, 3], initial=10) - >>> list(difference(it, initial=10)) - [1, 2, 3] - - """ - a, b = tee(iterable) - try: - first = [next(b)] - except StopIteration: - return iter([]) - - if initial is not None: - first = [] - - return chain(first, starmap(func, zip(b, a))) - - -class SequenceView(Sequence): - """Return a read-only view of the sequence object *target*. - - :class:`SequenceView` objects are analogous to Python's built-in - "dictionary view" types. They provide a dynamic view of a sequence's items, - meaning that when the sequence updates, so does the view. - - >>> seq = ['0', '1', '2'] - >>> view = SequenceView(seq) - >>> view - SequenceView(['0', '1', '2']) - >>> seq.append('3') - >>> view - SequenceView(['0', '1', '2', '3']) - - Sequence views support indexing, slicing, and length queries. They act - like the underlying sequence, except they don't allow assignment: - - >>> view[1] - '1' - >>> view[1:-1] - ['1', '2'] - >>> len(view) - 4 - - Sequence views are useful as an alternative to copying, as they don't - require (much) extra storage. - - """ - - def __init__(self, target): - if not isinstance(target, Sequence): - raise TypeError - self._target = target - - def __getitem__(self, index): - return self._target[index] - - def __len__(self): - return len(self._target) - - def __repr__(self): - return '{}({})'.format(self.__class__.__name__, repr(self._target)) - - -class seekable: - """Wrap an iterator to allow for seeking backward and forward. This - progressively caches the items in the source iterable so they can be - re-visited. - - Call :meth:`seek` with an index to seek to that position in the source - iterable. - - To "reset" an iterator, seek to ``0``: - - >>> from itertools import count - >>> it = seekable((str(n) for n in count())) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> it.seek(0) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> next(it) - '3' - - You can also seek forward: - - >>> it = seekable((str(n) for n in range(20))) - >>> it.seek(10) - >>> next(it) - '10' - >>> it.seek(20) # Seeking past the end of the source isn't a problem - >>> list(it) - [] - >>> it.seek(0) # Resetting works even after hitting the end - >>> next(it), next(it), next(it) - ('0', '1', '2') - - Call :meth:`peek` to look ahead one item without advancing the iterator: - - >>> it = seekable('1234') - >>> it.peek() - '1' - >>> list(it) - ['1', '2', '3', '4'] - >>> it.peek(default='empty') - 'empty' - - Before the iterator is at its end, calling :func:`bool` on it will return - ``True``. After it will return ``False``: - - >>> it = seekable('5678') - >>> bool(it) - True - >>> list(it) - ['5', '6', '7', '8'] - >>> bool(it) - False - - You may view the contents of the cache with the :meth:`elements` method. - That returns a :class:`SequenceView`, a view that updates automatically: - - >>> it = seekable((str(n) for n in range(10))) - >>> next(it), next(it), next(it) - ('0', '1', '2') - >>> elements = it.elements() - >>> elements - SequenceView(['0', '1', '2']) - >>> next(it) - '3' - >>> elements - SequenceView(['0', '1', '2', '3']) - - By default, the cache grows as the source iterable progresses, so beware of - wrapping very large or infinite iterables. Supply *maxlen* to limit the - size of the cache (this of course limits how far back you can seek). - - >>> from itertools import count - >>> it = seekable((str(n) for n in count()), maxlen=2) - >>> next(it), next(it), next(it), next(it) - ('0', '1', '2', '3') - >>> list(it.elements()) - ['2', '3'] - >>> it.seek(0) - >>> next(it), next(it), next(it), next(it) - ('2', '3', '4', '5') - >>> next(it) - '6' - - """ - - def __init__(self, iterable, maxlen=None): - self._source = iter(iterable) - if maxlen is None: - self._cache = [] - else: - self._cache = deque([], maxlen) - self._index = None - - def __iter__(self): - return self - - def __next__(self): - if self._index is not None: - try: - item = self._cache[self._index] - except IndexError: - self._index = None - else: - self._index += 1 - return item - - item = next(self._source) - self._cache.append(item) - return item - - def __bool__(self): - try: - self.peek() - except StopIteration: - return False - return True - - def peek(self, default=_marker): - try: - peeked = next(self) - except StopIteration: - if default is _marker: - raise - return default - if self._index is None: - self._index = len(self._cache) - self._index -= 1 - return peeked - - def elements(self): - return SequenceView(self._cache) - - def seek(self, index): - self._index = index - remainder = index - len(self._cache) - if remainder > 0: - consume(self, remainder) - - -class run_length: - """ - :func:`run_length.encode` compresses an iterable with run-length encoding. - It yields groups of repeated items with the count of how many times they - were repeated: - - >>> uncompressed = 'abbcccdddd' - >>> list(run_length.encode(uncompressed)) - [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - - :func:`run_length.decode` decompresses an iterable that was previously - compressed with run-length encoding. It yields the items of the - decompressed iterable: - - >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] - >>> list(run_length.decode(compressed)) - ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] - - """ - - @staticmethod - def encode(iterable): - return ((k, ilen(g)) for k, g in groupby(iterable)) - - @staticmethod - def decode(iterable): - return chain.from_iterable(repeat(k, n) for k, n in iterable) - - -def exactly_n(iterable, n, predicate=bool): - """Return ``True`` if exactly ``n`` items in the iterable are ``True`` - according to the *predicate* function. - - >>> exactly_n([True, True, False], 2) - True - >>> exactly_n([True, True, False], 1) - False - >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) - True - - The iterable will be advanced until ``n + 1`` truthy items are encountered, - so avoid calling it on infinite iterables. - - """ - return len(take(n + 1, filter(predicate, iterable))) == n - - -def circular_shifts(iterable): - """Return a list of circular shifts of *iterable*. - - >>> circular_shifts(range(4)) - [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] - """ - lst = list(iterable) - return take(len(lst), windowed(cycle(lst), len(lst))) - - -def make_decorator(wrapping_func, result_index=0): - """Return a decorator version of *wrapping_func*, which is a function that - modifies an iterable. *result_index* is the position in that function's - signature where the iterable goes. - - This lets you use itertools on the "production end," i.e. at function - definition. This can augment what the function returns without changing the - function's code. - - For example, to produce a decorator version of :func:`chunked`: - - >>> from more_itertools import chunked - >>> chunker = make_decorator(chunked, result_index=0) - >>> @chunker(3) - ... def iter_range(n): - ... return iter(range(n)) - ... - >>> list(iter_range(9)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8]] - - To only allow truthy items to be returned: - - >>> truth_serum = make_decorator(filter, result_index=1) - >>> @truth_serum(bool) - ... def boolean_test(): - ... return [0, 1, '', ' ', False, True] - ... - >>> list(boolean_test()) - [1, ' ', True] - - The :func:`peekable` and :func:`seekable` wrappers make for practical - decorators: - - >>> from more_itertools import peekable - >>> peekable_function = make_decorator(peekable) - >>> @peekable_function() - ... def str_range(*args): - ... return (str(x) for x in range(*args)) - ... - >>> it = str_range(1, 20, 2) - >>> next(it), next(it), next(it) - ('1', '3', '5') - >>> it.peek() - '7' - >>> next(it) - '7' - - """ - # See https://sites.google.com/site/bbayles/index/decorator_factory for - # notes on how this works. - def decorator(*wrapping_args, **wrapping_kwargs): - def outer_wrapper(f): - def inner_wrapper(*args, **kwargs): - result = f(*args, **kwargs) - wrapping_args_ = list(wrapping_args) - wrapping_args_.insert(result_index, result) - return wrapping_func(*wrapping_args_, **wrapping_kwargs) - - return inner_wrapper - - return outer_wrapper - - return decorator - - -def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): - """Return a dictionary that maps the items in *iterable* to categories - defined by *keyfunc*, transforms them with *valuefunc*, and - then summarizes them by category with *reducefunc*. - - *valuefunc* defaults to the identity function if it is unspecified. - If *reducefunc* is unspecified, no summarization takes place: - - >>> keyfunc = lambda x: x.upper() - >>> result = map_reduce('abbccc', keyfunc) - >>> sorted(result.items()) - [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] - - Specifying *valuefunc* transforms the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> result = map_reduce('abbccc', keyfunc, valuefunc) - >>> sorted(result.items()) - [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] - - Specifying *reducefunc* summarizes the categorized items: - - >>> keyfunc = lambda x: x.upper() - >>> valuefunc = lambda x: 1 - >>> reducefunc = sum - >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) - >>> sorted(result.items()) - [('A', 1), ('B', 2), ('C', 3)] - - You may want to filter the input iterable before applying the map/reduce - procedure: - - >>> all_items = range(30) - >>> items = [x for x in all_items if 10 <= x <= 20] # Filter - >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 - >>> categories = map_reduce(items, keyfunc=keyfunc) - >>> sorted(categories.items()) - [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] - >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) - >>> sorted(summaries.items()) - [(0, 90), (1, 75)] - - Note that all items in the iterable are gathered into a list before the - summarization step, which may require significant storage. - - The returned object is a :obj:`collections.defaultdict` with the - ``default_factory`` set to ``None``, such that it behaves like a normal - dictionary. - - """ - valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc - - ret = defaultdict(list) - for item in iterable: - key = keyfunc(item) - value = valuefunc(item) - ret[key].append(value) - - if reducefunc is not None: - for key, value_list in ret.items(): - ret[key] = reducefunc(value_list) - - ret.default_factory = None - return ret - - -def rlocate(iterable, pred=bool, window_size=None): - """Yield the index of each item in *iterable* for which *pred* returns - ``True``, starting from the right and moving left. - - *pred* defaults to :func:`bool`, which will select truthy items: - - >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 - [4, 2, 1] - - Set *pred* to a custom function to, e.g., find the indexes for a particular - item: - - >>> iterable = iter('abcb') - >>> pred = lambda x: x == 'b' - >>> list(rlocate(iterable, pred)) - [3, 1] - - If *window_size* is given, then the *pred* function will be called with - that many items. This enables searching for sub-sequences: - - >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] - >>> pred = lambda *args: args == (1, 2, 3) - >>> list(rlocate(iterable, pred=pred, window_size=3)) - [9, 5, 1] - - Beware, this function won't return anything for infinite iterables. - If *iterable* is reversible, ``rlocate`` will reverse it and search from - the right. Otherwise, it will search from the left and return the results - in reverse order. - - See :func:`locate` to for other example applications. - - """ - if window_size is None: - try: - len_iter = len(iterable) - return (len_iter - i - 1 for i in locate(reversed(iterable), pred)) - except TypeError: - pass - - return reversed(list(locate(iterable, pred, window_size))) - - -def replace(iterable, pred, substitutes, count=None, window_size=1): - """Yield the items from *iterable*, replacing the items for which *pred* - returns ``True`` with the items from the iterable *substitutes*. - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] - >>> pred = lambda x: x == 0 - >>> substitutes = (2, 3) - >>> list(replace(iterable, pred, substitutes)) - [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] - - If *count* is given, the number of replacements will be limited: - - >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] - >>> pred = lambda x: x == 0 - >>> substitutes = [None] - >>> list(replace(iterable, pred, substitutes, count=2)) - [1, 1, None, 1, 1, None, 1, 1, 0] - - Use *window_size* to control the number of items passed as arguments to - *pred*. This allows for locating and replacing subsequences. - - >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] - >>> window_size = 3 - >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred - >>> substitutes = [3, 4] # Splice in these items - >>> list(replace(iterable, pred, substitutes, window_size=window_size)) - [3, 4, 5, 3, 4, 5] - - """ - if window_size < 1: - raise ValueError('window_size must be at least 1') - - # Save the substitutes iterable, since it's used more than once - substitutes = tuple(substitutes) - - # Add padding such that the number of windows matches the length of the - # iterable - it = chain(iterable, [_marker] * (window_size - 1)) - windows = windowed(it, window_size) - - n = 0 - for w in windows: - # If the current window matches our predicate (and we haven't hit - # our maximum number of replacements), splice in the substitutes - # and then consume the following windows that overlap with this one. - # For example, if the iterable is (0, 1, 2, 3, 4...) - # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... - # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) - if pred(*w): - if (count is None) or (n < count): - n += 1 - yield from substitutes - consume(windows, window_size - 1) - continue - - # If there was no match (or we've reached the replacement limit), - # yield the first item from the window. - if w and (w[0] is not _marker): - yield w[0] - - -def partitions(iterable): - """Yield all possible order-preserving partitions of *iterable*. - - >>> iterable = 'abc' - >>> for part in partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['a', 'b', 'c'] - - This is unrelated to :func:`partition`. - - """ - sequence = list(iterable) - n = len(sequence) - for i in powerset(range(1, n)): - yield [sequence[i:j] for i, j in zip((0,) + i, i + (n,))] - - -def set_partitions(iterable, k=None): - """ - Yield the set partitions of *iterable* into *k* parts. Set partitions are - not order-preserving. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable, 2): - ... print([''.join(p) for p in part]) - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - - - If *k* is not given, every set partition is generated. - - >>> iterable = 'abc' - >>> for part in set_partitions(iterable): - ... print([''.join(p) for p in part]) - ['abc'] - ['a', 'bc'] - ['ab', 'c'] - ['b', 'ac'] - ['a', 'b', 'c'] - - """ - L = list(iterable) - n = len(L) - if k is not None: - if k < 1: - raise ValueError( - "Can't partition in a negative or zero number of groups" - ) - elif k > n: - return - - def set_partitions_helper(L, k): - n = len(L) - if k == 1: - yield [L] - elif n == k: - yield [[s] for s in L] - else: - e, *M = L - for p in set_partitions_helper(M, k - 1): - yield [[e], *p] - for p in set_partitions_helper(M, k): - for i in range(len(p)): - yield p[:i] + [[e] + p[i]] + p[i + 1 :] - - if k is None: - for k in range(1, n + 1): - yield from set_partitions_helper(L, k) - else: - yield from set_partitions_helper(L, k) - - -class time_limited: - """ - Yield items from *iterable* until *limit_seconds* have passed. - If the time limit expires before all items have been yielded, the - ``timed_out`` parameter will be set to ``True``. - - >>> from time import sleep - >>> def generator(): - ... yield 1 - ... yield 2 - ... sleep(0.2) - ... yield 3 - >>> iterable = time_limited(0.1, generator()) - >>> list(iterable) - [1, 2] - >>> iterable.timed_out - True - - Note that the time is checked before each item is yielded, and iteration - stops if the time elapsed is greater than *limit_seconds*. If your time - limit is 1 second, but it takes 2 seconds to generate the first item from - the iterable, the function will run for 2 seconds and not yield anything. - - """ - - def __init__(self, limit_seconds, iterable): - if limit_seconds < 0: - raise ValueError('limit_seconds must be positive') - self.limit_seconds = limit_seconds - self._iterable = iter(iterable) - self._start_time = monotonic() - self.timed_out = False - - def __iter__(self): - return self - - def __next__(self): - item = next(self._iterable) - if monotonic() - self._start_time > self.limit_seconds: - self.timed_out = True - raise StopIteration - - return item - - -def only(iterable, default=None, too_long=None): - """If *iterable* has only one item, return it. - If it has zero items, return *default*. - If it has more than one item, raise the exception given by *too_long*, - which is ``ValueError`` by default. - - >>> only([], default='missing') - 'missing' - >>> only([1]) - 1 - >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - ValueError: Expected exactly one item in iterable, but got 1, 2, - and perhaps more.' - >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - TypeError - - Note that :func:`only` attempts to advance *iterable* twice to ensure there - is only one item. See :func:`spy` or :func:`peekable` to check - iterable contents less destructively. - """ - it = iter(iterable) - first_value = next(it, default) - - try: - second_value = next(it) - except StopIteration: - pass - else: - msg = ( - 'Expected exactly one item in iterable, but got {!r}, {!r}, ' - 'and perhaps more.'.format(first_value, second_value) - ) - raise too_long or ValueError(msg) - - return first_value - - -def ichunked(iterable, n): - """Break *iterable* into sub-iterables with *n* elements each. - :func:`ichunked` is like :func:`chunked`, but it yields iterables - instead of lists. - - If the sub-iterables are read in order, the elements of *iterable* - won't be stored in memory. - If they are read out of order, :func:`itertools.tee` is used to cache - elements as necessary. - - >>> from itertools import count - >>> all_chunks = ichunked(count(), 4) - >>> c_1, c_2, c_3 = next(all_chunks), next(all_chunks), next(all_chunks) - >>> list(c_2) # c_1's elements have been cached; c_3's haven't been - [4, 5, 6, 7] - >>> list(c_1) - [0, 1, 2, 3] - >>> list(c_3) - [8, 9, 10, 11] - - """ - source = iter(iterable) - - while True: - # Check to see whether we're at the end of the source iterable - item = next(source, _marker) - if item is _marker: - return - - # Clone the source and yield an n-length slice - source, it = tee(chain([item], source)) - yield islice(it, n) - - # Advance the source iterable - consume(source, n) - - -def distinct_combinations(iterable, r): - """Yield the distinct combinations of *r* items taken from *iterable*. - - >>> list(distinct_combinations([0, 0, 1], 2)) - [(0, 0), (0, 1)] - - Equivalent to ``set(combinations(iterable))``, except duplicates are not - generated and thrown away. For larger input sequences this is much more - efficient. - - """ - if r < 0: - raise ValueError('r must be non-negative') - elif r == 0: - yield () - return - pool = tuple(iterable) - generators = [unique_everseen(enumerate(pool), key=itemgetter(1))] - current_combo = [None] * r - level = 0 - while generators: - try: - cur_idx, p = next(generators[-1]) - except StopIteration: - generators.pop() - level -= 1 - continue - current_combo[level] = p - if level + 1 == r: - yield tuple(current_combo) - else: - generators.append( - unique_everseen( - enumerate(pool[cur_idx + 1 :], cur_idx + 1), - key=itemgetter(1), - ) - ) - level += 1 - - -def filter_except(validator, iterable, *exceptions): - """Yield the items from *iterable* for which the *validator* function does - not raise one of the specified *exceptions*. - - *validator* is called for each item in *iterable*. - It should be a function that accepts one argument and raises an exception - if that item is not valid. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(filter_except(int, iterable, ValueError, TypeError)) - ['1', '2', '4'] - - If an exception other than one given by *exceptions* is raised by - *validator*, it is raised like normal. - """ - for item in iterable: - try: - validator(item) - except exceptions: - pass - else: - yield item - - -def map_except(function, iterable, *exceptions): - """Transform each item from *iterable* with *function* and yield the - result, unless *function* raises one of the specified *exceptions*. - - *function* is called to transform each item in *iterable*. - It should be a accept one argument. - - >>> iterable = ['1', '2', 'three', '4', None] - >>> list(map_except(int, iterable, ValueError, TypeError)) - [1, 2, 4] - - If an exception other than one given by *exceptions* is raised by - *function*, it is raised like normal. - """ - for item in iterable: - try: - yield function(item) - except exceptions: - pass - - -def _sample_unweighted(iterable, k): - # Implementation of "Algorithm L" from the 1994 paper by Kim-Hung Li: - # "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". - - # Fill up the reservoir (collection of samples) with the first `k` samples - reservoir = take(k, iterable) - - # Generate random number that's the largest in a sample of k U(0,1) numbers - # Largest order statistic: https://en.wikipedia.org/wiki/Order_statistic - W = exp(log(random()) / k) - - # The number of elements to skip before changing the reservoir is a random - # number with a geometric distribution. Sample it using random() and logs. - next_index = k + floor(log(random()) / log(1 - W)) - - for index, element in enumerate(iterable, k): - - if index == next_index: - reservoir[randrange(k)] = element - # The new W is the largest in a sample of k U(0, `old_W`) numbers - W *= exp(log(random()) / k) - next_index += floor(log(random()) / log(1 - W)) + 1 - - return reservoir - - -def _sample_weighted(iterable, k, weights): - # Implementation of "A-ExpJ" from the 2006 paper by Efraimidis et al. : - # "Weighted random sampling with a reservoir". - - # Log-transform for numerical stability for weights that are small/large - weight_keys = (log(random()) / weight for weight in weights) - - # Fill up the reservoir (collection of samples) with the first `k` - # weight-keys and elements, then heapify the list. - reservoir = take(k, zip(weight_keys, iterable)) - heapify(reservoir) - - # The number of jumps before changing the reservoir is a random variable - # with an exponential distribution. Sample it using random() and logs. - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - - for weight, element in zip(weights, iterable): - if weight >= weights_to_skip: - # The notation here is consistent with the paper, but we store - # the weight-keys in log-space for better numerical stability. - smallest_weight_key, _ = reservoir[0] - t_w = exp(weight * smallest_weight_key) - r_2 = uniform(t_w, 1) # generate U(t_w, 1) - weight_key = log(r_2) / weight - heapreplace(reservoir, (weight_key, element)) - smallest_weight_key, _ = reservoir[0] - weights_to_skip = log(random()) / smallest_weight_key - else: - weights_to_skip -= weight - - # Equivalent to [element for weight_key, element in sorted(reservoir)] - return [heappop(reservoir)[1] for _ in range(k)] - - -def sample(iterable, k, weights=None): - """Return a *k*-length list of elements chosen (without replacement) - from the *iterable*. Like :func:`random.sample`, but works on iterables - of unknown length. - - >>> iterable = range(100) - >>> sample(iterable, 5) # doctest: +SKIP - [81, 60, 96, 16, 4] - - An iterable with *weights* may also be given: - - >>> iterable = range(100) - >>> weights = (i * i + 1 for i in range(100)) - >>> sampled = sample(iterable, 5, weights=weights) # doctest: +SKIP - [79, 67, 74, 66, 78] - - The algorithm can also be used to generate weighted random permutations. - The relative weight of each item determines the probability that it - appears late in the permutation. - - >>> data = "abcdefgh" - >>> weights = range(1, len(data) + 1) - >>> sample(data, k=len(data), weights=weights) # doctest: +SKIP - ['c', 'a', 'b', 'e', 'g', 'd', 'h', 'f'] - """ - if k == 0: - return [] - - iterable = iter(iterable) - if weights is None: - return _sample_unweighted(iterable, k) - else: - weights = iter(weights) - return _sample_weighted(iterable, k, weights) - - -def is_sorted(iterable, key=None, reverse=False): - """Returns ``True`` if the items of iterable are in sorted order, and - ``False`` otherwise. *key* and *reverse* have the same meaning that they do - in the built-in :func:`sorted` function. - - >>> is_sorted(['1', '2', '3', '4', '5'], key=int) - True - >>> is_sorted([5, 4, 3, 1, 2], reverse=True) - False - - The function returns ``False`` after encountering the first out-of-order - item. If there are no out-of-order items, the iterable is exhausted. - """ - - compare = lt if reverse else gt - it = iterable if (key is None) else map(key, iterable) - return not any(starmap(compare, pairwise(it))) - - -class AbortThread(BaseException): - pass - - -class callback_iter: - """Convert a function that uses callbacks to an iterator. - - Let *func* be a function that takes a `callback` keyword argument. - For example: - - >>> def func(callback=None): - ... for i, c in [(1, 'a'), (2, 'b'), (3, 'c')]: - ... if callback: - ... callback(i, c) - ... return 4 - - - Use ``with callback_iter(func)`` to get an iterator over the parameters - that are delivered to the callback. - - >>> with callback_iter(func) as it: - ... for args, kwargs in it: - ... print(args) - (1, 'a') - (2, 'b') - (3, 'c') - - The function will be called in a background thread. The ``done`` property - indicates whether it has completed execution. - - >>> it.done - True - - If it completes successfully, its return value will be available - in the ``result`` property. - - >>> it.result - 4 - - Notes: - - * If the function uses some keyword argument besides ``callback``, supply - *callback_kwd*. - * If it finished executing, but raised an exception, accessing the - ``result`` property will raise the same exception. - * If it hasn't finished executing, accessing the ``result`` - property from within the ``with`` block will raise ``RuntimeError``. - * If it hasn't finished executing, accessing the ``result`` property from - outside the ``with`` block will raise a - ``more_itertools.AbortThread`` exception. - * Provide *wait_seconds* to adjust how frequently the it is polled for - output. - - """ - - def __init__(self, func, callback_kwd='callback', wait_seconds=0.1): - self._func = func - self._callback_kwd = callback_kwd - self._aborted = False - self._future = None - self._wait_seconds = wait_seconds - self._executor = __import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1) - self._iterator = self._reader() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._aborted = True - self._executor.shutdown() - - def __iter__(self): - return self - - def __next__(self): - return next(self._iterator) - - @property - def done(self): - if self._future is None: - return False - return self._future.done() - - @property - def result(self): - if not self.done: - raise RuntimeError('Function has not yet completed') - - return self._future.result() - - def _reader(self): - q = Queue() - - def callback(*args, **kwargs): - if self._aborted: - raise AbortThread('canceled by user') - - q.put((args, kwargs)) - - self._future = self._executor.submit( - self._func, **{self._callback_kwd: callback} - ) - - while True: - try: - item = q.get(timeout=self._wait_seconds) - except Empty: - pass - else: - q.task_done() - yield item - - if self._future.done(): - break - - remaining = [] - while True: - try: - item = q.get_nowait() - except Empty: - break - else: - q.task_done() - remaining.append(item) - q.join() - yield from remaining - - -def windowed_complete(iterable, n): - """ - Yield ``(beginning, middle, end)`` tuples, where: - - * Each ``middle`` has *n* items from *iterable* - * Each ``beginning`` has the items before the ones in ``middle`` - * Each ``end`` has the items after the ones in ``middle`` - - >>> iterable = range(7) - >>> n = 3 - >>> for beginning, middle, end in windowed_complete(iterable, n): - ... print(beginning, middle, end) - () (0, 1, 2) (3, 4, 5, 6) - (0,) (1, 2, 3) (4, 5, 6) - (0, 1) (2, 3, 4) (5, 6) - (0, 1, 2) (3, 4, 5) (6,) - (0, 1, 2, 3) (4, 5, 6) () - - Note that *n* must be at least 0 and most equal to the length of - *iterable*. - - This function will exhaust the iterable and may require significant - storage. - """ - if n < 0: - raise ValueError('n must be >= 0') - - seq = tuple(iterable) - size = len(seq) - - if n > size: - raise ValueError('n must be <= len(seq)') - - for i in range(size - n + 1): - beginning = seq[:i] - middle = seq[i : i + n] - end = seq[i + n :] - yield beginning, middle, end - - -def all_unique(iterable, key=None): - """ - Returns ``True`` if all the elements of *iterable* are unique (no two - elements are equal). - - >>> all_unique('ABCB') - False - - If a *key* function is specified, it will be used to make comparisons. - - >>> all_unique('ABCb') - True - >>> all_unique('ABCb', str.lower) - False - - The function returns as soon as the first non-unique element is - encountered. Iterables with a mix of hashable and unhashable items can - be used, but the function will be slower for unhashable items. - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - for element in map(key, iterable) if key else iterable: - try: - if element in seenset: - return False - seenset_add(element) - except TypeError: - if element in seenlist: - return False - seenlist_add(element) - return True - - -def nth_product(index, *args): - """Equivalent to ``list(product(*args))[index]``. - - The products of *args* can be ordered lexicographically. - :func:`nth_product` computes the product at sort position *index* without - computing the previous products. - - >>> nth_product(8, range(2), range(2), range(2), range(2)) - (1, 0, 0, 0) - - ``IndexError`` will be raised if the given *index* is invalid. - """ - pools = list(map(tuple, reversed(args))) - ns = list(map(len, pools)) - - c = reduce(mul, ns) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - result = [] - for pool, n in zip(pools, ns): - result.append(pool[index % n]) - index //= n - - return tuple(reversed(result)) - - -def nth_permutation(iterable, r, index): - """Equivalent to ``list(permutations(iterable, r))[index]``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`nth_permutation` - computes the subsequence at sort position *index* directly, without - computing the previous subsequences. - - >>> nth_permutation('ghijk', 2, 5) - ('h', 'i') - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = list(iterable) - n = len(pool) - - if r is None or r == n: - r, c = n, factorial(n) - elif not 0 <= r < n: - raise ValueError - else: - c = factorial(n) // factorial(n - r) - - if index < 0: - index += c - - if not 0 <= index < c: - raise IndexError - - if c == 0: - return tuple() - - result = [0] * r - q = index * factorial(n) // c if r < n else index - for d in range(1, n + 1): - q, i = divmod(q, d) - if 0 <= n - d < r: - result[n - d] = i - if q == 0: - break - - return tuple(map(pool.pop, result)) - - -def value_chain(*args): - """Yield all arguments passed to the function in the same order in which - they were passed. If an argument itself is iterable then iterate over its - values. - - >>> list(value_chain(1, 2, 3, [4, 5, 6])) - [1, 2, 3, 4, 5, 6] - - Binary and text strings are not considered iterable and are emitted - as-is: - - >>> list(value_chain('12', '34', ['56', '78'])) - ['12', '34', '56', '78'] - - - Multiple levels of nesting are not flattened. - - """ - for value in args: - if isinstance(value, (str, bytes)): - yield value - continue - try: - yield from value - except TypeError: - yield value - - -def product_index(element, *args): - """Equivalent to ``list(product(*args)).index(element)`` - - The products of *args* can be ordered lexicographically. - :func:`product_index` computes the first index of *element* without - computing the previous products. - - >>> product_index([8, 2], range(10), range(5)) - 42 - - ``ValueError`` will be raised if the given *element* isn't in the product - of *args*. - """ - index = 0 - - for x, pool in zip_longest(element, args, fillvalue=_marker): - if x is _marker or pool is _marker: - raise ValueError('element is not a product of args') - - pool = tuple(pool) - index = index * len(pool) + pool.index(x) - - return index - - -def combination_index(element, iterable): - """Equivalent to ``list(combinations(iterable, r)).index(element)`` - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`combination_index` computes the index of the - first *element*, without computing the previous combinations. - - >>> combination_index('adf', 'abcdefg') - 10 - - ``ValueError`` will be raised if the given *element* isn't one of the - combinations of *iterable*. - """ - element = enumerate(element) - k, y = next(element, (None, None)) - if k is None: - return 0 - - indexes = [] - pool = enumerate(iterable) - for n, x in pool: - if x == y: - indexes.append(n) - tmp, y = next(element, (None, None)) - if tmp is None: - break - else: - k = tmp - else: - raise ValueError('element is not a combination of iterable') - - n, _ = last(pool, default=(n, None)) - - # Python versiosn below 3.8 don't have math.comb - index = 1 - for i, j in enumerate(reversed(indexes), start=1): - j = n - j - if i <= j: - index += factorial(j) // (factorial(i) * factorial(j - i)) - - return factorial(n + 1) // (factorial(k + 1) * factorial(n - k)) - index - - -def permutation_index(element, iterable): - """Equivalent to ``list(permutations(iterable, r)).index(element)``` - - The subsequences of *iterable* that are of length *r* where order is - important can be ordered lexicographically. :func:`permutation_index` - computes the index of the first *element* directly, without computing - the previous permutations. - - >>> permutation_index([1, 3, 2], range(5)) - 19 - - ``ValueError`` will be raised if the given *element* isn't one of the - permutations of *iterable*. - """ - index = 0 - pool = list(iterable) - for i, x in zip(range(len(pool), -1, -1), element): - r = pool.index(x) - index = index * i + r - del pool[r] - - return index - - -class countable: - """Wrap *iterable* and keep a count of how many items have been consumed. - - The ``items_seen`` attribute starts at ``0`` and increments as the iterable - is consumed: - - >>> iterable = map(str, range(10)) - >>> it = countable(iterable) - >>> it.items_seen - 0 - >>> next(it), next(it) - ('0', '1') - >>> list(it) - ['2', '3', '4', '5', '6', '7', '8', '9'] - >>> it.items_seen - 10 - """ - - def __init__(self, iterable): - self._it = iter(iterable) - self.items_seen = 0 - - def __iter__(self): - return self - - def __next__(self): - item = next(self._it) - self.items_seen += 1 - - return item diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi deleted file mode 100644 index 2fba9cb300b..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/more.pyi +++ /dev/null @@ -1,480 +0,0 @@ -"""Stubs for more_itertools.more""" - -from typing import ( - Any, - Callable, - Container, - Dict, - Generic, - Hashable, - Iterable, - Iterator, - List, - Optional, - Reversible, - Sequence, - Sized, - Tuple, - Union, - TypeVar, - type_check_only, -) -from types import TracebackType -from typing_extensions import ContextManager, Protocol, Type, overload - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') -_V = TypeVar('_V') -_W = TypeVar('_W') -_T_co = TypeVar('_T_co', covariant=True) -_GenFn = TypeVar('_GenFn', bound=Callable[..., Iterator[object]]) -_Raisable = Union[BaseException, 'Type[BaseException]'] - -@type_check_only -class _SizedIterable(Protocol[_T_co], Sized, Iterable[_T_co]): ... - -@type_check_only -class _SizedReversible(Protocol[_T_co], Sized, Reversible[_T_co]): ... - -def chunked( - iterable: Iterable[_T], n: int, strict: bool = ... -) -> Iterator[List[_T]]: ... -@overload -def first(iterable: Iterable[_T]) -> _T: ... -@overload -def first(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... -@overload -def last(iterable: Iterable[_T]) -> _T: ... -@overload -def last(iterable: Iterable[_T], default: _U) -> Union[_T, _U]: ... -@overload -def nth_or_last(iterable: Iterable[_T], n: int) -> _T: ... -@overload -def nth_or_last( - iterable: Iterable[_T], n: int, default: _U -) -> Union[_T, _U]: ... - -class peekable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> peekable[_T]: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> Union[_T, _U]: ... - def prepend(self, *items: _T) -> None: ... - def __next__(self) -> _T: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> List[_T]: ... - -def collate(*iterables: Iterable[_T], **kwargs: Any) -> Iterable[_T]: ... -def consumer(func: _GenFn) -> _GenFn: ... -def ilen(iterable: Iterable[object]) -> int: ... -def iterate(func: Callable[[_T], _T], start: _T) -> Iterator[_T]: ... -def with_iter( - context_manager: ContextManager[Iterable[_T]], -) -> Iterator[_T]: ... -def one( - iterable: Iterable[_T], - too_short: Optional[_Raisable] = ..., - too_long: Optional[_Raisable] = ..., -) -> _T: ... -def distinct_permutations( - iterable: Iterable[_T], r: Optional[int] = ... -) -> Iterator[Tuple[_T, ...]]: ... -def intersperse( - e: _U, iterable: Iterable[_T], n: int = ... -) -> Iterator[Union[_T, _U]]: ... -def unique_to_each(*iterables: Iterable[_T]) -> List[List[_T]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, *, step: int = ... -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def windowed( - seq: Iterable[_T], n: int, fillvalue: _U, step: int = ... -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def substrings(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -def substrings_indexes( - seq: Sequence[_T], reverse: bool = ... -) -> Iterator[Tuple[Sequence[_T], int, int]]: ... - -class bucket(Generic[_T, _U], Container[_U]): - def __init__( - self, - iterable: Iterable[_T], - key: Callable[[_T], _U], - validator: Optional[Callable[[object], object]] = ..., - ) -> None: ... - def __contains__(self, value: object) -> bool: ... - def __iter__(self) -> Iterator[_U]: ... - def __getitem__(self, value: object) -> Iterator[_T]: ... - -def spy( - iterable: Iterable[_T], n: int = ... -) -> Tuple[List[_T], Iterator[_T]]: ... -def interleave(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def interleave_longest(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def collapse( - iterable: Iterable[Any], - base_type: Optional[type] = ..., - levels: Optional[int] = ..., -) -> Iterator[Any]: ... -@overload -def side_effect( - func: Callable[[_T], object], - iterable: Iterable[_T], - chunk_size: None = ..., - before: Optional[Callable[[], object]] = ..., - after: Optional[Callable[[], object]] = ..., -) -> Iterator[_T]: ... -@overload -def side_effect( - func: Callable[[List[_T]], object], - iterable: Iterable[_T], - chunk_size: int, - before: Optional[Callable[[], object]] = ..., - after: Optional[Callable[[], object]] = ..., -) -> Iterator[_T]: ... -def sliced( - seq: Sequence[_T], n: int, strict: bool = ... -) -> Iterator[Sequence[_T]]: ... -def split_at( - iterable: Iterable[_T], - pred: Callable[[_T], object], - maxsplit: int = ..., - keep_separator: bool = ..., -) -> Iterator[List[_T]]: ... -def split_before( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[List[_T]]: ... -def split_after( - iterable: Iterable[_T], pred: Callable[[_T], object], maxsplit: int = ... -) -> Iterator[List[_T]]: ... -def split_when( - iterable: Iterable[_T], - pred: Callable[[_T, _T], object], - maxsplit: int = ..., -) -> Iterator[List[_T]]: ... -def split_into( - iterable: Iterable[_T], sizes: Iterable[Optional[int]] -) -> Iterator[List[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - *, - n: Optional[int] = ..., - next_multiple: bool = ... -) -> Iterator[Optional[_T]]: ... -@overload -def padded( - iterable: Iterable[_T], - fillvalue: _U, - n: Optional[int] = ..., - next_multiple: bool = ..., -) -> Iterator[Union[_T, _U]]: ... -@overload -def repeat_last(iterable: Iterable[_T]) -> Iterator[_T]: ... -@overload -def repeat_last( - iterable: Iterable[_T], default: _U -) -> Iterator[Union[_T, _U]]: ... -def distribute(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def stagger( - iterable: Iterable[_T], - offsets: _SizedIterable[int] = ..., - longest: bool = ..., - fillvalue: _U = ..., -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... - -class UnequalIterablesError(ValueError): - def __init__( - self, details: Optional[Tuple[int, int, int]] = ... - ) -> None: ... - -def zip_equal(*iterables: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -@overload -def zip_offset( - *iterables: Iterable[_T], offsets: _SizedIterable[int], longest: bool = ... -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def zip_offset( - *iterables: Iterable[_T], - offsets: _SizedIterable[int], - longest: bool = ..., - fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def sort_together( - iterables: Iterable[Iterable[_T]], - key_list: Iterable[int] = ..., - key: Optional[Callable[..., Any]] = ..., - reverse: bool = ..., -) -> List[Tuple[_T, ...]]: ... -def unzip(iterable: Iterable[Sequence[_T]]) -> Tuple[Iterator[_T], ...]: ... -def divide(n: int, iterable: Iterable[_T]) -> List[Iterator[_T]]: ... -def always_iterable( - obj: object, - base_type: Union[ - type, Tuple[Union[type, Tuple[Any, ...]], ...], None - ] = ..., -) -> Iterator[Any]: ... -def adjacent( - predicate: Callable[[_T], bool], - iterable: Iterable[_T], - distance: int = ..., -) -> Iterator[Tuple[bool, _T]]: ... -def groupby_transform( - iterable: Iterable[_T], - keyfunc: Optional[Callable[[_T], _U]] = ..., - valuefunc: Optional[Callable[[_T], _V]] = ..., - reducefunc: Optional[Callable[..., _W]] = ..., -) -> Iterator[Tuple[_T, _W]]: ... - -class numeric_range(Generic[_T, _U], Sequence[_T], Hashable, Reversible[_T]): - @overload - def __init__(self, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T) -> None: ... - @overload - def __init__(self, __start: _T, __stop: _T, __step: _U) -> None: ... - def __bool__(self) -> bool: ... - def __contains__(self, elem: object) -> bool: ... - def __eq__(self, other: object) -> bool: ... - @overload - def __getitem__(self, key: int) -> _T: ... - @overload - def __getitem__(self, key: slice) -> numeric_range[_T, _U]: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[_T]: ... - def __len__(self) -> int: ... - def __reduce__( - self, - ) -> Tuple[Type[numeric_range[_T, _U]], Tuple[_T, _T, _U]]: ... - def __repr__(self) -> str: ... - def __reversed__(self) -> Iterator[_T]: ... - def count(self, value: _T) -> int: ... - def index(self, value: _T) -> int: ... # type: ignore - -def count_cycle( - iterable: Iterable[_T], n: Optional[int] = ... -) -> Iterable[Tuple[int, _T]]: ... -def mark_ends( - iterable: Iterable[_T], -) -> Iterable[Tuple[bool, bool, _T]]: ... -def locate( - iterable: Iterable[object], - pred: Callable[..., Any] = ..., - window_size: Optional[int] = ..., -) -> Iterator[int]: ... -def lstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def rstrip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... -def strip( - iterable: Iterable[_T], pred: Callable[[_T], object] -) -> Iterator[_T]: ... - -class islice_extended(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], *args: Optional[int] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - def __getitem__(self, index: slice) -> islice_extended[_T]: ... - -def always_reversible(iterable: Iterable[_T]) -> Iterator[_T]: ... -def consecutive_groups( - iterable: Iterable[_T], ordering: Callable[[_T], int] = ... -) -> Iterator[Iterator[_T]]: ... -@overload -def difference( - iterable: Iterable[_T], - func: Callable[[_T, _T], _U] = ..., - *, - initial: None = ... -) -> Iterator[Union[_T, _U]]: ... -@overload -def difference( - iterable: Iterable[_T], func: Callable[[_T, _T], _U] = ..., *, initial: _U -) -> Iterator[_U]: ... - -class SequenceView(Generic[_T], Sequence[_T]): - def __init__(self, target: Sequence[_T]) -> None: ... - @overload - def __getitem__(self, index: int) -> _T: ... - @overload - def __getitem__(self, index: slice) -> Sequence[_T]: ... - def __len__(self) -> int: ... - -class seekable(Generic[_T], Iterator[_T]): - def __init__( - self, iterable: Iterable[_T], maxlen: Optional[int] = ... - ) -> None: ... - def __iter__(self) -> seekable[_T]: ... - def __next__(self) -> _T: ... - def __bool__(self) -> bool: ... - @overload - def peek(self) -> _T: ... - @overload - def peek(self, default: _U) -> Union[_T, _U]: ... - def elements(self) -> SequenceView[_T]: ... - def seek(self, index: int) -> None: ... - -class run_length: - @staticmethod - def encode(iterable: Iterable[_T]) -> Iterator[Tuple[_T, int]]: ... - @staticmethod - def decode(iterable: Iterable[Tuple[_T, int]]) -> Iterator[_T]: ... - -def exactly_n( - iterable: Iterable[_T], n: int, predicate: Callable[[_T], object] = ... -) -> bool: ... -def circular_shifts(iterable: Iterable[_T]) -> List[Tuple[_T, ...]]: ... -def make_decorator( - wrapping_func: Callable[..., _U], result_index: int = ... -) -> Callable[..., Callable[[Callable[..., Any]], Callable[..., _U]]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: None = ..., -) -> Dict[_U, List[_T]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: None = ..., -) -> Dict[_U, List[_V]]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: None = ..., - reducefunc: Callable[[List[_T]], _W] = ..., -) -> Dict[_U, _W]: ... -@overload -def map_reduce( - iterable: Iterable[_T], - keyfunc: Callable[[_T], _U], - valuefunc: Callable[[_T], _V], - reducefunc: Callable[[List[_V]], _W], -) -> Dict[_U, _W]: ... -def rlocate( - iterable: Iterable[_T], - pred: Callable[..., object] = ..., - window_size: Optional[int] = ..., -) -> Iterator[int]: ... -def replace( - iterable: Iterable[_T], - pred: Callable[..., object], - substitutes: Iterable[_U], - count: Optional[int] = ..., - window_size: int = ..., -) -> Iterator[Union[_T, _U]]: ... -def partitions(iterable: Iterable[_T]) -> Iterator[List[List[_T]]]: ... -def set_partitions( - iterable: Iterable[_T], k: Optional[int] = ... -) -> Iterator[List[List[_T]]]: ... - -class time_limited(Generic[_T], Iterator[_T]): - def __init__( - self, limit_seconds: float, iterable: Iterable[_T] - ) -> None: ... - def __iter__(self) -> islice_extended[_T]: ... - def __next__(self) -> _T: ... - -@overload -def only( - iterable: Iterable[_T], *, too_long: Optional[_Raisable] = ... -) -> Optional[_T]: ... -@overload -def only( - iterable: Iterable[_T], default: _U, too_long: Optional[_Raisable] = ... -) -> Union[_T, _U]: ... -def ichunked(iterable: Iterable[_T], n: int) -> Iterator[Iterator[_T]]: ... -def distinct_combinations( - iterable: Iterable[_T], r: int -) -> Iterator[Tuple[_T, ...]]: ... -def filter_except( - validator: Callable[[Any], object], - iterable: Iterable[_T], - *exceptions: Type[BaseException] -) -> Iterator[_T]: ... -def map_except( - function: Callable[[Any], _U], - iterable: Iterable[_T], - *exceptions: Type[BaseException] -) -> Iterator[_U]: ... -def sample( - iterable: Iterable[_T], - k: int, - weights: Optional[Iterable[float]] = ..., -) -> List[_T]: ... -def is_sorted( - iterable: Iterable[_T], - key: Optional[Callable[[_T], _U]] = ..., - reverse: bool = False, -) -> bool: ... - -class AbortThread(BaseException): - pass - -class callback_iter(Generic[_T], Iterator[_T]): - def __init__( - self, - func: Callable[..., Any], - callback_kwd: str = ..., - wait_seconds: float = ..., - ) -> None: ... - def __enter__(self) -> callback_iter[_T]: ... - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> Optional[bool]: ... - def __iter__(self) -> callback_iter[_T]: ... - def __next__(self) -> _T: ... - def _reader(self) -> Iterator[_T]: ... - @property - def done(self) -> bool: ... - @property - def result(self) -> Any: ... - -def windowed_complete( - iterable: Iterable[_T], n: int -) -> Iterator[Tuple[_T, ...]]: ... -def all_unique( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... -) -> bool: ... -def nth_product(index: int, *args: Iterable[_T]) -> Tuple[_T, ...]: ... -def nth_permutation( - iterable: Iterable[_T], r: int, index: int -) -> Tuple[_T, ...]: ... -def value_chain(*args: Union[_T, Iterable[_T]]) -> Iterable[_T]: ... -def product_index(element: Iterable[_T], *args: Iterable[_T]) -> int: ... -def combination_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... -def permutation_index( - element: Iterable[_T], iterable: Iterable[_T] -) -> int: ... - -class countable(Generic[_T], Iterator[_T]): - def __init__(self, iterable: Iterable[_T]) -> None: ... - def __iter__(self) -> countable[_T]: ... - def __next__(self) -> _T: ... diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py deleted file mode 100644 index 521abd7c2ca..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.py +++ /dev/null @@ -1,620 +0,0 @@ -"""Imported from the recipes section of the itertools documentation. - -All functions taken from the recipes section of the itertools library docs -[1]_. -Some backward-compatible usability improvements have been made. - -.. [1] http://docs.python.org/library/itertools.html#recipes - -""" -import warnings -from collections import deque -from itertools import ( - chain, - combinations, - count, - cycle, - groupby, - islice, - repeat, - starmap, - tee, - zip_longest, -) -import operator -from random import randrange, sample, choice - -__all__ = [ - 'all_equal', - 'consume', - 'convolve', - 'dotproduct', - 'first_true', - 'flatten', - 'grouper', - 'iter_except', - 'ncycles', - 'nth', - 'nth_combination', - 'padnone', - 'pad_none', - 'pairwise', - 'partition', - 'powerset', - 'prepend', - 'quantify', - 'random_combination_with_replacement', - 'random_combination', - 'random_permutation', - 'random_product', - 'repeatfunc', - 'roundrobin', - 'tabulate', - 'tail', - 'take', - 'unique_everseen', - 'unique_justseen', -] - - -def take(n, iterable): - """Return first *n* items of the iterable as a list. - - >>> take(3, range(10)) - [0, 1, 2] - - If there are fewer than *n* items in the iterable, all of them are - returned. - - >>> take(10, range(3)) - [0, 1, 2] - - """ - return list(islice(iterable, n)) - - -def tabulate(function, start=0): - """Return an iterator over the results of ``func(start)``, - ``func(start + 1)``, ``func(start + 2)``... - - *func* should be a function that accepts one integer argument. - - If *start* is not specified it defaults to 0. It will be incremented each - time the iterator is advanced. - - >>> square = lambda x: x ** 2 - >>> iterator = tabulate(square, -3) - >>> take(4, iterator) - [9, 4, 1, 0] - - """ - return map(function, count(start)) - - -def tail(n, iterable): - """Return an iterator over the last *n* items of *iterable*. - - >>> t = tail(3, 'ABCDEFG') - >>> list(t) - ['E', 'F', 'G'] - - """ - return iter(deque(iterable, maxlen=n)) - - -def consume(iterator, n=None): - """Advance *iterable* by *n* steps. If *n* is ``None``, consume it - entirely. - - Efficiently exhausts an iterator without returning values. Defaults to - consuming the whole iterator, but an optional second argument may be - provided to limit consumption. - - >>> i = (x for x in range(10)) - >>> next(i) - 0 - >>> consume(i, 3) - >>> next(i) - 4 - >>> consume(i) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - If the iterator has fewer items remaining than the provided limit, the - whole iterator will be consumed. - - >>> i = (x for x in range(3)) - >>> consume(i, 5) - >>> next(i) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - - """ - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - -def nth(iterable, n, default=None): - """Returns the nth item or a default value. - - >>> l = range(10) - >>> nth(l, 3) - 3 - >>> nth(l, 20, "zebra") - 'zebra' - - """ - return next(islice(iterable, n, None), default) - - -def all_equal(iterable): - """ - Returns ``True`` if all the elements are equal to each other. - - >>> all_equal('aaaa') - True - >>> all_equal('aaab') - False - - """ - g = groupby(iterable) - return next(g, True) and not next(g, False) - - -def quantify(iterable, pred=bool): - """Return the how many times the predicate is true. - - >>> quantify([True, False, True]) - 2 - - """ - return sum(map(pred, iterable)) - - -def pad_none(iterable): - """Returns the sequence of elements and then returns ``None`` indefinitely. - - >>> take(5, pad_none(range(3))) - [0, 1, 2, None, None] - - Useful for emulating the behavior of the built-in :func:`map` function. - - See also :func:`padded`. - - """ - return chain(iterable, repeat(None)) - - -padnone = pad_none - - -def ncycles(iterable, n): - """Returns the sequence elements *n* times - - >>> list(ncycles(["a", "b"], 3)) - ['a', 'b', 'a', 'b', 'a', 'b'] - - """ - return chain.from_iterable(repeat(tuple(iterable), n)) - - -def dotproduct(vec1, vec2): - """Returns the dot product of the two iterables. - - >>> dotproduct([10, 10], [20, 20]) - 400 - - """ - return sum(map(operator.mul, vec1, vec2)) - - -def flatten(listOfLists): - """Return an iterator flattening one level of nesting in a list of lists. - - >>> list(flatten([[0, 1], [2, 3]])) - [0, 1, 2, 3] - - See also :func:`collapse`, which can flatten multiple levels of nesting. - - """ - return chain.from_iterable(listOfLists) - - -def repeatfunc(func, times=None, *args): - """Call *func* with *args* repeatedly, returning an iterable over the - results. - - If *times* is specified, the iterable will terminate after that many - repetitions: - - >>> from operator import add - >>> times = 4 - >>> args = 3, 5 - >>> list(repeatfunc(add, times, *args)) - [8, 8, 8, 8] - - If *times* is ``None`` the iterable will not terminate: - - >>> from random import randrange - >>> times = None - >>> args = 1, 11 - >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP - [2, 4, 8, 1, 8, 4] - - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - -def _pairwise(iterable): - """Returns an iterator of paired items, overlapping, from the original - - >>> take(4, pairwise(count())) - [(0, 1), (1, 2), (2, 3), (3, 4)] - - On Python 3.10 and above, this is an alias for :func:`itertools.pairwise`. - - """ - a, b = tee(iterable) - next(b, None) - yield from zip(a, b) - - -try: - from itertools import pairwise as itertools_pairwise -except ImportError: - pairwise = _pairwise -else: - - def pairwise(iterable): - yield from itertools_pairwise(iterable) - - pairwise.__doc__ = _pairwise.__doc__ - - -def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks. - - >>> list(grouper('ABCDEFG', 3, 'x')) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] - - """ - if isinstance(iterable, int): - warnings.warn( - "grouper expects iterable as first parameter", DeprecationWarning - ) - n, iterable = iterable, n - args = [iter(iterable)] * n - return zip_longest(fillvalue=fillvalue, *args) - - -def roundrobin(*iterables): - """Yields an item from each iterable, alternating between them. - - >>> list(roundrobin('ABC', 'D', 'EF')) - ['A', 'D', 'E', 'B', 'F', 'C'] - - This function produces the same output as :func:`interleave_longest`, but - may perform better for some inputs (in particular when the number of - iterables is small). - - """ - # Recipe credited to George Sakkis - pending = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while pending: - try: - for next in nexts: - yield next() - except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) - - -def partition(pred, iterable): - """ - Returns a 2-tuple of iterables derived from the input iterable. - The first yields the items that have ``pred(item) == False``. - The second yields the items that have ``pred(item) == True``. - - >>> is_odd = lambda x: x % 2 != 0 - >>> iterable = range(10) - >>> even_items, odd_items = partition(is_odd, iterable) - >>> list(even_items), list(odd_items) - ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) - - If *pred* is None, :func:`bool` is used. - - >>> iterable = [0, 1, False, True, '', ' '] - >>> false_items, true_items = partition(None, iterable) - >>> list(false_items), list(true_items) - ([0, False, ''], [1, True, ' ']) - - """ - if pred is None: - pred = bool - - evaluations = ((pred(x), x) for x in iterable) - t1, t2 = tee(evaluations) - return ( - (x for (cond, x) in t1 if not cond), - (x for (cond, x) in t2 if cond), - ) - - -def powerset(iterable): - """Yields all possible subsets of the iterable. - - >>> list(powerset([1, 2, 3])) - [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - - :func:`powerset` will operate on iterables that aren't :class:`set` - instances, so repeated elements in the input will produce repeated elements - in the output. Use :func:`unique_everseen` on the input to avoid generating - duplicates: - - >>> seq = [1, 1, 0] - >>> list(powerset(seq)) - [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] - >>> from more_itertools import unique_everseen - >>> list(powerset(unique_everseen(seq))) - [(), (1,), (0,), (1, 0)] - - """ - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) - - -def unique_everseen(iterable, key=None): - """ - Yield unique elements, preserving order. - - >>> list(unique_everseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'D'] - - Sequences with a mix of hashable and unhashable items can be used. - The function will be slower (i.e., `O(n^2)`) for unhashable items. - - Remember that ``list`` objects are unhashable - you can use the *key* - parameter to transform the list to a tuple (which is hashable) to - avoid a slowdown. - - >>> iterable = ([1, 2], [2, 3], [1, 2]) - >>> list(unique_everseen(iterable)) # Slow - [[1, 2], [2, 3]] - >>> list(unique_everseen(iterable, key=tuple)) # Faster - [[1, 2], [2, 3]] - - Similary, you may want to convert unhashable ``set`` objects with - ``key=frozenset``. For ``dict`` objects, - ``key=lambda x: frozenset(x.items())`` can be used. - - """ - seenset = set() - seenset_add = seenset.add - seenlist = [] - seenlist_add = seenlist.append - use_key = key is not None - - for element in iterable: - k = key(element) if use_key else element - try: - if k not in seenset: - seenset_add(k) - yield element - except TypeError: - if k not in seenlist: - seenlist_add(k) - yield element - - -def unique_justseen(iterable, key=None): - """Yields elements in order, ignoring serial duplicates - - >>> list(unique_justseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'A', 'D'] - - """ - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - -def iter_except(func, exception, first=None): - """Yields results from a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel - to end the loop. - - >>> l = [0, 1, 2] - >>> list(iter_except(l.pop, IndexError)) - [2, 1, 0] - - """ - try: - if first is not None: - yield first() - while 1: - yield func() - except exception: - pass - - -def first_true(iterable, default=None, pred=None): - """ - Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item for which - ``pred(item) == True`` . - - >>> first_true(range(10)) - 1 - >>> first_true(range(10), pred=lambda x: x > 5) - 6 - >>> first_true(range(10), default='missing', pred=lambda x: x > 9) - 'missing' - - """ - return next(filter(pred, iterable), default) - - -def random_product(*args, repeat=1): - """Draw an item at random from each of the input iterables. - - >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP - ('c', 3, 'Z') - - If *repeat* is provided as a keyword argument, that many items will be - drawn from each iterable. - - >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP - ('a', 2, 'd', 3) - - This equivalent to taking a random selection from - ``itertools.product(*args, **kwarg)``. - - """ - pools = [tuple(pool) for pool in args] * repeat - return tuple(choice(pool) for pool in pools) - - -def random_permutation(iterable, r=None): - """Return a random *r* length permutation of the elements in *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length of - *iterable*. - - >>> random_permutation(range(5)) # doctest:+SKIP - (3, 4, 0, 1, 2) - - This equivalent to taking a random selection from - ``itertools.permutations(iterable, r)``. - - """ - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(sample(pool, r)) - - -def random_combination(iterable, r): - """Return a random *r* length subsequence of the elements in *iterable*. - - >>> random_combination(range(5), 3) # doctest:+SKIP - (2, 3, 4) - - This equivalent to taking a random selection from - ``itertools.combinations(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(sample(range(n), r)) - return tuple(pool[i] for i in indices) - - -def random_combination_with_replacement(iterable, r): - """Return a random *r* length subsequence of elements in *iterable*, - allowing individual elements to be repeated. - - >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP - (0, 0, 1, 2, 2) - - This equivalent to taking a random selection from - ``itertools.combinations_with_replacement(iterable, r)``. - - """ - pool = tuple(iterable) - n = len(pool) - indices = sorted(randrange(n) for i in range(r)) - return tuple(pool[i] for i in indices) - - -def nth_combination(iterable, r, index): - """Equivalent to ``list(combinations(iterable, r))[index]``. - - The subsequences of *iterable* that are of length *r* can be ordered - lexicographically. :func:`nth_combination` computes the subsequence at - sort position *index* directly, without computing the previous - subsequences. - - >>> nth_combination(range(5), 3, 5) - (0, 3, 4) - - ``ValueError`` will be raised If *r* is negative or greater than the length - of *iterable*. - ``IndexError`` will be raised if the given *index* is invalid. - """ - pool = tuple(iterable) - n = len(pool) - if (r < 0) or (r > n): - raise ValueError - - c = 1 - k = min(r, n - r) - for i in range(1, k + 1): - c = c * (n - k + i) // i - - if index < 0: - index += c - - if (index < 0) or (index >= c): - raise IndexError - - result = [] - while r: - c, n, r = c * r // n, n - 1, r - 1 - while index >= c: - index -= c - c, n = c * (n - r) // n, n - 1 - result.append(pool[-1 - n]) - - return tuple(result) - - -def prepend(value, iterator): - """Yield *value*, followed by the elements in *iterator*. - - >>> value = '0' - >>> iterator = ['1', '2', '3'] - >>> list(prepend(value, iterator)) - ['0', '1', '2', '3'] - - To prepend multiple values, see :func:`itertools.chain` - or :func:`value_chain`. - - """ - return chain([value], iterator) - - -def convolve(signal, kernel): - """Convolve the iterable *signal* with the iterable *kernel*. - - >>> signal = (1, 2, 3, 4, 5) - >>> kernel = [3, 2, 1] - >>> list(convolve(signal, kernel)) - [3, 8, 14, 20, 26, 14, 5] - - Note: the input arguments are not interchangeable, as the *kernel* - is immediately consumed and stored. - - """ - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n - 1)): - window.append(x) - yield sum(map(operator.mul, kernel, window)) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi b/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi deleted file mode 100644 index 5e39d963907..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/more_itertools/recipes.pyi +++ /dev/null @@ -1,103 +0,0 @@ -"""Stubs for more_itertools.recipes""" -from typing import ( - Any, - Callable, - Iterable, - Iterator, - List, - Optional, - Tuple, - TypeVar, - Union, -) -from typing_extensions import overload, Type - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - -def take(n: int, iterable: Iterable[_T]) -> List[_T]: ... -def tabulate( - function: Callable[[int], _T], start: int = ... -) -> Iterator[_T]: ... -def tail(n: int, iterable: Iterable[_T]) -> Iterator[_T]: ... -def consume(iterator: Iterable[object], n: Optional[int] = ...) -> None: ... -@overload -def nth(iterable: Iterable[_T], n: int) -> Optional[_T]: ... -@overload -def nth(iterable: Iterable[_T], n: int, default: _U) -> Union[_T, _U]: ... -def all_equal(iterable: Iterable[object]) -> bool: ... -def quantify( - iterable: Iterable[_T], pred: Callable[[_T], bool] = ... -) -> int: ... -def pad_none(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... -def padnone(iterable: Iterable[_T]) -> Iterator[Optional[_T]]: ... -def ncycles(iterable: Iterable[_T], n: int) -> Iterator[_T]: ... -def dotproduct(vec1: Iterable[object], vec2: Iterable[object]) -> object: ... -def flatten(listOfLists: Iterable[Iterable[_T]]) -> Iterator[_T]: ... -def repeatfunc( - func: Callable[..., _U], times: Optional[int] = ..., *args: Any -) -> Iterator[_U]: ... -def pairwise(iterable: Iterable[_T]) -> Iterator[Tuple[_T, _T]]: ... -@overload -def grouper( - iterable: Iterable[_T], n: int -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def grouper( - iterable: Iterable[_T], n: int, fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -@overload -def grouper( # Deprecated interface - iterable: int, n: Iterable[_T] -) -> Iterator[Tuple[Optional[_T], ...]]: ... -@overload -def grouper( # Deprecated interface - iterable: int, n: Iterable[_T], fillvalue: _U -) -> Iterator[Tuple[Union[_T, _U], ...]]: ... -def roundrobin(*iterables: Iterable[_T]) -> Iterator[_T]: ... -def partition( - pred: Optional[Callable[[_T], object]], iterable: Iterable[_T] -) -> Tuple[Iterator[_T], Iterator[_T]]: ... -def powerset(iterable: Iterable[_T]) -> Iterator[Tuple[_T, ...]]: ... -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = ... -) -> Iterator[_T]: ... -def unique_justseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], object]] = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], exception: Type[BaseException], first: None = ... -) -> Iterator[_T]: ... -@overload -def iter_except( - func: Callable[[], _T], - exception: Type[BaseException], - first: Callable[[], _U], -) -> Iterator[Union[_T, _U]]: ... -@overload -def first_true( - iterable: Iterable[_T], *, pred: Optional[Callable[[_T], object]] = ... -) -> Optional[_T]: ... -@overload -def first_true( - iterable: Iterable[_T], - default: _U, - pred: Optional[Callable[[_T], object]] = ..., -) -> Union[_T, _U]: ... -def random_product( - *args: Iterable[_T], repeat: int = ... -) -> Tuple[_T, ...]: ... -def random_permutation( - iterable: Iterable[_T], r: Optional[int] = ... -) -> Tuple[_T, ...]: ... -def random_combination(iterable: Iterable[_T], r: int) -> Tuple[_T, ...]: ... -def random_combination_with_replacement( - iterable: Iterable[_T], r: int -) -> Tuple[_T, ...]: ... -def nth_combination( - iterable: Iterable[_T], r: int, index: int -) -> Tuple[_T, ...]: ... -def prepend(value: _T, iterator: Iterable[_U]) -> Iterator[Union[_T, _U]]: ... -def convolve(signal: Iterable[_T], kernel: Iterable[_T]) -> Iterator[_T]: ... diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py deleted file mode 100644 index e7c0aa12ca9..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -__title__ = "packaging" -__summary__ = "Core utilities for Python packages" -__uri__ = "https://github.com/pypa/packaging" - -__version__ = "24.0" - -__author__ = "Donald Stufft and individual contributors" -__email__ = "donald@stufft.io" - -__license__ = "BSD-2-Clause or Apache-2.0" -__copyright__ = "2014 %s" % __author__ diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py deleted file mode 100644 index 6fb19b30bb5..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_elffile.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -ELF file parser. - -This provides a class ``ELFFile`` that parses an ELF executable in a similar -interface to ``ZipFile``. Only the read interface is implemented. - -Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca -ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html -""" - -import enum -import os -import struct -from typing import IO, Optional, Tuple - - -class ELFInvalid(ValueError): - pass - - -class EIClass(enum.IntEnum): - C32 = 1 - C64 = 2 - - -class EIData(enum.IntEnum): - Lsb = 1 - Msb = 2 - - -class EMachine(enum.IntEnum): - I386 = 3 - S390 = 22 - Arm = 40 - X8664 = 62 - AArc64 = 183 - - -class ELFFile: - """ - Representation of an ELF executable. - """ - - def __init__(self, f: IO[bytes]) -> None: - self._f = f - - try: - ident = self._read("16B") - except struct.error: - raise ELFInvalid("unable to parse identification") - magic = bytes(ident[:4]) - if magic != b"\x7fELF": - raise ELFInvalid(f"invalid magic: {magic!r}") - - self.capacity = ident[4] # Format for program header (bitness). - self.encoding = ident[5] # Data structure encoding (endianness). - - try: - # e_fmt: Format for program header. - # p_fmt: Format for section header. - # p_idx: Indexes to find p_type, p_offset, and p_filesz. - e_fmt, self._p_fmt, self._p_idx = { - (1, 1): ("<HHIIIIIHHH", "<IIIIIIII", (0, 1, 4)), # 32-bit LSB. - (1, 2): (">HHIIIIIHHH", ">IIIIIIII", (0, 1, 4)), # 32-bit MSB. - (2, 1): ("<HHIQQQIHHH", "<IIQQQQQQ", (0, 2, 5)), # 64-bit LSB. - (2, 2): (">HHIQQQIHHH", ">IIQQQQQQ", (0, 2, 5)), # 64-bit MSB. - }[(self.capacity, self.encoding)] - except KeyError: - raise ELFInvalid( - f"unrecognized capacity ({self.capacity}) or " - f"encoding ({self.encoding})" - ) - - try: - ( - _, - self.machine, # Architecture type. - _, - _, - self._e_phoff, # Offset of program header. - _, - self.flags, # Processor-specific flags. - _, - self._e_phentsize, # Size of section. - self._e_phnum, # Number of sections. - ) = self._read(e_fmt) - except struct.error as e: - raise ELFInvalid("unable to parse machine and section information") from e - - def _read(self, fmt: str) -> Tuple[int, ...]: - return struct.unpack(fmt, self._f.read(struct.calcsize(fmt))) - - @property - def interpreter(self) -> Optional[str]: - """ - The path recorded in the ``PT_INTERP`` section header. - """ - for index in range(self._e_phnum): - self._f.seek(self._e_phoff + self._e_phentsize * index) - try: - data = self._read(self._p_fmt) - except struct.error: - continue - if data[self._p_idx[0]] != 3: # Not PT_INTERP. - continue - self._f.seek(data[self._p_idx[1]]) - return os.fsdecode(self._f.read(data[self._p_idx[2]])).strip("\0") - return None diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py deleted file mode 100644 index 86419df9d70..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_musllinux.py +++ /dev/null @@ -1,83 +0,0 @@ -"""PEP 656 support. - -This module implements logic to detect if the currently running Python is -linked against musl, and what musl version is used. -""" - -import functools -import re -import subprocess -import sys -from typing import Iterator, NamedTuple, Optional, Sequence - -from ._elffile import ELFFile - - -class _MuslVersion(NamedTuple): - major: int - minor: int - - -def _parse_musl_version(output: str) -> Optional[_MuslVersion]: - lines = [n for n in (n.strip() for n in output.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) - - -@functools.lru_cache() -def _get_musl_version(executable: str) -> Optional[_MuslVersion]: - """Detect currently-running musl runtime version. - - This is done by checking the specified executable's dynamic linking - information, and invoking the loader to parse its output for a version - string. If the loader is musl, the output would be something like:: - - musl libc (x86_64) - Version 1.2.2 - Dynamic Program Loader - """ - try: - with open(executable, "rb") as f: - ld = ELFFile(f).interpreter - except (OSError, TypeError, ValueError): - return None - if ld is None or "musl" not in ld: - return None - proc = subprocess.run([ld], stderr=subprocess.PIPE, text=True) - return _parse_musl_version(proc.stderr) - - -def platform_tags(archs: Sequence[str]) -> Iterator[str]: - """Generate musllinux tags compatible to the current platform. - - :param archs: Sequence of compatible architectures. - The first one shall be the closest to the actual architecture and be the part of - platform tag after the ``linux_`` prefix, e.g. ``x86_64``. - The ``linux_`` prefix is assumed as a prerequisite for the current platform to - be musllinux-compatible. - - :returns: An iterator of compatible musllinux tags. - """ - sys_musl = _get_musl_version(sys.executable) - if sys_musl is None: # Python not dynamically linked against musl. - return - for arch in archs: - for minor in range(sys_musl.minor, -1, -1): - yield f"musllinux_{sys_musl.major}_{minor}_{arch}" - - -if __name__ == "__main__": # pragma: no cover - import sysconfig - - plat = sysconfig.get_platform() - assert plat.startswith("linux-"), "not linux" - - print("plat:", plat) - print("musl:", _get_musl_version(sys.executable)) - print("tags:", end=" ") - for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])): - print(t, end="\n ") diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py deleted file mode 100644 index 90a6465f968..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_structures.py +++ /dev/null @@ -1,61 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - - -class InfinityType: - def __repr__(self) -> str: - return "Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return False - - def __le__(self, other: object) -> bool: - return False - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return True - - def __ge__(self, other: object) -> bool: - return True - - def __neg__(self: object) -> "NegativeInfinityType": - return NegativeInfinity - - -Infinity = InfinityType() - - -class NegativeInfinityType: - def __repr__(self) -> str: - return "-Infinity" - - def __hash__(self) -> int: - return hash(repr(self)) - - def __lt__(self, other: object) -> bool: - return True - - def __le__(self, other: object) -> bool: - return True - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) - - def __gt__(self, other: object) -> bool: - return False - - def __ge__(self, other: object) -> bool: - return False - - def __neg__(self: object) -> InfinityType: - return Infinity - - -NegativeInfinity = NegativeInfinityType() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py deleted file mode 100644 index dd0d648d49a..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_tokenizer.py +++ /dev/null @@ -1,192 +0,0 @@ -import contextlib -import re -from dataclasses import dataclass -from typing import Dict, Iterator, NoReturn, Optional, Tuple, Union - -from .specifiers import Specifier - - -@dataclass -class Token: - name: str - text: str - position: int - - -class ParserSyntaxError(Exception): - """The provided source text could not be parsed correctly.""" - - def __init__( - self, - message: str, - *, - source: str, - span: Tuple[int, int], - ) -> None: - self.span = span - self.message = message - self.source = source - - super().__init__() - - def __str__(self) -> str: - marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" - return "\n ".join([self.message, self.source, marker]) - - -DEFAULT_RULES: "Dict[str, Union[str, re.Pattern[str]]]" = { - "LEFT_PARENTHESIS": r"\(", - "RIGHT_PARENTHESIS": r"\)", - "LEFT_BRACKET": r"\[", - "RIGHT_BRACKET": r"\]", - "SEMICOLON": r";", - "COMMA": r",", - "QUOTED_STRING": re.compile( - r""" - ( - ('[^']*') - | - ("[^"]*") - ) - """, - re.VERBOSE, - ), - "OP": r"(===|==|~=|!=|<=|>=|<|>)", - "BOOLOP": r"\b(or|and)\b", - "IN": r"\bin\b", - "NOT": r"\bnot\b", - "VARIABLE": re.compile( - r""" - \b( - python_version - |python_full_version - |os[._]name - |sys[._]platform - |platform_(release|system) - |platform[._](version|machine|python_implementation) - |python_implementation - |implementation_(name|version) - |extra - )\b - """, - re.VERBOSE, - ), - "SPECIFIER": re.compile( - Specifier._operator_regex_str + Specifier._version_regex_str, - re.VERBOSE | re.IGNORECASE, - ), - "AT": r"\@", - "URL": r"[^ \t]+", - "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", - "VERSION_PREFIX_TRAIL": r"\.\*", - "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", - "WS": r"[ \t]+", - "END": r"$", -} - - -class Tokenizer: - """Context-sensitive token parsing. - - Provides methods to examine the input stream to check whether the next token - matches. - """ - - def __init__( - self, - source: str, - *, - rules: "Dict[str, Union[str, re.Pattern[str]]]", - ) -> None: - self.source = source - self.rules: Dict[str, re.Pattern[str]] = { - name: re.compile(pattern) for name, pattern in rules.items() - } - self.next_token: Optional[Token] = None - self.position = 0 - - def consume(self, name: str) -> None: - """Move beyond provided token name, if at current position.""" - if self.check(name): - self.read() - - def check(self, name: str, *, peek: bool = False) -> bool: - """Check whether the next token has the provided name. - - By default, if the check succeeds, the token *must* be read before - another check. If `peek` is set to `True`, the token is not loaded and - would need to be checked again. - """ - assert ( - self.next_token is None - ), f"Cannot check for {name!r}, already have {self.next_token!r}" - assert name in self.rules, f"Unknown token name: {name!r}" - - expression = self.rules[name] - - match = expression.match(self.source, self.position) - if match is None: - return False - if not peek: - self.next_token = Token(name, match[0], self.position) - return True - - def expect(self, name: str, *, expected: str) -> Token: - """Expect a certain token name next, failing with a syntax error otherwise. - - The token is *not* read. - """ - if not self.check(name): - raise self.raise_syntax_error(f"Expected {expected}") - return self.read() - - def read(self) -> Token: - """Consume the next token and return it.""" - token = self.next_token - assert token is not None - - self.position += len(token.text) - self.next_token = None - - return token - - def raise_syntax_error( - self, - message: str, - *, - span_start: Optional[int] = None, - span_end: Optional[int] = None, - ) -> NoReturn: - """Raise ParserSyntaxError at the given position.""" - span = ( - self.position if span_start is None else span_start, - self.position if span_end is None else span_end, - ) - raise ParserSyntaxError( - message, - source=self.source, - span=span, - ) - - @contextlib.contextmanager - def enclosing_tokens( - self, open_token: str, close_token: str, *, around: str - ) -> Iterator[None]: - if self.check(open_token): - open_position = self.position - self.read() - else: - open_position = None - - yield - - if open_position is None: - return - - if not self.check(close_token): - self.raise_syntax_error( - f"Expected matching {close_token} for {open_token}, after {around}", - span_start=open_position, - ) - - self.read() diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py deleted file mode 100644 index fb274930799..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/metadata.py +++ /dev/null @@ -1,825 +0,0 @@ -import email.feedparser -import email.header -import email.message -import email.parser -import email.policy -import sys -import typing -from typing import ( - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - Union, - cast, -) - -from . import requirements, specifiers, utils, version as version_module - -T = typing.TypeVar("T") -if sys.version_info[:2] >= (3, 8): # pragma: no cover - from typing import Literal, TypedDict -else: # pragma: no cover - if typing.TYPE_CHECKING: - from typing_extensions import Literal, TypedDict - else: - try: - from typing_extensions import Literal, TypedDict - except ImportError: - - class Literal: - def __init_subclass__(*_args, **_kwargs): - pass - - class TypedDict: - def __init_subclass__(*_args, **_kwargs): - pass - - -try: - ExceptionGroup -except NameError: # pragma: no cover - - class ExceptionGroup(Exception): # noqa: N818 - """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. - - If :external:exc:`ExceptionGroup` is already defined by Python itself, - that version is used instead. - """ - - message: str - exceptions: List[Exception] - - def __init__(self, message: str, exceptions: List[Exception]) -> None: - self.message = message - self.exceptions = exceptions - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" - -else: # pragma: no cover - ExceptionGroup = ExceptionGroup - - -class InvalidMetadata(ValueError): - """A metadata field contains invalid data.""" - - field: str - """The name of the field that contains invalid data.""" - - def __init__(self, field: str, message: str) -> None: - self.field = field - super().__init__(message) - - -# The RawMetadata class attempts to make as few assumptions about the underlying -# serialization formats as possible. The idea is that as long as a serialization -# formats offer some very basic primitives in *some* way then we can support -# serializing to and from that format. -class RawMetadata(TypedDict, total=False): - """A dictionary of raw core metadata. - - Each field in core metadata maps to a key of this dictionary (when data is - provided). The key is lower-case and underscores are used instead of dashes - compared to the equivalent core metadata field. Any core metadata field that - can be specified multiple times or can hold multiple values in a single - field have a key with a plural name. See :class:`Metadata` whose attributes - match the keys of this dictionary. - - Core metadata fields that can be specified multiple times are stored as a - list or dict depending on which is appropriate for the field. Any fields - which hold multiple values in a single field are stored as a list. - - """ - - # Metadata 1.0 - PEP 241 - metadata_version: str - name: str - version: str - platforms: List[str] - summary: str - description: str - keywords: List[str] - home_page: str - author: str - author_email: str - license: str - - # Metadata 1.1 - PEP 314 - supported_platforms: List[str] - download_url: str - classifiers: List[str] - requires: List[str] - provides: List[str] - obsoletes: List[str] - - # Metadata 1.2 - PEP 345 - maintainer: str - maintainer_email: str - requires_dist: List[str] - provides_dist: List[str] - obsoletes_dist: List[str] - requires_python: str - requires_external: List[str] - project_urls: Dict[str, str] - - # Metadata 2.0 - # PEP 426 attempted to completely revamp the metadata format - # but got stuck without ever being able to build consensus on - # it and ultimately ended up withdrawn. - # - # However, a number of tools had started emitting METADATA with - # `2.0` Metadata-Version, so for historical reasons, this version - # was skipped. - - # Metadata 2.1 - PEP 566 - description_content_type: str - provides_extra: List[str] - - # Metadata 2.2 - PEP 643 - dynamic: List[str] - - # Metadata 2.3 - PEP 685 - # No new fields were added in PEP 685, just some edge case were - # tightened up to provide better interoptability. - - -_STRING_FIELDS = { - "author", - "author_email", - "description", - "description_content_type", - "download_url", - "home_page", - "license", - "maintainer", - "maintainer_email", - "metadata_version", - "name", - "requires_python", - "summary", - "version", -} - -_LIST_FIELDS = { - "classifiers", - "dynamic", - "obsoletes", - "obsoletes_dist", - "platforms", - "provides", - "provides_dist", - "provides_extra", - "requires", - "requires_dist", - "requires_external", - "supported_platforms", -} - -_DICT_FIELDS = { - "project_urls", -} - - -def _parse_keywords(data: str) -> List[str]: - """Split a string of comma-separate keyboards into a list of keywords.""" - return [k.strip() for k in data.split(",")] - - -def _parse_project_urls(data: List[str]) -> Dict[str, str]: - """Parse a list of label/URL string pairings separated by a comma.""" - urls = {} - for pair in data: - # Our logic is slightly tricky here as we want to try and do - # *something* reasonable with malformed data. - # - # The main thing that we have to worry about, is data that does - # not have a ',' at all to split the label from the Value. There - # isn't a singular right answer here, and we will fail validation - # later on (if the caller is validating) so it doesn't *really* - # matter, but since the missing value has to be an empty str - # and our return value is dict[str, str], if we let the key - # be the missing value, then they'd have multiple '' values that - # overwrite each other in a accumulating dict. - # - # The other potentional issue is that it's possible to have the - # same label multiple times in the metadata, with no solid "right" - # answer with what to do in that case. As such, we'll do the only - # thing we can, which is treat the field as unparseable and add it - # to our list of unparsed fields. - parts = [p.strip() for p in pair.split(",", 1)] - parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items - - # TODO: The spec doesn't say anything about if the keys should be - # considered case sensitive or not... logically they should - # be case-preserving and case-insensitive, but doing that - # would open up more cases where we might have duplicate - # entries. - label, url = parts - if label in urls: - # The label already exists in our set of urls, so this field - # is unparseable, and we can just add the whole thing to our - # unparseable data and stop processing it. - raise KeyError("duplicate labels in project urls") - urls[label] = url - - return urls - - -def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: - """Get the body of the message.""" - # If our source is a str, then our caller has managed encodings for us, - # and we don't need to deal with it. - if isinstance(source, str): - payload: str = msg.get_payload() - return payload - # If our source is a bytes, then we're managing the encoding and we need - # to deal with it. - else: - bpayload: bytes = msg.get_payload(decode=True) - try: - return bpayload.decode("utf8", "strict") - except UnicodeDecodeError: - raise ValueError("payload in an invalid encoding") - - -# The various parse_FORMAT functions here are intended to be as lenient as -# possible in their parsing, while still returning a correctly typed -# RawMetadata. -# -# To aid in this, we also generally want to do as little touching of the -# data as possible, except where there are possibly some historic holdovers -# that make valid data awkward to work with. -# -# While this is a lower level, intermediate format than our ``Metadata`` -# class, some light touch ups can make a massive difference in usability. - -# Map METADATA fields to RawMetadata. -_EMAIL_TO_RAW_MAPPING = { - "author": "author", - "author-email": "author_email", - "classifier": "classifiers", - "description": "description", - "description-content-type": "description_content_type", - "download-url": "download_url", - "dynamic": "dynamic", - "home-page": "home_page", - "keywords": "keywords", - "license": "license", - "maintainer": "maintainer", - "maintainer-email": "maintainer_email", - "metadata-version": "metadata_version", - "name": "name", - "obsoletes": "obsoletes", - "obsoletes-dist": "obsoletes_dist", - "platform": "platforms", - "project-url": "project_urls", - "provides": "provides", - "provides-dist": "provides_dist", - "provides-extra": "provides_extra", - "requires": "requires", - "requires-dist": "requires_dist", - "requires-external": "requires_external", - "requires-python": "requires_python", - "summary": "summary", - "supported-platform": "supported_platforms", - "version": "version", -} -_RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()} - - -def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: - """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``). - - This function returns a two-item tuple of dicts. The first dict is of - recognized fields from the core metadata specification. Fields that can be - parsed and translated into Python's built-in types are converted - appropriately. All other fields are left as-is. Fields that are allowed to - appear multiple times are stored as lists. - - The second dict contains all other fields from the metadata. This includes - any unrecognized fields. It also includes any fields which are expected to - be parsed into a built-in type but were not formatted appropriately. Finally, - any fields that are expected to appear only once but are repeated are - included in this dict. - - """ - raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} - unparsed: Dict[str, List[str]] = {} - - if isinstance(data, str): - parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) - else: - parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) - - # We have to wrap parsed.keys() in a set, because in the case of multiple - # values for a key (a list), the key will appear multiple times in the - # list of keys, but we're avoiding that by using get_all(). - for name in frozenset(parsed.keys()): - # Header names in RFC are case insensitive, so we'll normalize to all - # lower case to make comparisons easier. - name = name.lower() - - # We use get_all() here, even for fields that aren't multiple use, - # because otherwise someone could have e.g. two Name fields, and we - # would just silently ignore it rather than doing something about it. - headers = parsed.get_all(name) or [] - - # The way the email module works when parsing bytes is that it - # unconditionally decodes the bytes as ascii using the surrogateescape - # handler. When you pull that data back out (such as with get_all() ), - # it looks to see if the str has any surrogate escapes, and if it does - # it wraps it in a Header object instead of returning the string. - # - # As such, we'll look for those Header objects, and fix up the encoding. - value = [] - # Flag if we have run into any issues processing the headers, thus - # signalling that the data belongs in 'unparsed'. - valid_encoding = True - for h in headers: - # It's unclear if this can return more types than just a Header or - # a str, so we'll just assert here to make sure. - assert isinstance(h, (email.header.Header, str)) - - # If it's a header object, we need to do our little dance to get - # the real data out of it. In cases where there is invalid data - # we're going to end up with mojibake, but there's no obvious, good - # way around that without reimplementing parts of the Header object - # ourselves. - # - # That should be fine since, if mojibacked happens, this key is - # going into the unparsed dict anyways. - if isinstance(h, email.header.Header): - # The Header object stores it's data as chunks, and each chunk - # can be independently encoded, so we'll need to check each - # of them. - chunks: List[Tuple[bytes, Optional[str]]] = [] - for bin, encoding in email.header.decode_header(h): - try: - bin.decode("utf8", "strict") - except UnicodeDecodeError: - # Enable mojibake. - encoding = "latin1" - valid_encoding = False - else: - encoding = "utf8" - chunks.append((bin, encoding)) - - # Turn our chunks back into a Header object, then let that - # Header object do the right thing to turn them into a - # string for us. - value.append(str(email.header.make_header(chunks))) - # This is already a string, so just add it. - else: - value.append(h) - - # We've processed all of our values to get them into a list of str, - # but we may have mojibake data, in which case this is an unparsed - # field. - if not valid_encoding: - unparsed[name] = value - continue - - raw_name = _EMAIL_TO_RAW_MAPPING.get(name) - if raw_name is None: - # This is a bit of a weird situation, we've encountered a key that - # we don't know what it means, so we don't know whether it's meant - # to be a list or not. - # - # Since we can't really tell one way or another, we'll just leave it - # as a list, even though it may be a single item list, because that's - # what makes the most sense for email headers. - unparsed[name] = value - continue - - # If this is one of our string fields, then we'll check to see if our - # value is a list of a single item. If it is then we'll assume that - # it was emitted as a single string, and unwrap the str from inside - # the list. - # - # If it's any other kind of data, then we haven't the faintest clue - # what we should parse it as, and we have to just add it to our list - # of unparsed stuff. - if raw_name in _STRING_FIELDS and len(value) == 1: - raw[raw_name] = value[0] - # If this is one of our list of string fields, then we can just assign - # the value, since email *only* has strings, and our get_all() call - # above ensures that this is a list. - elif raw_name in _LIST_FIELDS: - raw[raw_name] = value - # Special Case: Keywords - # The keywords field is implemented in the metadata spec as a str, - # but it conceptually is a list of strings, and is serialized using - # ", ".join(keywords), so we'll do some light data massaging to turn - # this into what it logically is. - elif raw_name == "keywords" and len(value) == 1: - raw[raw_name] = _parse_keywords(value[0]) - # Special Case: Project-URL - # The project urls is implemented in the metadata spec as a list of - # specially-formatted strings that represent a key and a value, which - # is fundamentally a mapping, however the email format doesn't support - # mappings in a sane way, so it was crammed into a list of strings - # instead. - # - # We will do a little light data massaging to turn this into a map as - # it logically should be. - elif raw_name == "project_urls": - try: - raw[raw_name] = _parse_project_urls(value) - except KeyError: - unparsed[name] = value - # Nothing that we've done has managed to parse this, so it'll just - # throw it in our unparseable data and move on. - else: - unparsed[name] = value - - # We need to support getting the Description from the message payload in - # addition to getting it from the the headers. This does mean, though, there - # is the possibility of it being set both ways, in which case we put both - # in 'unparsed' since we don't know which is right. - try: - payload = _get_payload(parsed, data) - except ValueError: - unparsed.setdefault("description", []).append( - parsed.get_payload(decode=isinstance(data, bytes)) - ) - else: - if payload: - # Check to see if we've already got a description, if so then both - # it, and this body move to unparseable. - if "description" in raw: - description_header = cast(str, raw.pop("description")) - unparsed.setdefault("description", []).extend( - [description_header, payload] - ) - elif "description" in unparsed: - unparsed["description"].append(payload) - else: - raw["description"] = payload - - # We need to cast our `raw` to a metadata, because a TypedDict only support - # literal key names, but we're computing our key names on purpose, but the - # way this function is implemented, our `TypedDict` can only have valid key - # names. - return cast(RawMetadata, raw), unparsed - - -_NOT_FOUND = object() - - -# Keep the two values in sync. -_VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] -_MetadataVersion = Literal["1.0", "1.1", "1.2", "2.1", "2.2", "2.3"] - -_REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"]) - - -class _Validator(Generic[T]): - """Validate a metadata field. - - All _process_*() methods correspond to a core metadata field. The method is - called with the field's raw value. If the raw value is valid it is returned - in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field). - If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause - as appropriate). - """ - - name: str - raw_name: str - added: _MetadataVersion - - def __init__( - self, - *, - added: _MetadataVersion = "1.0", - ) -> None: - self.added = added - - def __set_name__(self, _owner: "Metadata", name: str) -> None: - self.name = name - self.raw_name = _RAW_TO_EMAIL_MAPPING[name] - - def __get__(self, instance: "Metadata", _owner: Type["Metadata"]) -> T: - # With Python 3.8, the caching can be replaced with functools.cached_property(). - # No need to check the cache as attribute lookup will resolve into the - # instance's __dict__ before __get__ is called. - cache = instance.__dict__ - value = instance._raw.get(self.name) - - # To make the _process_* methods easier, we'll check if the value is None - # and if this field is NOT a required attribute, and if both of those - # things are true, we'll skip the the converter. This will mean that the - # converters never have to deal with the None union. - if self.name in _REQUIRED_ATTRS or value is not None: - try: - converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}") - except AttributeError: - pass - else: - value = converter(value) - - cache[self.name] = value - try: - del instance._raw[self.name] # type: ignore[misc] - except KeyError: - pass - - return cast(T, value) - - def _invalid_metadata( - self, msg: str, cause: Optional[Exception] = None - ) -> InvalidMetadata: - exc = InvalidMetadata( - self.raw_name, msg.format_map({"field": repr(self.raw_name)}) - ) - exc.__cause__ = cause - return exc - - def _process_metadata_version(self, value: str) -> _MetadataVersion: - # Implicitly makes Metadata-Version required. - if value not in _VALID_METADATA_VERSIONS: - raise self._invalid_metadata(f"{value!r} is not a valid metadata version") - return cast(_MetadataVersion, value) - - def _process_name(self, value: str) -> str: - if not value: - raise self._invalid_metadata("{field} is a required field") - # Validate the name as a side-effect. - try: - utils.canonicalize_name(value, validate=True) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - else: - return value - - def _process_version(self, value: str) -> version_module.Version: - if not value: - raise self._invalid_metadata("{field} is a required field") - try: - return version_module.parse(value) - except version_module.InvalidVersion as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_summary(self, value: str) -> str: - """Check the field contains no newlines.""" - if "\n" in value: - raise self._invalid_metadata("{field} must be a single line") - return value - - def _process_description_content_type(self, value: str) -> str: - content_types = {"text/plain", "text/x-rst", "text/markdown"} - message = email.message.EmailMessage() - message["content-type"] = value - - content_type, parameters = ( - # Defaults to `text/plain` if parsing failed. - message.get_content_type().lower(), - message["content-type"].params, - ) - # Check if content-type is valid or defaulted to `text/plain` and thus was - # not parseable. - if content_type not in content_types or content_type not in value.lower(): - raise self._invalid_metadata( - f"{{field}} must be one of {list(content_types)}, not {value!r}" - ) - - charset = parameters.get("charset", "UTF-8") - if charset != "UTF-8": - raise self._invalid_metadata( - f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" - ) - - markdown_variants = {"GFM", "CommonMark"} - variant = parameters.get("variant", "GFM") # Use an acceptable default. - if content_type == "text/markdown" and variant not in markdown_variants: - raise self._invalid_metadata( - f"valid Markdown variants for {{field}} are {list(markdown_variants)}, " - f"not {variant!r}", - ) - return value - - def _process_dynamic(self, value: List[str]) -> List[str]: - for dynamic_field in map(str.lower, value): - if dynamic_field in {"name", "version", "metadata-version"}: - raise self._invalid_metadata( - f"{value!r} is not allowed as a dynamic field" - ) - elif dynamic_field not in _EMAIL_TO_RAW_MAPPING: - raise self._invalid_metadata(f"{value!r} is not a valid dynamic field") - return list(map(str.lower, value)) - - def _process_provides_extra( - self, - value: List[str], - ) -> List[utils.NormalizedName]: - normalized_names = [] - try: - for name in value: - normalized_names.append(utils.canonicalize_name(name, validate=True)) - except utils.InvalidName as exc: - raise self._invalid_metadata( - f"{name!r} is invalid for {{field}}", cause=exc - ) - else: - return normalized_names - - def _process_requires_python(self, value: str) -> specifiers.SpecifierSet: - try: - return specifiers.SpecifierSet(value) - except specifiers.InvalidSpecifier as exc: - raise self._invalid_metadata( - f"{value!r} is invalid for {{field}}", cause=exc - ) - - def _process_requires_dist( - self, - value: List[str], - ) -> List[requirements.Requirement]: - reqs = [] - try: - for req in value: - reqs.append(requirements.Requirement(req)) - except requirements.InvalidRequirement as exc: - raise self._invalid_metadata(f"{req!r} is invalid for {{field}}", cause=exc) - else: - return reqs - - -class Metadata: - """Representation of distribution metadata. - - Compared to :class:`RawMetadata`, this class provides objects representing - metadata fields instead of only using built-in types. Any invalid metadata - will cause :exc:`InvalidMetadata` to be raised (with a - :py:attr:`~BaseException.__cause__` attribute as appropriate). - """ - - _raw: RawMetadata - - @classmethod - def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> "Metadata": - """Create an instance from :class:`RawMetadata`. - - If *validate* is true, all metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - ins = cls() - ins._raw = data.copy() # Mutations occur due to caching enriched values. - - if validate: - exceptions: List[Exception] = [] - try: - metadata_version = ins.metadata_version - metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) - except InvalidMetadata as metadata_version_exc: - exceptions.append(metadata_version_exc) - metadata_version = None - - # Make sure to check for the fields that are present, the required - # fields (so their absence can be reported). - fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS - # Remove fields that have already been checked. - fields_to_check -= {"metadata_version"} - - for key in fields_to_check: - try: - if metadata_version: - # Can't use getattr() as that triggers descriptor protocol which - # will fail due to no value for the instance argument. - try: - field_metadata_version = cls.__dict__[key].added - except KeyError: - exc = InvalidMetadata(key, f"unrecognized field: {key!r}") - exceptions.append(exc) - continue - field_age = _VALID_METADATA_VERSIONS.index( - field_metadata_version - ) - if field_age > metadata_age: - field = _RAW_TO_EMAIL_MAPPING[key] - exc = InvalidMetadata( - field, - "{field} introduced in metadata version " - "{field_metadata_version}, not {metadata_version}", - ) - exceptions.append(exc) - continue - getattr(ins, key) - except InvalidMetadata as exc: - exceptions.append(exc) - - if exceptions: - raise ExceptionGroup("invalid metadata", exceptions) - - return ins - - @classmethod - def from_email( - cls, data: Union[bytes, str], *, validate: bool = True - ) -> "Metadata": - """Parse metadata from email headers. - - If *validate* is true, the metadata will be validated. All exceptions - related to validation will be gathered and raised as an :class:`ExceptionGroup`. - """ - raw, unparsed = parse_email(data) - - if validate: - exceptions: list[Exception] = [] - for unparsed_key in unparsed: - if unparsed_key in _EMAIL_TO_RAW_MAPPING: - message = f"{unparsed_key!r} has invalid data" - else: - message = f"unrecognized field: {unparsed_key!r}" - exceptions.append(InvalidMetadata(unparsed_key, message)) - - if exceptions: - raise ExceptionGroup("unparsed", exceptions) - - try: - return cls.from_raw(raw, validate=validate) - except ExceptionGroup as exc_group: - raise ExceptionGroup( - "invalid or unparsed metadata", exc_group.exceptions - ) from None - - metadata_version: _Validator[_MetadataVersion] = _Validator() - """:external:ref:`core-metadata-metadata-version` - (required; validated to be a valid metadata version)""" - name: _Validator[str] = _Validator() - """:external:ref:`core-metadata-name` - (required; validated using :func:`~packaging.utils.canonicalize_name` and its - *validate* parameter)""" - version: _Validator[version_module.Version] = _Validator() - """:external:ref:`core-metadata-version` (required)""" - dynamic: _Validator[Optional[List[str]]] = _Validator( - added="2.2", - ) - """:external:ref:`core-metadata-dynamic` - (validated against core metadata field names and lowercased)""" - platforms: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-platform`""" - supported_platforms: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-supported-platform`""" - summary: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-summary` (validated to contain no newlines)""" - description: _Validator[Optional[str]] = _Validator() # TODO 2.1: can be in body - """:external:ref:`core-metadata-description`""" - description_content_type: _Validator[Optional[str]] = _Validator(added="2.1") - """:external:ref:`core-metadata-description-content-type` (validated)""" - keywords: _Validator[Optional[List[str]]] = _Validator() - """:external:ref:`core-metadata-keywords`""" - home_page: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-home-page`""" - download_url: _Validator[Optional[str]] = _Validator(added="1.1") - """:external:ref:`core-metadata-download-url`""" - author: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author`""" - author_email: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-author-email`""" - maintainer: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer`""" - maintainer_email: _Validator[Optional[str]] = _Validator(added="1.2") - """:external:ref:`core-metadata-maintainer-email`""" - license: _Validator[Optional[str]] = _Validator() - """:external:ref:`core-metadata-license`""" - classifiers: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """:external:ref:`core-metadata-classifier`""" - requires_dist: _Validator[Optional[List[requirements.Requirement]]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-dist`""" - requires_python: _Validator[Optional[specifiers.SpecifierSet]] = _Validator( - added="1.2" - ) - """:external:ref:`core-metadata-requires-python`""" - # Because `Requires-External` allows for non-PEP 440 version specifiers, we - # don't do any processing on the values. - requires_external: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-requires-external`""" - project_urls: _Validator[Optional[Dict[str, str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-project-url`""" - # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation - # regardless of metadata version. - provides_extra: _Validator[Optional[List[utils.NormalizedName]]] = _Validator( - added="2.1", - ) - """:external:ref:`core-metadata-provides-extra`""" - provides_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-provides-dist`""" - obsoletes_dist: _Validator[Optional[List[str]]] = _Validator(added="1.2") - """:external:ref:`core-metadata-obsoletes-dist`""" - requires: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Requires`` (deprecated)""" - provides: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Provides`` (deprecated)""" - obsoletes: _Validator[Optional[List[str]]] = _Validator(added="1.1") - """``Obsoletes`` (deprecated)""" diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/py.typed +++ /dev/null diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py deleted file mode 100644 index bdc43a7e98d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/requirements.py +++ /dev/null @@ -1,90 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from typing import Any, Iterator, Optional, Set - -from ._parser import parse_requirement as _parse_requirement -from ._tokenizer import ParserSyntaxError -from .markers import Marker, _normalize_extra_values -from .specifiers import SpecifierSet -from .utils import canonicalize_name - - -class InvalidRequirement(ValueError): - """ - An invalid requirement was found, users should refer to PEP 508. - """ - - -class Requirement: - """Parse a requirement. - - Parse a given requirement string into its parts, such as name, specifier, - URL, and extras. Raises InvalidRequirement on a badly-formed requirement - string. - """ - - # TODO: Can we test whether something is contained within a requirement? - # If so how do we do that? Do we need to test against the _name_ of - # the thing as well as the version? What about the markers? - # TODO: Can we normalize the name and extra name? - - def __init__(self, requirement_string: str) -> None: - try: - parsed = _parse_requirement(requirement_string) - except ParserSyntaxError as e: - raise InvalidRequirement(str(e)) from e - - self.name: str = parsed.name - self.url: Optional[str] = parsed.url or None - self.extras: Set[str] = set(parsed.extras or []) - self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) - self.marker: Optional[Marker] = None - if parsed.marker is not None: - self.marker = Marker.__new__(Marker) - self.marker._markers = _normalize_extra_values(parsed.marker) - - def _iter_parts(self, name: str) -> Iterator[str]: - yield name - - if self.extras: - formatted_extras = ",".join(sorted(self.extras)) - yield f"[{formatted_extras}]" - - if self.specifier: - yield str(self.specifier) - - if self.url: - yield f"@ {self.url}" - if self.marker: - yield " " - - if self.marker: - yield f"; {self.marker}" - - def __str__(self) -> str: - return "".join(self._iter_parts(self.name)) - - def __repr__(self) -> str: - return f"<Requirement('{self}')>" - - def __hash__(self) -> int: - return hash( - ( - self.__class__.__name__, - *self._iter_parts(canonicalize_name(self.name)), - ) - ) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Requirement): - return NotImplemented - - return ( - canonicalize_name(self.name) == canonicalize_name(other.name) - and self.extras == other.extras - and self.specifier == other.specifier - and self.url == other.url - and self.marker == other.marker - ) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py deleted file mode 100644 index 89f1926137d..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/tags.py +++ /dev/null @@ -1,571 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import logging -import platform -import re -import struct -import subprocess -import sys -import sysconfig -from importlib.machinery import EXTENSION_SUFFIXES -from typing import ( - Dict, - FrozenSet, - Iterable, - Iterator, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) - -from . import _manylinux, _musllinux - -logger = logging.getLogger(__name__) - -PythonVersion = Sequence[int] -MacVersion = Tuple[int, int] - -INTERPRETER_SHORT_NAMES: Dict[str, str] = { - "python": "py", # Generic. - "cpython": "cp", - "pypy": "pp", - "ironpython": "ip", - "jython": "jy", -} - - -_32_BIT_INTERPRETER = struct.calcsize("P") == 4 - - -class Tag: - """ - A representation of the tag triple for a wheel. - - Instances are considered immutable and thus are hashable. Equality checking - is also supported. - """ - - __slots__ = ["_interpreter", "_abi", "_platform", "_hash"] - - def __init__(self, interpreter: str, abi: str, platform: str) -> None: - self._interpreter = interpreter.lower() - self._abi = abi.lower() - self._platform = platform.lower() - # The __hash__ of every single element in a Set[Tag] will be evaluated each time - # that a set calls its `.disjoint()` method, which may be called hundreds of - # times when scanning a page of links for packages with tags matching that - # Set[Tag]. Pre-computing the value here produces significant speedups for - # downstream consumers. - self._hash = hash((self._interpreter, self._abi, self._platform)) - - @property - def interpreter(self) -> str: - return self._interpreter - - @property - def abi(self) -> str: - return self._abi - - @property - def platform(self) -> str: - return self._platform - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Tag): - return NotImplemented - - return ( - (self._hash == other._hash) # Short-circuit ASAP for perf reasons. - and (self._platform == other._platform) - and (self._abi == other._abi) - and (self._interpreter == other._interpreter) - ) - - def __hash__(self) -> int: - return self._hash - - def __str__(self) -> str: - return f"{self._interpreter}-{self._abi}-{self._platform}" - - def __repr__(self) -> str: - return f"<{self} @ {id(self)}>" - - -def parse_tag(tag: str) -> FrozenSet[Tag]: - """ - Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. - - Returning a set is required due to the possibility that the tag is a - compressed tag set. - """ - tags = set() - interpreters, abis, platforms = tag.split("-") - for interpreter in interpreters.split("."): - for abi in abis.split("."): - for platform_ in platforms.split("."): - tags.add(Tag(interpreter, abi, platform_)) - return frozenset(tags) - - -def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]: - value: Union[int, str, None] = sysconfig.get_config_var(name) - if value is None and warn: - logger.debug( - "Config variable '%s' is unset, Python ABI tag may be incorrect", name - ) - return value - - -def _normalize_string(string: str) -> str: - return string.replace(".", "_").replace("-", "_").replace(" ", "_") - - -def _is_threaded_cpython(abis: List[str]) -> bool: - """ - Determine if the ABI corresponds to a threaded (`--disable-gil`) build. - - The threaded builds are indicated by a "t" in the abiflags. - """ - if len(abis) == 0: - return False - # expect e.g., cp313 - m = re.match(r"cp\d+(.*)", abis[0]) - if not m: - return False - abiflags = m.group(1) - return "t" in abiflags - - -def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: - """ - Determine if the Python version supports abi3. - - PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) - builds do not support abi3. - """ - return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading - - -def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> List[str]: - py_version = tuple(py_version) # To allow for version comparison. - abis = [] - version = _version_nodot(py_version[:2]) - threading = debug = pymalloc = ucs4 = "" - with_debug = _get_config_var("Py_DEBUG", warn) - has_refcount = hasattr(sys, "gettotalrefcount") - # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled - # extension modules is the best option. - # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 - has_ext = "_d.pyd" in EXTENSION_SUFFIXES - if with_debug or (with_debug is None and (has_refcount or has_ext)): - debug = "d" - if py_version >= (3, 13) and _get_config_var("Py_GIL_DISABLED", warn): - threading = "t" - if py_version < (3, 8): - with_pymalloc = _get_config_var("WITH_PYMALLOC", warn) - if with_pymalloc or with_pymalloc is None: - pymalloc = "m" - if py_version < (3, 3): - unicode_size = _get_config_var("Py_UNICODE_SIZE", warn) - if unicode_size == 4 or ( - unicode_size is None and sys.maxunicode == 0x10FFFF - ): - ucs4 = "u" - elif debug: - # Debug builds can also load "normal" extension modules. - # We can also assume no UCS-4 or pymalloc requirement. - abis.append(f"cp{version}{threading}") - abis.insert(0, f"cp{version}{threading}{debug}{pymalloc}{ucs4}") - return abis - - -def cpython_tags( - python_version: Optional[PythonVersion] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a CPython interpreter. - - The tags consist of: - - cp<python_version>-<abi>-<platform> - - cp<python_version>-abi3-<platform> - - cp<python_version>-none-<platform> - - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. - - If python_version only specifies a major version then user-provided ABIs and - the 'none' ABItag will be used. - - If 'abi3' or 'none' are specified in 'abis' then they will be yielded at - their normal position and not at the beginning. - """ - if not python_version: - python_version = sys.version_info[:2] - - interpreter = f"cp{_version_nodot(python_version[:2])}" - - if abis is None: - if len(python_version) > 1: - abis = _cpython_abis(python_version, warn) - else: - abis = [] - abis = list(abis) - # 'abi3' and 'none' are explicitly handled later. - for explicit_abi in ("abi3", "none"): - try: - abis.remove(explicit_abi) - except ValueError: - pass - - platforms = list(platforms or platform_tags()) - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - threading = _is_threaded_cpython(abis) - use_abi3 = _abi3_applies(python_version, threading) - if use_abi3: - yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) - yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) - - if use_abi3: - for minor_version in range(python_version[1] - 1, 1, -1): - for platform_ in platforms: - interpreter = "cp{version}".format( - version=_version_nodot((python_version[0], minor_version)) - ) - yield Tag(interpreter, "abi3", platform_) - - -def _generic_abi() -> List[str]: - """ - Return the ABI tag based on EXT_SUFFIX. - """ - # The following are examples of `EXT_SUFFIX`. - # We want to keep the parts which are related to the ABI and remove the - # parts which are related to the platform: - # - linux: '.cpython-310-x86_64-linux-gnu.so' => cp310 - # - mac: '.cpython-310-darwin.so' => cp310 - # - win: '.cp310-win_amd64.pyd' => cp310 - # - win: '.pyd' => cp37 (uses _cpython_abis()) - # - pypy: '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73 - # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib' - # => graalpy_38_native - - ext_suffix = _get_config_var("EXT_SUFFIX", warn=True) - if not isinstance(ext_suffix, str) or ext_suffix[0] != ".": - raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')") - parts = ext_suffix.split(".") - if len(parts) < 3: - # CPython3.7 and earlier uses ".pyd" on Windows. - return _cpython_abis(sys.version_info[:2]) - soabi = parts[1] - if soabi.startswith("cpython"): - # non-windows - abi = "cp" + soabi.split("-")[1] - elif soabi.startswith("cp"): - # windows - abi = soabi.split("-")[0] - elif soabi.startswith("pypy"): - abi = "-".join(soabi.split("-")[:2]) - elif soabi.startswith("graalpy"): - abi = "-".join(soabi.split("-")[:3]) - elif soabi: - # pyston, ironpython, others? - abi = soabi - else: - return [] - return [_normalize_string(abi)] - - -def generic_tags( - interpreter: Optional[str] = None, - abis: Optional[Iterable[str]] = None, - platforms: Optional[Iterable[str]] = None, - *, - warn: bool = False, -) -> Iterator[Tag]: - """ - Yields the tags for a generic interpreter. - - The tags consist of: - - <interpreter>-<abi>-<platform> - - The "none" ABI will be added if it was not explicitly provided. - """ - if not interpreter: - interp_name = interpreter_name() - interp_version = interpreter_version(warn=warn) - interpreter = "".join([interp_name, interp_version]) - if abis is None: - abis = _generic_abi() - else: - abis = list(abis) - platforms = list(platforms or platform_tags()) - if "none" not in abis: - abis.append("none") - for abi in abis: - for platform_ in platforms: - yield Tag(interpreter, abi, platform_) - - -def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]: - """ - Yields Python versions in descending order. - - After the latest version, the major-only version will be yielded, and then - all previous versions of that major version. - """ - if len(py_version) > 1: - yield f"py{_version_nodot(py_version[:2])}" - yield f"py{py_version[0]}" - if len(py_version) > 1: - for minor in range(py_version[1] - 1, -1, -1): - yield f"py{_version_nodot((py_version[0], minor))}" - - -def compatible_tags( - python_version: Optional[PythonVersion] = None, - interpreter: Optional[str] = None, - platforms: Optional[Iterable[str]] = None, -) -> Iterator[Tag]: - """ - Yields the sequence of tags that are compatible with a specific version of Python. - - The tags consist of: - - py*-none-<platform> - - <interpreter>-none-any # ... if `interpreter` is provided. - - py*-none-any - """ - if not python_version: - python_version = sys.version_info[:2] - platforms = list(platforms or platform_tags()) - for version in _py_interpreter_range(python_version): - for platform_ in platforms: - yield Tag(version, "none", platform_) - if interpreter: - yield Tag(interpreter, "none", "any") - for version in _py_interpreter_range(python_version): - yield Tag(version, "none", "any") - - -def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str: - if not is_32bit: - return arch - - if arch.startswith("ppc"): - return "ppc" - - return "i386" - - -def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> List[str]: - formats = [cpu_arch] - if cpu_arch == "x86_64": - if version < (10, 4): - return [] - formats.extend(["intel", "fat64", "fat32"]) - - elif cpu_arch == "i386": - if version < (10, 4): - return [] - formats.extend(["intel", "fat32", "fat"]) - - elif cpu_arch == "ppc64": - # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? - if version > (10, 5) or version < (10, 4): - return [] - formats.append("fat64") - - elif cpu_arch == "ppc": - if version > (10, 6): - return [] - formats.extend(["fat32", "fat"]) - - if cpu_arch in {"arm64", "x86_64"}: - formats.append("universal2") - - if cpu_arch in {"x86_64", "i386", "ppc64", "ppc", "intel"}: - formats.append("universal") - - return formats - - -def mac_platforms( - version: Optional[MacVersion] = None, arch: Optional[str] = None -) -> Iterator[str]: - """ - Yields the platform tags for a macOS system. - - The `version` parameter is a two-item tuple specifying the macOS version to - generate platform tags for. The `arch` parameter is the CPU architecture to - generate platform tags for. Both parameters default to the appropriate value - for the current system. - """ - version_str, _, cpu_arch = platform.mac_ver() - if version is None: - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - if version == (10, 16): - # When built against an older macOS SDK, Python will report macOS 10.16 - # instead of the real version. - version_str = subprocess.run( - [ - sys.executable, - "-sS", - "-c", - "import platform; print(platform.mac_ver()[0])", - ], - check=True, - env={"SYSTEM_VERSION_COMPAT": "0"}, - stdout=subprocess.PIPE, - text=True, - ).stdout - version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) - else: - version = version - if arch is None: - arch = _mac_arch(cpu_arch) - else: - arch = arch - - if (10, 0) <= version and version < (11, 0): - # Prior to Mac OS 11, each yearly release of Mac OS bumped the - # "minor" version number. The major version was always 10. - for minor_version in range(version[1], -1, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=10, minor=minor_version, binary_format=binary_format - ) - - if version >= (11, 0): - # Starting with Mac OS 11, each yearly release bumps the major version - # number. The minor versions are now the midyear updates. - for major_version in range(version[0], 10, -1): - compat_version = major_version, 0 - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=major_version, minor=0, binary_format=binary_format - ) - - if version >= (11, 0): - # Mac OS 11 on x86_64 is compatible with binaries from previous releases. - # Arm64 support was introduced in 11.0, so no Arm binaries from previous - # releases exist. - # - # However, the "universal2" binary format can have a - # macOS version earlier than 11.0 when the x86_64 part of the binary supports - # that version of macOS. - if arch == "x86_64": - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_formats = _mac_binary_formats(compat_version, arch) - for binary_format in binary_formats: - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - else: - for minor_version in range(16, 3, -1): - compat_version = 10, minor_version - binary_format = "universal2" - yield "macosx_{major}_{minor}_{binary_format}".format( - major=compat_version[0], - minor=compat_version[1], - binary_format=binary_format, - ) - - -def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: - linux = _normalize_string(sysconfig.get_platform()) - if not linux.startswith("linux_"): - # we should never be here, just yield the sysconfig one and return - yield linux - return - if is_32bit: - if linux == "linux_x86_64": - linux = "linux_i686" - elif linux == "linux_aarch64": - linux = "linux_armv8l" - _, arch = linux.split("_", 1) - archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch]) - yield from _manylinux.platform_tags(archs) - yield from _musllinux.platform_tags(archs) - for arch in archs: - yield f"linux_{arch}" - - -def _generic_platforms() -> Iterator[str]: - yield _normalize_string(sysconfig.get_platform()) - - -def platform_tags() -> Iterator[str]: - """ - Provides the platform tags for this installation. - """ - if platform.system() == "Darwin": - return mac_platforms() - elif platform.system() == "Linux": - return _linux_platforms() - else: - return _generic_platforms() - - -def interpreter_name() -> str: - """ - Returns the name of the running interpreter. - - Some implementations have a reserved, two-letter abbreviation which will - be returned when appropriate. - """ - name = sys.implementation.name - return INTERPRETER_SHORT_NAMES.get(name) or name - - -def interpreter_version(*, warn: bool = False) -> str: - """ - Returns the version of the running interpreter. - """ - version = _get_config_var("py_version_nodot", warn=warn) - if version: - version = str(version) - else: - version = _version_nodot(sys.version_info[:2]) - return version - - -def _version_nodot(version: PythonVersion) -> str: - return "".join(map(str, version)) - - -def sys_tags(*, warn: bool = False) -> Iterator[Tag]: - """ - Returns the sequence of tag triples for the running interpreter. - - The order of the sequence corresponds to priority order for the - interpreter, from most to least important. - """ - - interp_name = interpreter_name() - if interp_name == "cp": - yield from cpython_tags(warn=warn) - else: - yield from generic_tags() - - if interp_name == "pp": - interp = "pp3" - elif interp_name == "cp": - interp = "cp" + interpreter_version(warn=warn) - else: - interp = None - yield from compatible_tags(interpreter=interp) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py deleted file mode 100644 index c2c2f75aa80..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/utils.py +++ /dev/null @@ -1,172 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -import re -from typing import FrozenSet, NewType, Tuple, Union, cast - -from .tags import Tag, parse_tag -from .version import InvalidVersion, Version - -BuildTag = Union[Tuple[()], Tuple[int, str]] -NormalizedName = NewType("NormalizedName", str) - - -class InvalidName(ValueError): - """ - An invalid distribution name; users should refer to the packaging user guide. - """ - - -class InvalidWheelFilename(ValueError): - """ - An invalid wheel filename was found, users should refer to PEP 427. - """ - - -class InvalidSdistFilename(ValueError): - """ - An invalid sdist filename was found, users should refer to the packaging user guide. - """ - - -# Core metadata spec for `Name` -_validate_regex = re.compile( - r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE -) -_canonicalize_regex = re.compile(r"[-_.]+") -_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") -# PEP 427: The build number must start with a digit. -_build_tag_regex = re.compile(r"(\d+)(.*)") - - -def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: - if validate and not _validate_regex.match(name): - raise InvalidName(f"name is invalid: {name!r}") - # This is taken from PEP 503. - value = _canonicalize_regex.sub("-", name).lower() - return cast(NormalizedName, value) - - -def is_normalized_name(name: str) -> bool: - return _normalized_regex.match(name) is not None - - -def canonicalize_version( - version: Union[Version, str], *, strip_trailing_zero: bool = True -) -> str: - """ - This is very similar to Version.__str__, but has one subtle difference - with the way it handles the release segment. - """ - if isinstance(version, str): - try: - parsed = Version(version) - except InvalidVersion: - # Legacy versions cannot be normalized - return version - else: - parsed = version - - parts = [] - - # Epoch - if parsed.epoch != 0: - parts.append(f"{parsed.epoch}!") - - # Release segment - release_segment = ".".join(str(x) for x in parsed.release) - if strip_trailing_zero: - # NB: This strips trailing '.0's to normalize - release_segment = re.sub(r"(\.0)+$", "", release_segment) - parts.append(release_segment) - - # Pre-release - if parsed.pre is not None: - parts.append("".join(str(x) for x in parsed.pre)) - - # Post-release - if parsed.post is not None: - parts.append(f".post{parsed.post}") - - # Development release - if parsed.dev is not None: - parts.append(f".dev{parsed.dev}") - - # Local version segment - if parsed.local is not None: - parts.append(f"+{parsed.local}") - - return "".join(parts) - - -def parse_wheel_filename( - filename: str, -) -> Tuple[NormalizedName, Version, BuildTag, FrozenSet[Tag]]: - if not filename.endswith(".whl"): - raise InvalidWheelFilename( - f"Invalid wheel filename (extension must be '.whl'): {filename}" - ) - - filename = filename[:-4] - dashes = filename.count("-") - if dashes not in (4, 5): - raise InvalidWheelFilename( - f"Invalid wheel filename (wrong number of parts): {filename}" - ) - - parts = filename.split("-", dashes - 2) - name_part = parts[0] - # See PEP 427 for the rules on escaping the project name. - if "__" in name_part or re.match(r"^[\w\d._]*$", name_part, re.UNICODE) is None: - raise InvalidWheelFilename(f"Invalid project name: {filename}") - name = canonicalize_name(name_part) - - try: - version = Version(parts[1]) - except InvalidVersion as e: - raise InvalidWheelFilename( - f"Invalid wheel filename (invalid version): {filename}" - ) from e - - if dashes == 5: - build_part = parts[2] - build_match = _build_tag_regex.match(build_part) - if build_match is None: - raise InvalidWheelFilename( - f"Invalid build number: {build_part} in '{filename}'" - ) - build = cast(BuildTag, (int(build_match.group(1)), build_match.group(2))) - else: - build = () - tags = parse_tag(parts[-1]) - return (name, version, build, tags) - - -def parse_sdist_filename(filename: str) -> Tuple[NormalizedName, Version]: - if filename.endswith(".tar.gz"): - file_stem = filename[: -len(".tar.gz")] - elif filename.endswith(".zip"): - file_stem = filename[: -len(".zip")] - else: - raise InvalidSdistFilename( - f"Invalid sdist filename (extension must be '.tar.gz' or '.zip'):" - f" {filename}" - ) - - # We are requiring a PEP 440 version, which cannot contain dashes, - # so we split on the last dash. - name_part, sep, version_part = file_stem.rpartition("-") - if not sep: - raise InvalidSdistFilename(f"Invalid sdist filename: {filename}") - - name = canonicalize_name(name_part) - - try: - version = Version(version_part) - except InvalidVersion as e: - raise InvalidSdistFilename( - f"Invalid sdist filename (invalid version): {filename}" - ) from e - - return (name, version) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py b/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py deleted file mode 100644 index 5faab9bd0dc..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/version.py +++ /dev/null @@ -1,563 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. -""" -.. testsetup:: - - from packaging.version import parse, Version -""" - -import itertools -import re -from typing import Any, Callable, NamedTuple, Optional, SupportsInt, Tuple, Union - -from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType - -__all__ = ["VERSION_PATTERN", "parse", "Version", "InvalidVersion"] - -LocalType = Tuple[Union[int, str], ...] - -CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] -CmpLocalType = Union[ - NegativeInfinityType, - Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], -] -CmpKey = Tuple[ - int, - Tuple[int, ...], - CmpPrePostDevType, - CmpPrePostDevType, - CmpPrePostDevType, - CmpLocalType, -] -VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] - - -class _Version(NamedTuple): - epoch: int - release: Tuple[int, ...] - dev: Optional[Tuple[str, int]] - pre: Optional[Tuple[str, int]] - post: Optional[Tuple[str, int]] - local: Optional[LocalType] - - -def parse(version: str) -> "Version": - """Parse the given version string. - - >>> parse('1.0.dev1') - <Version('1.0.dev1')> - - :param version: The version string to parse. - :raises InvalidVersion: When the version string is not a valid version. - """ - return Version(version) - - -class InvalidVersion(ValueError): - """Raised when a version string is not a valid version. - - >>> Version("invalid") - Traceback (most recent call last): - ... - packaging.version.InvalidVersion: Invalid version: 'invalid' - """ - - -class _BaseVersion: - _key: Tuple[Any, ...] - - def __hash__(self) -> int: - return hash(self._key) - - # Please keep the duplicated `isinstance` check - # in the six comparisons hereunder - # unless you find a way to avoid adding overhead function calls. - def __lt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key < other._key - - def __le__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key <= other._key - - def __eq__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key == other._key - - def __ge__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key >= other._key - - def __gt__(self, other: "_BaseVersion") -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key > other._key - - def __ne__(self, other: object) -> bool: - if not isinstance(other, _BaseVersion): - return NotImplemented - - return self._key != other._key - - -# Deliberately not anchored to the start and end of the string, to make it -# easier for 3rd party code to reuse -_VERSION_PATTERN = r""" - v? - (?: - (?:(?P<epoch>[0-9]+)!)? # epoch - (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment - (?P<pre> # pre-release - [-_\.]? - (?P<pre_l>alpha|a|beta|b|preview|pre|c|rc) - [-_\.]? - (?P<pre_n>[0-9]+)? - )? - (?P<post> # post release - (?:-(?P<post_n1>[0-9]+)) - | - (?: - [-_\.]? - (?P<post_l>post|rev|r) - [-_\.]? - (?P<post_n2>[0-9]+)? - ) - )? - (?P<dev> # dev release - [-_\.]? - (?P<dev_l>dev) - [-_\.]? - (?P<dev_n>[0-9]+)? - )? - ) - (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version -""" - -VERSION_PATTERN = _VERSION_PATTERN -""" -A string containing the regular expression used to match a valid version. - -The pattern is not anchored at either end, and is intended for embedding in larger -expressions (for example, matching a version number as part of a file name). The -regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE`` -flags set. - -:meta hide-value: -""" - - -class Version(_BaseVersion): - """This class abstracts handling of a project's versions. - - A :class:`Version` instance is comparison aware and can be compared and - sorted using the standard Python interfaces. - - >>> v1 = Version("1.0a5") - >>> v2 = Version("1.0") - >>> v1 - <Version('1.0a5')> - >>> v2 - <Version('1.0')> - >>> v1 < v2 - True - >>> v1 == v2 - False - >>> v1 > v2 - False - >>> v1 >= v2 - False - >>> v1 <= v2 - True - """ - - _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) - _key: CmpKey - - def __init__(self, version: str) -> None: - """Initialize a Version object. - - :param version: - The string representation of a version which will be parsed and normalized - before use. - :raises InvalidVersion: - If the ``version`` does not conform to PEP 440 in any way then this - exception will be raised. - """ - - # Validate the version and parse it into pieces - match = self._regex.search(version) - if not match: - raise InvalidVersion(f"Invalid version: '{version}'") - - # Store the parsed out pieces of the version - self._version = _Version( - epoch=int(match.group("epoch")) if match.group("epoch") else 0, - release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), - post=_parse_letter_version( - match.group("post_l"), match.group("post_n1") or match.group("post_n2") - ), - dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), - local=_parse_local_version(match.group("local")), - ) - - # Generate a key which will be used for sorting - self._key = _cmpkey( - self._version.epoch, - self._version.release, - self._version.pre, - self._version.post, - self._version.dev, - self._version.local, - ) - - def __repr__(self) -> str: - """A representation of the Version that shows all internal state. - - >>> Version('1.0.0') - <Version('1.0.0')> - """ - return f"<Version('{self}')>" - - def __str__(self) -> str: - """A string representation of the version that can be rounded-tripped. - - >>> str(Version("1.0a5")) - '1.0a5' - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - # Pre-release - if self.pre is not None: - parts.append("".join(str(x) for x in self.pre)) - - # Post-release - if self.post is not None: - parts.append(f".post{self.post}") - - # Development release - if self.dev is not None: - parts.append(f".dev{self.dev}") - - # Local version segment - if self.local is not None: - parts.append(f"+{self.local}") - - return "".join(parts) - - @property - def epoch(self) -> int: - """The epoch of the version. - - >>> Version("2.0.0").epoch - 0 - >>> Version("1!2.0.0").epoch - 1 - """ - return self._version.epoch - - @property - def release(self) -> Tuple[int, ...]: - """The components of the "release" segment of the version. - - >>> Version("1.2.3").release - (1, 2, 3) - >>> Version("2.0.0").release - (2, 0, 0) - >>> Version("1!2.0.0.post0").release - (2, 0, 0) - - Includes trailing zeroes but not the epoch or any pre-release / development / - post-release suffixes. - """ - return self._version.release - - @property - def pre(self) -> Optional[Tuple[str, int]]: - """The pre-release segment of the version. - - >>> print(Version("1.2.3").pre) - None - >>> Version("1.2.3a1").pre - ('a', 1) - >>> Version("1.2.3b1").pre - ('b', 1) - >>> Version("1.2.3rc1").pre - ('rc', 1) - """ - return self._version.pre - - @property - def post(self) -> Optional[int]: - """The post-release number of the version. - - >>> print(Version("1.2.3").post) - None - >>> Version("1.2.3.post1").post - 1 - """ - return self._version.post[1] if self._version.post else None - - @property - def dev(self) -> Optional[int]: - """The development number of the version. - - >>> print(Version("1.2.3").dev) - None - >>> Version("1.2.3.dev1").dev - 1 - """ - return self._version.dev[1] if self._version.dev else None - - @property - def local(self) -> Optional[str]: - """The local version segment of the version. - - >>> print(Version("1.2.3").local) - None - >>> Version("1.2.3+abc").local - 'abc' - """ - if self._version.local: - return ".".join(str(x) for x in self._version.local) - else: - return None - - @property - def public(self) -> str: - """The public portion of the version. - - >>> Version("1.2.3").public - '1.2.3' - >>> Version("1.2.3+abc").public - '1.2.3' - >>> Version("1.2.3+abc.dev1").public - '1.2.3' - """ - return str(self).split("+", 1)[0] - - @property - def base_version(self) -> str: - """The "base version" of the version. - - >>> Version("1.2.3").base_version - '1.2.3' - >>> Version("1.2.3+abc").base_version - '1.2.3' - >>> Version("1!1.2.3+abc.dev1").base_version - '1!1.2.3' - - The "base version" is the public version of the project without any pre or post - release markers. - """ - parts = [] - - # Epoch - if self.epoch != 0: - parts.append(f"{self.epoch}!") - - # Release segment - parts.append(".".join(str(x) for x in self.release)) - - return "".join(parts) - - @property - def is_prerelease(self) -> bool: - """Whether this version is a pre-release. - - >>> Version("1.2.3").is_prerelease - False - >>> Version("1.2.3a1").is_prerelease - True - >>> Version("1.2.3b1").is_prerelease - True - >>> Version("1.2.3rc1").is_prerelease - True - >>> Version("1.2.3dev1").is_prerelease - True - """ - return self.dev is not None or self.pre is not None - - @property - def is_postrelease(self) -> bool: - """Whether this version is a post-release. - - >>> Version("1.2.3").is_postrelease - False - >>> Version("1.2.3.post1").is_postrelease - True - """ - return self.post is not None - - @property - def is_devrelease(self) -> bool: - """Whether this version is a development release. - - >>> Version("1.2.3").is_devrelease - False - >>> Version("1.2.3.dev1").is_devrelease - True - """ - return self.dev is not None - - @property - def major(self) -> int: - """The first item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").major - 1 - """ - return self.release[0] if len(self.release) >= 1 else 0 - - @property - def minor(self) -> int: - """The second item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").minor - 2 - >>> Version("1").minor - 0 - """ - return self.release[1] if len(self.release) >= 2 else 0 - - @property - def micro(self) -> int: - """The third item of :attr:`release` or ``0`` if unavailable. - - >>> Version("1.2.3").micro - 3 - >>> Version("1").micro - 0 - """ - return self.release[2] if len(self.release) >= 3 else 0 - - -def _parse_letter_version( - letter: Optional[str], number: Union[str, bytes, SupportsInt, None] -) -> Optional[Tuple[str, int]]: - - if letter: - # We consider there to be an implicit 0 in a pre-release if there is - # not a numeral associated with it. - if number is None: - number = 0 - - # We normalize any letters to their lower case form - letter = letter.lower() - - # We consider some words to be alternate spellings of other words and - # in those cases we want to normalize the spellings to our preferred - # spelling. - if letter == "alpha": - letter = "a" - elif letter == "beta": - letter = "b" - elif letter in ["c", "pre", "preview"]: - letter = "rc" - elif letter in ["rev", "r"]: - letter = "post" - - return letter, int(number) - if not letter and number: - # We assume if we are given a number, but we are not given a letter - # then this is using the implicit post release syntax (e.g. 1.0-1) - letter = "post" - - return letter, int(number) - - return None - - -_local_version_separators = re.compile(r"[\._-]") - - -def _parse_local_version(local: Optional[str]) -> Optional[LocalType]: - """ - Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). - """ - if local is not None: - return tuple( - part.lower() if not part.isdigit() else int(part) - for part in _local_version_separators.split(local) - ) - return None - - -def _cmpkey( - epoch: int, - release: Tuple[int, ...], - pre: Optional[Tuple[str, int]], - post: Optional[Tuple[str, int]], - dev: Optional[Tuple[str, int]], - local: Optional[LocalType], -) -> CmpKey: - - # When we compare a release version, we want to compare it with all of the - # trailing zeros removed. So we'll use a reverse the list, drop all the now - # leading zeros until we come to something non zero, then take the rest - # re-reverse it back into the correct order and make it a tuple and use - # that for our sorting key. - _release = tuple( - reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) - ) - - # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. - # We'll do this by abusing the pre segment, but we _only_ want to do this - # if there is not a pre or a post segment. If we have one of those then - # the normal sorting rules will handle this case correctly. - if pre is None and post is None and dev is not None: - _pre: CmpPrePostDevType = NegativeInfinity - # Versions without a pre-release (except as noted above) should sort after - # those with one. - elif pre is None: - _pre = Infinity - else: - _pre = pre - - # Versions without a post segment should sort before those with one. - if post is None: - _post: CmpPrePostDevType = NegativeInfinity - - else: - _post = post - - # Versions without a development segment should sort after those with one. - if dev is None: - _dev: CmpPrePostDevType = Infinity - - else: - _dev = dev - - if local is None: - # Versions without a local segment should sort before those with one. - _local: CmpLocalType = NegativeInfinity - else: - # Versions with a local segment need that segment parsed to implement - # the sorting rules in PEP440. - # - Alpha numeric segments sort before numeric segments - # - Alpha numeric segments sort lexicographically - # - Numeric segments sort numerically - # - Shorter versions sort before longer versions when the prefixes - # match exactly - _local = tuple( - (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local - ) - - return epoch, _release, _pre, _post, _dev, _local diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py deleted file mode 100644 index 4c6ec97ec69..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -__all__ = ("loads", "load", "TOMLDecodeError") -__version__ = "2.0.1" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT - -from ._parser import TOMLDecodeError, load, loads - -# Pretend this exception was created here. -TOMLDecodeError.__module__ = __name__ diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py deleted file mode 100644 index f1bb0aa19a5..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_parser.py +++ /dev/null @@ -1,691 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from __future__ import annotations - -from collections.abc import Iterable -import string -from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple - -from ._re import ( - RE_DATETIME, - RE_LOCALTIME, - RE_NUMBER, - match_to_datetime, - match_to_localtime, - match_to_number, -) -from ._types import Key, ParseFloat, Pos - -ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) - -# Neither of these sets include quotation mark or backslash. They are -# currently handled as separate cases in the parser functions. -ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t") -ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n") - -ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS -ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS - -ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS - -TOML_WS = frozenset(" \t") -TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") -KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) - -BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( - { - "\\b": "\u0008", # backspace - "\\t": "\u0009", # tab - "\\n": "\u000A", # linefeed - "\\f": "\u000C", # form feed - "\\r": "\u000D", # carriage return - '\\"': "\u0022", # quote - "\\\\": "\u005C", # backslash - } -) - - -class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" - - -def load(__fp: BinaryIO, *, parse_float: ParseFloat = float) -> dict[str, Any]: - """Parse TOML from a binary file object.""" - b = __fp.read() - try: - s = b.decode() - except AttributeError: - raise TypeError( - "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`" - ) from None - return loads(s, parse_float=parse_float) - - -def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901 - """Parse TOML from a string.""" - - # The spec allows converting "\r\n" to "\n", even in string - # literals. Let's do so to simplify parsing. - src = __s.replace("\r\n", "\n") - pos = 0 - out = Output(NestedDict(), Flags()) - header: Key = () - parse_float = make_safe_parse_float(parse_float) - - # Parse one statement at a time - # (typically means one line in TOML source) - while True: - # 1. Skip line leading whitespace - pos = skip_chars(src, pos, TOML_WS) - - # 2. Parse rules. Expect one of the following: - # - end of file - # - end of line - # - comment - # - key/value pair - # - append dict to list (and move to its namespace) - # - create dict (and move to its namespace) - # Skip trailing whitespace when applicable. - try: - char = src[pos] - except IndexError: - break - if char == "\n": - pos += 1 - continue - if char in KEY_INITIAL_CHARS: - pos = key_value_rule(src, pos, out, header, parse_float) - pos = skip_chars(src, pos, TOML_WS) - elif char == "[": - try: - second_char: str | None = src[pos + 1] - except IndexError: - second_char = None - out.flags.finalize_pending() - if second_char == "[": - pos, header = create_list_rule(src, pos, out) - else: - pos, header = create_dict_rule(src, pos, out) - pos = skip_chars(src, pos, TOML_WS) - elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") - - # 3. Skip comment - pos = skip_comment(src, pos) - - # 4. Expect end of line or end of file - try: - char = src[pos] - except IndexError: - break - if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" - ) - pos += 1 - - return out.data.dict - - -class Flags: - """Flags that map to parsed keys/namespaces.""" - - # Marks an immutable namespace (inline array or inline table). - FROZEN = 0 - # Marks a nest that has been explicitly created and can no longer - # be opened using the "[table]" syntax. - EXPLICIT_NEST = 1 - - def __init__(self) -> None: - self._flags: dict[str, dict] = {} - self._pending_flags: set[tuple[Key, int]] = set() - - def add_pending(self, key: Key, flag: int) -> None: - self._pending_flags.add((key, flag)) - - def finalize_pending(self) -> None: - for key, flag in self._pending_flags: - self.set(key, flag, recursive=False) - self._pending_flags.clear() - - def unset_all(self, key: Key) -> None: - cont = self._flags - for k in key[:-1]: - if k not in cont: - return - cont = cont[k]["nested"] - cont.pop(key[-1], None) - - def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003 - cont = self._flags - key_parent, key_stem = key[:-1], key[-1] - for k in key_parent: - if k not in cont: - cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} - cont = cont[k]["nested"] - if key_stem not in cont: - cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}} - cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag) - - def is_(self, key: Key, flag: int) -> bool: - if not key: - return False # document root has no flags - cont = self._flags - for k in key[:-1]: - if k not in cont: - return False - inner_cont = cont[k] - if flag in inner_cont["recursive_flags"]: - return True - cont = inner_cont["nested"] - key_stem = key[-1] - if key_stem in cont: - cont = cont[key_stem] - return flag in cont["flags"] or flag in cont["recursive_flags"] - return False - - -class NestedDict: - def __init__(self) -> None: - # The parsed content of the TOML document - self.dict: dict[str, Any] = {} - - def get_or_create_nest( - self, - key: Key, - *, - access_lists: bool = True, - ) -> dict: - cont: Any = self.dict - for k in key: - if k not in cont: - cont[k] = {} - cont = cont[k] - if access_lists and isinstance(cont, list): - cont = cont[-1] - if not isinstance(cont, dict): - raise KeyError("There is no nest behind this key") - return cont - - def append_nest_to_list(self, key: Key) -> None: - cont = self.get_or_create_nest(key[:-1]) - last_key = key[-1] - if last_key in cont: - list_ = cont[last_key] - if not isinstance(list_, list): - raise KeyError("An object other than list found behind this key") - list_.append({}) - else: - cont[last_key] = [{}] - - -class Output(NamedTuple): - data: NestedDict - flags: Flags - - -def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: - try: - while src[pos] in chars: - pos += 1 - except IndexError: - pass - return pos - - -def skip_until( - src: str, - pos: Pos, - expect: str, - *, - error_on: frozenset[str], - error_on_eof: bool, -) -> Pos: - try: - new_pos = src.index(expect, pos) - except ValueError: - new_pos = len(src) - if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None - - if not error_on.isdisjoint(src[pos:new_pos]): - while src[pos] not in error_on: - pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") - return new_pos - - -def skip_comment(src: str, pos: Pos) -> Pos: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char == "#": - return skip_until( - src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False - ) - return pos - - -def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos: - while True: - pos_before_skip = pos - pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) - pos = skip_comment(src, pos) - if pos == pos_before_skip: - return pos - - -def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: - pos += 1 # Skip "[" - pos = skip_chars(src, pos, TOML_WS) - pos, key = parse_key(src, pos) - - if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") - out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) - try: - out.data.get_or_create_nest(key) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - - if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") - return pos + 1, key - - -def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: - pos += 2 # Skip "[[" - pos = skip_chars(src, pos, TOML_WS) - pos, key = parse_key(src, pos) - - if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") - # Free the namespace now that it points to another empty list item... - out.flags.unset_all(key) - # ...but this key precisely is still prohibited from table declaration - out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) - try: - out.data.append_nest_to_list(key) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - - if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") - return pos + 2, key - - -def key_value_rule( - src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat -) -> Pos: - pos, key, value = parse_key_value_pair(src, pos, parse_float) - key_parent, key_stem = key[:-1], key[-1] - abs_key_parent = header + key_parent - - relative_path_cont_keys = (header + key[:i] for i in range(1, len(key))) - for cont_key in relative_path_cont_keys: - # Check that dotted key syntax does not redefine an existing table - if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") - # Containers in the relative path can't be opened with the table syntax or - # dotted key/value syntax in following table sections. - out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) - - if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" - ) - - try: - nest = out.data.get_or_create_nest(abs_key_parent) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") - # Mark inline table and array namespaces recursively immutable - if isinstance(value, (dict, list)): - out.flags.set(header + key, Flags.FROZEN, recursive=True) - nest[key_stem] = value - return pos - - -def parse_key_value_pair( - src: str, pos: Pos, parse_float: ParseFloat -) -> tuple[Pos, Key, Any]: - pos, key = parse_key(src, pos) - try: - char: str | None = src[pos] - except IndexError: - char = None - if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - pos, value = parse_value(src, pos, parse_float) - return pos, key, value - - -def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: - pos, key_part = parse_key_part(src, pos) - key: Key = (key_part,) - pos = skip_chars(src, pos, TOML_WS) - while True: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char != ".": - return pos, key - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - pos, key_part = parse_key_part(src, pos) - key += (key_part,) - pos = skip_chars(src, pos, TOML_WS) - - -def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: - try: - char: str | None = src[pos] - except IndexError: - char = None - if char in BARE_KEY_CHARS: - start_pos = pos - pos = skip_chars(src, pos, BARE_KEY_CHARS) - return pos, src[start_pos:pos] - if char == "'": - return parse_literal_str(src, pos) - if char == '"': - return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") - - -def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: - pos += 1 - return parse_basic_str(src, pos, multiline=False) - - -def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]: - pos += 1 - array: list = [] - - pos = skip_comments_and_array_ws(src, pos) - if src.startswith("]", pos): - return pos + 1, array - while True: - pos, val = parse_value(src, pos, parse_float) - array.append(val) - pos = skip_comments_and_array_ws(src, pos) - - c = src[pos : pos + 1] - if c == "]": - return pos + 1, array - if c != ",": - raise suffixed_err(src, pos, "Unclosed array") - pos += 1 - - pos = skip_comments_and_array_ws(src, pos) - if src.startswith("]", pos): - return pos + 1, array - - -def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]: - pos += 1 - nested_dict = NestedDict() - flags = Flags() - - pos = skip_chars(src, pos, TOML_WS) - if src.startswith("}", pos): - return pos + 1, nested_dict.dict - while True: - pos, key, value = parse_key_value_pair(src, pos, parse_float) - key_parent, key_stem = key[:-1], key[-1] - if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") - try: - nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) - except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None - if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") - nest[key_stem] = value - pos = skip_chars(src, pos, TOML_WS) - c = src[pos : pos + 1] - if c == "}": - return pos + 1, nested_dict.dict - if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") - if isinstance(value, (dict, list)): - flags.set(key, Flags.FROZEN, recursive=True) - pos += 1 - pos = skip_chars(src, pos, TOML_WS) - - -def parse_basic_str_escape( - src: str, pos: Pos, *, multiline: bool = False -) -> tuple[Pos, str]: - escape_id = src[pos : pos + 2] - pos += 2 - if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}: - # Skip whitespace until next non-whitespace character or end of - # the doc. Error if non-whitespace is found before newline. - if escape_id != "\\\n": - pos = skip_chars(src, pos, TOML_WS) - try: - char = src[pos] - except IndexError: - return pos, "" - if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") - pos += 1 - pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) - return pos, "" - if escape_id == "\\u": - return parse_hex_char(src, pos, 4) - if escape_id == "\\U": - return parse_hex_char(src, pos, 8) - try: - return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] - except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None - - -def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: - return parse_basic_str_escape(src, pos, multiline=True) - - -def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: - hex_str = src[pos : pos + hex_len] - if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") - pos += hex_len - hex_int = int(hex_str, 16) - if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") - return pos, chr(hex_int) - - -def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]: - pos += 1 # Skip starting apostrophe - start_pos = pos - pos = skip_until( - src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True - ) - return pos + 1, src[start_pos:pos] # Skip ending apostrophe - - -def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]: - pos += 3 - if src.startswith("\n", pos): - pos += 1 - - if literal: - delim = "'" - end_pos = skip_until( - src, - pos, - "'''", - error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS, - error_on_eof=True, - ) - result = src[pos:end_pos] - pos = end_pos + 3 - else: - delim = '"' - pos, result = parse_basic_str(src, pos, multiline=True) - - # Add at maximum two extra apostrophes/quotes if the end sequence - # is 4 or 5 chars long instead of just 3. - if not src.startswith(delim, pos): - return pos, result - pos += 1 - if not src.startswith(delim, pos): - return pos, result + delim - pos += 1 - return pos, result + (delim * 2) - - -def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: - if multiline: - error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS - parse_escapes = parse_basic_str_escape_multiline - else: - error_on = ILLEGAL_BASIC_STR_CHARS - parse_escapes = parse_basic_str_escape - result = "" - start_pos = pos - while True: - try: - char = src[pos] - except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None - if char == '"': - if not multiline: - return pos + 1, result + src[start_pos:pos] - if src.startswith('"""', pos): - return pos + 3, result + src[start_pos:pos] - pos += 1 - continue - if char == "\\": - result += src[start_pos:pos] - pos, parsed_escape = parse_escapes(src, pos) - result += parsed_escape - start_pos = pos - continue - if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") - pos += 1 - - -def parse_value( # noqa: C901 - src: str, pos: Pos, parse_float: ParseFloat -) -> tuple[Pos, Any]: - try: - char: str | None = src[pos] - except IndexError: - char = None - - # IMPORTANT: order conditions based on speed of checking and likelihood - - # Basic strings - if char == '"': - if src.startswith('"""', pos): - return parse_multiline_str(src, pos, literal=False) - return parse_one_line_basic_str(src, pos) - - # Literal strings - if char == "'": - if src.startswith("'''", pos): - return parse_multiline_str(src, pos, literal=True) - return parse_literal_str(src, pos) - - # Booleans - if char == "t": - if src.startswith("true", pos): - return pos + 4, True - if char == "f": - if src.startswith("false", pos): - return pos + 5, False - - # Arrays - if char == "[": - return parse_array(src, pos, parse_float) - - # Inline tables - if char == "{": - return parse_inline_table(src, pos, parse_float) - - # Dates and times - datetime_match = RE_DATETIME.match(src, pos) - if datetime_match: - try: - datetime_obj = match_to_datetime(datetime_match) - except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e - return datetime_match.end(), datetime_obj - localtime_match = RE_LOCALTIME.match(src, pos) - if localtime_match: - return localtime_match.end(), match_to_localtime(localtime_match) - - # Integers and "normal" floats. - # The regex will greedily match any type starting with a decimal - # char, so needs to be located after handling of dates and times. - number_match = RE_NUMBER.match(src, pos) - if number_match: - return number_match.end(), match_to_number(number_match, parse_float) - - # Special floats - first_three = src[pos : pos + 3] - if first_three in {"inf", "nan"}: - return pos + 3, parse_float(first_three) - first_four = src[pos : pos + 4] - if first_four in {"-inf", "+inf", "-nan", "+nan"}: - return pos + 4, parse_float(first_four) - - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") - - -def is_unicode_scalar_value(codepoint: int) -> bool: - return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111) - - -def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat: - """A decorator to make `parse_float` safe. - - `parse_float` must not return dicts or lists, because these types - would be mixed with parsed TOML tables and arrays, thus confusing - the parser. The returned decorated callable raises `ValueError` - instead of returning illegal types. - """ - # The default `float` callable never returns illegal types. Optimize it. - if parse_float is float: # type: ignore[comparison-overlap] - return float - - def safe_parse_float(float_str: str) -> Any: - float_value = parse_float(float_str) - if isinstance(float_value, (dict, list)): - raise ValueError("parse_float must not return dicts or lists") - return float_value - - return safe_parse_float diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py deleted file mode 100644 index 994bb7493fd..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_re.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from __future__ import annotations - -from datetime import date, datetime, time, timedelta, timezone, tzinfo -from functools import lru_cache -import re -from typing import Any - -from ._types import ParseFloat - -# E.g. -# - 00:32:00.999999 -# - 00:32:00 -_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?" - -RE_NUMBER = re.compile( - r""" -0 -(?: - x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex - | - b[01](?:_?[01])* # bin - | - o[0-7](?:_?[0-7])* # oct -) -| -[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part -(?P<floatpart> - (?:\.[0-9](?:_?[0-9])*)? # optional fractional part - (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part -) -""", - flags=re.VERBOSE, -) -RE_LOCALTIME = re.compile(_TIME_RE_STR) -RE_DATETIME = re.compile( - rf""" -([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27 -(?: - [Tt ] - {_TIME_RE_STR} - (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset -)? -""", - flags=re.VERBOSE, -) - - -def match_to_datetime(match: re.Match) -> datetime | date: - """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. - - Raises ValueError if the match does not correspond to a valid date - or datetime. - """ - ( - year_str, - month_str, - day_str, - hour_str, - minute_str, - sec_str, - micros_str, - zulu_time, - offset_sign_str, - offset_hour_str, - offset_minute_str, - ) = match.groups() - year, month, day = int(year_str), int(month_str), int(day_str) - if hour_str is None: - return date(year, month, day) - hour, minute, sec = int(hour_str), int(minute_str), int(sec_str) - micros = int(micros_str.ljust(6, "0")) if micros_str else 0 - if offset_sign_str: - tz: tzinfo | None = cached_tz( - offset_hour_str, offset_minute_str, offset_sign_str - ) - elif zulu_time: - tz = timezone.utc - else: # local date-time - tz = None - return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) - - -@lru_cache(maxsize=None) -def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: - sign = 1 if sign_str == "+" else -1 - return timezone( - timedelta( - hours=sign * int(hour_str), - minutes=sign * int(minute_str), - ) - ) - - -def match_to_localtime(match: re.Match) -> time: - hour_str, minute_str, sec_str, micros_str = match.groups() - micros = int(micros_str.ljust(6, "0")) if micros_str else 0 - return time(int(hour_str), int(minute_str), int(sec_str), micros) - - -def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any: - if match.group("floatpart"): - return parse_float(match.group()) - return int(match.group(), 0) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py deleted file mode 100644 index d949412e03b..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/_types.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Taneli Hukkinen -# Licensed to PSF under a Contributor Agreement. - -from typing import Any, Callable, Tuple - -# Type annotations -ParseFloat = Callable[[str], Any] -Key = Tuple[str, ...] -Pos = int diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed b/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed deleted file mode 100644 index 7632ecf7754..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/tomli/py.typed +++ /dev/null @@ -1 +0,0 @@ -# Marker file for PEP 561 diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py b/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py deleted file mode 100644 index 26b723c1fd3..00000000000 --- a/contrib/python/setuptools/py3/setuptools/_vendor/zipp.py +++ /dev/null @@ -1,329 +0,0 @@ -import io -import posixpath -import zipfile -import itertools -import contextlib -import sys -import pathlib - -if sys.version_info < (3, 7): - from collections import OrderedDict -else: - OrderedDict = dict - - -__all__ = ['Path'] - - -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = OrderedDict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(zipfile.ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, zipfile.ZipFile): - return cls(_pathlib_compat(source)) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -def _pathlib_compat(path): - """ - For path-like objects, convert to a filename for compatibility - on Python 3.6.1 and earlier. - """ - try: - return path.__fspath__() - except AttributeError: - return str(path) - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = zipfile.ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix - - @property - def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes - - @property - def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *map(_pathlib_compat, other)) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) diff --git a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py index 1f1967e7aa3..ef35d183e86 100644 --- a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py +++ b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py @@ -15,8 +15,8 @@ from itertools import filterfalse from typing import Dict, Mapping, TypeVar from .. import _reqs -from ..extern.jaraco.text import yield_lines -from ..extern.packaging.requirements import Requirement +from jaraco.text import yield_lines +from packaging.requirements import Requirement # dict can work as an ordered set diff --git a/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py b/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py index d8cdd4e4060..5b9bcec60cf 100644 --- a/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py +++ b/contrib/python/setuptools/py3/setuptools/command/bdist_wheel.py @@ -23,10 +23,10 @@ from typing import TYPE_CHECKING, Callable, Iterable, Literal, Sequence, cast from zipfile import ZIP_DEFLATED, ZIP_STORED from .. import Command, __version__ -from ..extern.wheel.metadata import pkginfo_to_metadata -from ..extern.packaging import tags -from ..extern.packaging import version as _packaging_version -from ..extern.wheel.wheelfile import WheelFile +from wheel.metadata import pkginfo_to_metadata +from packaging import tags +from packaging import version as _packaging_version +from wheel.wheelfile import WheelFile if TYPE_CHECKING: import types @@ -67,8 +67,8 @@ def python_tag() -> str: def get_platform(archive_root: str | None) -> str: """Return our platform name 'win32', 'linux_x86_64'""" result = sysconfig.get_platform() - if result.startswith("macosx") and archive_root is not None: - from ..extern.wheel.macosx_libfile import calculate_macosx_platform_tag + if result.startswith("macosx") and archive_root is not None: # pragma: no cover + from wheel.macosx_libfile import calculate_macosx_platform_tag result = calculate_macosx_platform_tag(archive_root, result) elif _is_32bit_interpreter(): @@ -451,7 +451,7 @@ class bdist_wheel(Command): def write_wheelfile( self, wheelfile_base: str, generator: str = f"setuptools ({__version__})" - ): + ) -> None: from email.message import Message msg = Message() @@ -525,7 +525,7 @@ class bdist_wheel(Command): return files - def egg2dist(self, egginfo_path: str, distinfo_path: str): + def egg2dist(self, egginfo_path: str, distinfo_path: str) -> None: """Convert an .egg-info directory into a .dist-info directory""" def adios(p: str) -> None: diff --git a/contrib/python/setuptools/py3/setuptools/command/build_py.py b/contrib/python/setuptools/py3/setuptools/command/build_py.py index ab49874635f..15a4f63fdd4 100644 --- a/contrib/python/setuptools/py3/setuptools/command/build_py.py +++ b/contrib/python/setuptools/py3/setuptools/command/build_py.py @@ -13,7 +13,7 @@ import stat from pathlib import Path from typing import Iterable, Iterator -from ..extern.more_itertools import unique_everseen +from more_itertools import unique_everseen from ..warnings import SetuptoolsDeprecationWarning diff --git a/contrib/python/setuptools/py3/setuptools/command/easy_install.py b/contrib/python/setuptools/py3/setuptools/command/easy_install.py index e6ce3fcc055..36114d40ed9 100644 --- a/contrib/python/setuptools/py3/setuptools/command/easy_install.py +++ b/contrib/python/setuptools/py3/setuptools/command/easy_install.py @@ -76,7 +76,7 @@ from pkg_resources import ( import pkg_resources from ..compat import py39, py311 from .._path import ensure_directory -from ..extern.jaraco.text import yield_lines +from jaraco.text import yield_lines # Turn on PEP440Warnings diff --git a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py index ae31bb4c79b..49fd609b151 100644 --- a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py +++ b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py @@ -299,7 +299,7 @@ class editable_wheel(Command): build = self.get_finalized_command("build") for name in build.get_sub_commands(): cmd = self.get_finalized_command(name) - if name == "build_py" and type(cmd) != build_py_cls: + if name == "build_py" and type(cmd) is not build_py_cls: self._safely_run(name) else: self.run_command(name) @@ -333,7 +333,7 @@ class editable_wheel(Command): ) def _create_wheel_file(self, bdist_wheel): - from ..extern.wheel.wheelfile import WheelFile + from wheel.wheelfile import WheelFile dist_info = self.get_finalized_command("dist_info") dist_name = dist_info.name @@ -443,8 +443,7 @@ class _LinkTree(_StaticPth): ): self.auxiliary_dir = Path(auxiliary_dir) self.build_lib = Path(build_lib).resolve() - # TODO: Update typeshed distutils stubs to overload non-None return type by default - self._file = dist.get_command_obj("build_py").copy_file # type: ignore[union-attr] + self._file = dist.get_command_obj("build_py").copy_file super().__init__(dist, name, [self.auxiliary_dir]) def __call__(self, wheel: WheelFile, files: list[str], mapping: dict[str, str]): @@ -462,9 +461,7 @@ class _LinkTree(_StaticPth): dest = self.auxiliary_dir / relative_output if not dest.parent.is_dir(): dest.parent.mkdir(parents=True) - # TODO: Update typeshed distutils stubs so distutils.cmd.Command.copy_file, accepts PathLike - # same with methods used by copy_file - self._file(src_file, dest, link=link) # type: ignore[arg-type] + self._file(src_file, dest, link=link) def _create_links(self, outputs, output_mapping): self.auxiliary_dir.mkdir(parents=True, exist_ok=True) @@ -805,7 +802,7 @@ PATH_PLACEHOLDER = {name!r} + ".__path_hook__" class _EditableFinder: # MetaPathFinder @classmethod - def find_spec(cls, fullname: str, _path=None, _target=None) -> ModuleSpec | None: + def find_spec(cls, fullname: str, path=None, target=None) -> ModuleSpec | None: # type: ignore # Top-level packages and modules (we know these exist in the FS) if fullname in MAPPING: pkg_path = MAPPING[fullname] @@ -851,7 +848,7 @@ class _EditableNamespaceFinder: # PathEntryFinder return [*paths, PATH_PLACEHOLDER] @classmethod - def find_spec(cls, fullname: str, _target=None) -> ModuleSpec | None: + def find_spec(cls, fullname: str, target=None) -> ModuleSpec | None: # type: ignore if fullname in NAMESPACES: spec = ModuleSpec(fullname, None, is_package=True) spec.submodule_search_locations = cls._paths(fullname) diff --git a/contrib/python/setuptools/py3/setuptools/command/egg_info.py b/contrib/python/setuptools/py3/setuptools/command/egg_info.py index 2f203033413..30b62f5f2ef 100644 --- a/contrib/python/setuptools/py3/setuptools/command/egg_info.py +++ b/contrib/python/setuptools/py3/setuptools/command/egg_info.py @@ -27,7 +27,7 @@ from setuptools.command import bdist_egg import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob -from setuptools.extern import packaging +import packaging from ..warnings import SetuptoolsDeprecationWarning @@ -250,17 +250,6 @@ class egg_info(InfoCommon, Command): # self.distribution.metadata.version = self.egg_version - # If we bootstrapped around the lack of a PKG-INFO, as might be the - # case in a fresh checkout, make sure that any special tags get added - # to the version info - # - pd = self.distribution._patched_dist - key = getattr(pd, "key", None) or getattr(pd, "name", None) - if pd is not None and key == self.egg_name.lower(): - pd._version = self.egg_version - pd._parsed_version = packaging.version.Version(self.egg_version) - self.distribution._patched_dist = None - def _get_egg_basename(self, py_version=PY_MAJOR, platform=None): """Compute filename of the output egg. Private API.""" return _egg_basename(self.egg_name, self.egg_version, py_version, platform) diff --git a/contrib/python/setuptools/py3/setuptools/command/install.py b/contrib/python/setuptools/py3/setuptools/command/install.py index c49fcda9397..f1ea2adf1d0 100644 --- a/contrib/python/setuptools/py3/setuptools/command/install.py +++ b/contrib/python/setuptools/py3/setuptools/command/install.py @@ -1,9 +1,12 @@ +from __future__ import annotations + +from collections.abc import Callable from distutils.errors import DistutilsArgError import inspect import glob import platform import distutils.command.install as orig -from typing import cast +from typing import Any, ClassVar, cast import setuptools from ..warnings import SetuptoolsDeprecationWarning, SetuptoolsWarning @@ -29,7 +32,9 @@ class install(orig.install): 'old-and-unmanageable', 'single-version-externally-managed', ] - new_commands = [ + # Type the same as distutils.command.install.install.sub_commands + # Must keep the second tuple item potentially None due to invariance + new_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] = [ ('install_egg_info', lambda self: True), ('install_scripts', lambda self: True), ] diff --git a/contrib/python/setuptools/py3/setuptools/command/install_lib.py b/contrib/python/setuptools/py3/setuptools/command/install_lib.py index 5e74be247e5..3c77c6ebc66 100644 --- a/contrib/python/setuptools/py3/setuptools/command/install_lib.py +++ b/contrib/python/setuptools/py3/setuptools/command/install_lib.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import sys from itertools import product, starmap @@ -92,21 +93,21 @@ class install_lib(orig.install_lib): preserve_times=True, preserve_symlinks=False, level=1, - ): + ) -> list[str]: assert preserve_mode and preserve_times and not preserve_symlinks exclude = self.get_exclusions() if not exclude: - return orig.install_lib.copy_tree(self, infile, outfile) # type: ignore[arg-type] # Fixed upstream + return orig.install_lib.copy_tree(self, infile, outfile) # Exclude namespace package __init__.py* files from the output from setuptools.archive_util import unpack_directory from distutils import log - outfiles = [] + outfiles: list[str] = [] - def pf(src, dst): + def pf(src: str, dst: str): if dst in exclude: log.warn("Skipping installation of %s (namespace package)", dst) return False diff --git a/contrib/python/setuptools/py3/setuptools/command/test.py b/contrib/python/setuptools/py3/setuptools/command/test.py index af1349e1c6e..fbdf9fb9423 100644 --- a/contrib/python/setuptools/py3/setuptools/command/test.py +++ b/contrib/python/setuptools/py3/setuptools/command/test.py @@ -19,8 +19,8 @@ from pkg_resources import ( ) from .._importlib import metadata from setuptools import Command -from setuptools.extern.more_itertools import unique_everseen -from setuptools.extern.jaraco.functools import pass_none +from more_itertools import unique_everseen +from jaraco.functools import pass_none class ScanningLoader(TestLoader): diff --git a/contrib/python/setuptools/py3/setuptools/command/upload_docs.py b/contrib/python/setuptools/py3/setuptools/command/upload_docs.py index 3fbbb62553f..32c9abd796f 100644 --- a/contrib/python/setuptools/py3/setuptools/command/upload_docs.py +++ b/contrib/python/setuptools/py3/setuptools/command/upload_docs.py @@ -50,7 +50,7 @@ class upload_docs(upload): and metadata.entry_points(group='distutils.commands', name='build_sphinx') ) - sub_commands = [('build_sphinx', has_sphinx)] # type: ignore[list-item] # TODO: Fix in typeshed distutils stubs + sub_commands = [('build_sphinx', has_sphinx)] def initialize_options(self): upload.initialize_options(self) diff --git a/contrib/python/setuptools/py3/setuptools/compat/py310.py b/contrib/python/setuptools/py3/setuptools/compat/py310.py index f7d53d6de94..cc875c004b6 100644 --- a/contrib/python/setuptools/py3/setuptools/compat/py310.py +++ b/contrib/python/setuptools/py3/setuptools/compat/py310.py @@ -7,4 +7,4 @@ __all__ = ['tomllib'] if sys.version_info >= (3, 11): import tomllib else: # pragma: no cover - from setuptools.extern import tomli as tomllib + import tomli as tomllib diff --git a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py index f44271c5ddb..6cc59d2b951 100644 --- a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py @@ -203,8 +203,8 @@ def _project_urls(dist: Distribution, val: dict, _root_dir): _set_config(dist, "project_urls", val) -def _python_requires(dist: Distribution, val: dict, _root_dir): - from setuptools.extern.packaging.specifiers import SpecifierSet +def _python_requires(dist: Distribution, val: str, _root_dir): + from packaging.specifiers import SpecifierSet _set_config(dist, "python_requires", SpecifierSet(val)) diff --git a/contrib/python/setuptools/py3/setuptools/config/expand.py b/contrib/python/setuptools/py3/setuptools/config/expand.py index e5f5dc586e8..de6339fa424 100644 --- a/contrib/python/setuptools/py3/setuptools/config/expand.py +++ b/contrib/python/setuptools/py3/setuptools/config/expand.py @@ -31,6 +31,7 @@ from importlib.machinery import ModuleSpec, all_suffixes from itertools import chain from typing import ( TYPE_CHECKING, + Any, Callable, Iterable, Iterator, @@ -122,7 +123,7 @@ def read_files( (By default ``root_dir`` is the current directory). """ - from setuptools.extern.more_itertools import always_iterable + from more_itertools import always_iterable root_dir = os.path.abspath(root_dir or os.getcwd()) _filepaths = (os.path.join(root_dir, path) for path in always_iterable(filepaths)) @@ -158,7 +159,7 @@ def read_attr( attr_desc: str, package_dir: Mapping[str, str] | None = None, root_dir: StrPath | None = None, -): +) -> Any: """Reads the value of an attribute from a module. This function will try to read the attributed statically first @@ -287,7 +288,7 @@ def find_packages( :rtype: list """ from setuptools.discovery import construct_package_dir - from setuptools.extern.more_itertools import unique_everseen, always_iterable + from more_itertools import unique_everseen, always_iterable if namespaces: from setuptools.discovery import PEP420PackageFinder as PackageFinder diff --git a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py index d41c956cbd4..a83e43bb356 100644 --- a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py @@ -15,7 +15,7 @@ import logging import os from contextlib import contextmanager from functools import partial -from typing import TYPE_CHECKING, Callable, Mapping +from typing import TYPE_CHECKING, Any, Callable, Mapping from .._path import StrPath from ..errors import FileError, InvalidConfigError @@ -76,7 +76,7 @@ def read_configuration( expand=True, ignore_option_errors=False, dist: Distribution | None = None, -): +) -> dict[str, Any]: """Read given configuration file and returns options from it as a dict. :param str|unicode filepath: Path to configuration file in the ``pyproject.toml`` @@ -278,7 +278,7 @@ class _ConfigExpander: def _expand_directive( self, specifier: str, directive, package_dir: Mapping[str, str] ): - from setuptools.extern.more_itertools import always_iterable + from more_itertools import always_iterable with _ignore_errors(self.ignore_option_errors): root_dir = self.root_dir diff --git a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py index 80ebe3d9bd5..772b8d00e02 100644 --- a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py +++ b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py @@ -24,6 +24,7 @@ from typing import ( Dict, Generic, Iterable, + Iterator, Tuple, TypeVar, Union, @@ -31,10 +32,10 @@ from typing import ( from .._path import StrPath from ..errors import FileError, OptionError -from ..extern.packaging.markers import default_environment as marker_env -from ..extern.packaging.requirements import InvalidRequirement, Requirement -from ..extern.packaging.specifiers import SpecifierSet -from ..extern.packaging.version import InvalidVersion, Version +from packaging.markers import default_environment as marker_env +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version from ..warnings import SetuptoolsDeprecationWarning from . import expand @@ -260,7 +261,9 @@ class ConfigHandler(Generic[Target]): """ @classmethod - def _section_options(cls, options: AllCommandOptions): + def _section_options( + cls, options: AllCommandOptions + ) -> Iterator[tuple[str, SingleCommandOptions]]: for full_name, value in options.items(): pre, sep, name = full_name.partition(cls.section_prefix) if pre: diff --git a/contrib/python/setuptools/py3/setuptools/depends.py b/contrib/python/setuptools/py3/setuptools/depends.py index 2226b6784af..871a0925ef7 100644 --- a/contrib/python/setuptools/py3/setuptools/depends.py +++ b/contrib/python/setuptools/py3/setuptools/depends.py @@ -6,7 +6,7 @@ import dis from . import _imp from ._imp import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE -from .extern.packaging.version import Version +from packaging.version import Version __all__ = ['Require', 'find_module'] diff --git a/contrib/python/setuptools/py3/setuptools/dist.py b/contrib/python/setuptools/py3/setuptools/dist.py index 32e8d43c64a..b4496ab9860 100644 --- a/contrib/python/setuptools/py3/setuptools/dist.py +++ b/contrib/python/setuptools/py3/setuptools/dist.py @@ -6,7 +6,6 @@ import numbers import os import re import sys -from contextlib import suppress from glob import iglob from pathlib import Path from typing import TYPE_CHECKING, MutableMapping @@ -21,14 +20,13 @@ from distutils.errors import DistutilsOptionError, DistutilsSetupError from distutils.fancy_getopt import translate_longopt from distutils.util import strtobool -from .extern.more_itertools import partition, unique_everseen -from .extern.ordered_set import OrderedSet -from .extern.packaging.markers import InvalidMarker, Marker -from .extern.packaging.specifiers import InvalidSpecifier, SpecifierSet -from .extern.packaging.version import Version +from more_itertools import partition, unique_everseen +from ordered_set import OrderedSet +from packaging.markers import InvalidMarker, Marker +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import Version from . import _entry_points -from . import _normalization from . import _reqs from . import command as _ # noqa -- imported for side-effects from ._importlib import metadata @@ -269,24 +267,9 @@ class Distribution(_Distribution): 'extras_require': dict, } - _patched_dist = None # Used by build_py, editable_wheel and install_lib commands for legacy namespaces namespace_packages: list[str] #: :meta private: DEPRECATED - def patch_missing_pkg_info(self, attrs): - # Fake up a replacement for the data that would normally come from - # PKG-INFO, but which might not yet be built if this is a fresh - # checkout. - # - if not attrs or 'name' not in attrs or 'version' not in attrs: - return - name = _normalization.safe_name(str(attrs['name'])).lower() - with suppress(metadata.PackageNotFoundError): - dist = metadata.distribution(name) - if dist is not None and not dist.read_text('PKG-INFO'): - dist._version = _normalization.safe_version(str(attrs['version'])) - self._patched_dist = dist - def __init__(self, attrs: MutableMapping | None = None) -> None: have_package_data = hasattr(self, "package_data") if not have_package_data: @@ -295,7 +278,6 @@ class Distribution(_Distribution): self.dist_files: list[tuple[str, str, str]] = [] # Filter-out setuptools' specific options. self.src_root = attrs.pop("src_root", None) - self.patch_missing_pkg_info(attrs) self.dependency_links = attrs.pop('dependency_links', []) self.setup_requires = attrs.pop('setup_requires', []) for ep in metadata.entry_points(group='distutils.setup_keywords'): diff --git a/contrib/python/setuptools/py3/setuptools/extern/__init__.py b/contrib/python/setuptools/py3/setuptools/extern/__init__.py deleted file mode 100644 index f9b6eea70db..00000000000 --- a/contrib/python/setuptools/py3/setuptools/extern/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -import importlib.util -import sys - - -class VendorImporter: - """ - A PEP 302 meta path importer for finding optionally-vendored - or otherwise naturally-installed packages from root_name. - """ - - def __init__(self, root_name, vendored_names=(), vendor_pkg=None): - self.root_name = root_name - self.vendored_names = set(vendored_names) - self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor') - - @property - def search_path(self): - """ - Search first the vendor package then as a natural package. - """ - yield self.vendor_pkg + '.' - yield '' - - def _module_matches_namespace(self, fullname): - """Figure out if the target module is vendored.""" - root, base, target = fullname.partition(self.root_name + '.') - return not root and any(map(target.startswith, self.vendored_names)) - - def load_module(self, fullname): - """ - Iterate over the search path to locate and load fullname. - """ - root, base, target = fullname.partition(self.root_name + '.') - for prefix in self.search_path: - extant = prefix + target - try: - __import__(extant) - except ImportError: - continue - mod = sys.modules[extant] - sys.modules[fullname] = mod - return mod - else: - raise ImportError( - "The '{target}' package is required; " - "normally this is bundled with this package so if you get " - "this warning, consult the packager of your " - "distribution.".format(**locals()) - ) - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - pass - - def find_spec(self, fullname, path=None, target=None): - """Return a module spec for vendored names.""" - return ( - importlib.util.spec_from_loader(fullname, self) - if self._module_matches_namespace(fullname) - else None - ) - - def install(self): - """ - Install this importer into sys.meta_path if not already present. - """ - if self not in sys.meta_path: - sys.meta_path.append(self) - - -# [[[cog -# import cog -# from tools.vendored import yield_top_level -# names = "\n".join(f" {x!r}," for x in yield_top_level('setuptools')) -# cog.outl(f"names = (\n{names}\n)") -# ]]] -names = ( - 'backports', - 'importlib_metadata', - 'importlib_resources', - 'jaraco', - 'more_itertools', - 'ordered_set', - 'packaging', - 'tomli', - 'wheel', - 'zipp', -) -# [[[end]]] -VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/contrib/python/setuptools/py3/setuptools/msvc.py b/contrib/python/setuptools/py3/setuptools/msvc.py index a3d350fe509..2768059213f 100644 --- a/contrib/python/setuptools/py3/setuptools/msvc.py +++ b/contrib/python/setuptools/py3/setuptools/msvc.py @@ -23,7 +23,8 @@ import itertools import subprocess import distutils.errors from typing import TYPE_CHECKING -from setuptools.extern.more_itertools import unique_everseen + +from more_itertools import unique_everseen # https://github.com/python/mypy/issues/8166 if not TYPE_CHECKING and platform.system() == 'Windows': diff --git a/contrib/python/setuptools/py3/setuptools/package_index.py b/contrib/python/setuptools/py3/setuptools/package_index.py index 2c807f6b4e6..c24c783762b 100644 --- a/contrib/python/setuptools/py3/setuptools/package_index.py +++ b/contrib/python/setuptools/py3/setuptools/package_index.py @@ -39,7 +39,8 @@ from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.wheel import Wheel -from setuptools.extern.more_itertools import unique_everseen + +from more_itertools import unique_everseen from .unicode_utils import _read_utf8_with_fallback, _cfg_read_utf8_with_fallback diff --git a/contrib/python/setuptools/py3/setuptools/warnings.py b/contrib/python/setuptools/py3/setuptools/warnings.py index 5d9cca6c375..8c94bc96e60 100644 --- a/contrib/python/setuptools/py3/setuptools/warnings.py +++ b/contrib/python/setuptools/py3/setuptools/warnings.py @@ -32,7 +32,7 @@ class SetuptoolsWarning(UserWarning): see_url: str | None = None, stacklevel: int = 2, **kwargs, - ): + ) -> None: """Private: reserved for ``setuptools`` internal use only""" # Default values: summary_ = summary or getattr(cls, "_SUMMARY", None) or "" @@ -56,7 +56,7 @@ class SetuptoolsWarning(UserWarning): due_date: date | None = None, see_url: str | None = None, format_args: dict | None = None, - ): + ) -> str: """Private: reserved for ``setuptools`` internal use only""" today = date.today() summary = cleandoc(summary).format_map(format_args or {}) diff --git a/contrib/python/setuptools/py3/setuptools/wheel.py b/contrib/python/setuptools/py3/setuptools/wheel.py index e06daec4d0e..a05cd98d1f5 100644 --- a/contrib/python/setuptools/py3/setuptools/wheel.py +++ b/contrib/python/setuptools/py3/setuptools/wheel.py @@ -9,12 +9,13 @@ import re import zipfile import contextlib +from packaging.version import Version as parse_version +from packaging.tags import sys_tags +from packaging.utils import canonicalize_name + from distutils.util import get_platform import setuptools -from setuptools.extern.packaging.version import Version as parse_version -from setuptools.extern.packaging.tags import sys_tags -from setuptools.extern.packaging.utils import canonicalize_name from setuptools.command.egg_info import write_requirements, _egg_basename from setuptools.archive_util import _unpack_zipfile_obj diff --git a/contrib/python/setuptools/py3/ya.make b/contrib/python/setuptools/py3/ya.make index 38047edfe40..189d7113414 100644 --- a/contrib/python/setuptools/py3/ya.make +++ b/contrib/python/setuptools/py3/ya.make @@ -2,11 +2,21 @@ PY3_LIBRARY() -VERSION(70.3.0) +VERSION(71.1.0) LICENSE(MIT) PEERDIR( + contrib/python/jaraco.context + contrib/python/jaraco.functools + contrib/python/jaraco.text + contrib/python/more-itertools + contrib/python/ordered-set + contrib/python/packaging + contrib/python/platformdirs + contrib/python/typeguard + contrib/python/typing-extensions + contrib/python/wheel library/python/resource ) @@ -23,53 +33,6 @@ PY_SRCS( _distutils_hack/__init__.py _distutils_hack/override.py pkg_resources/__init__.py - pkg_resources/_vendor/__init__.py - pkg_resources/_vendor/backports/__init__.py - pkg_resources/_vendor/backports/tarfile.py - pkg_resources/_vendor/importlib_resources/__init__.py - pkg_resources/_vendor/importlib_resources/_adapters.py - pkg_resources/_vendor/importlib_resources/_common.py - pkg_resources/_vendor/importlib_resources/_compat.py - pkg_resources/_vendor/importlib_resources/_itertools.py - pkg_resources/_vendor/importlib_resources/_legacy.py - pkg_resources/_vendor/importlib_resources/abc.py - pkg_resources/_vendor/importlib_resources/readers.py - pkg_resources/_vendor/importlib_resources/simple.py - pkg_resources/_vendor/jaraco/__init__.py - pkg_resources/_vendor/jaraco/context.py - pkg_resources/_vendor/jaraco/functools/__init__.py - pkg_resources/_vendor/jaraco/functools/__init__.pyi - pkg_resources/_vendor/jaraco/text/__init__.py - pkg_resources/_vendor/more_itertools/__init__.py - pkg_resources/_vendor/more_itertools/__init__.pyi - pkg_resources/_vendor/more_itertools/more.py - pkg_resources/_vendor/more_itertools/more.pyi - pkg_resources/_vendor/more_itertools/recipes.py - pkg_resources/_vendor/more_itertools/recipes.pyi - pkg_resources/_vendor/packaging/__init__.py - pkg_resources/_vendor/packaging/_elffile.py - pkg_resources/_vendor/packaging/_manylinux.py - pkg_resources/_vendor/packaging/_musllinux.py - pkg_resources/_vendor/packaging/_parser.py - pkg_resources/_vendor/packaging/_structures.py - pkg_resources/_vendor/packaging/_tokenizer.py - pkg_resources/_vendor/packaging/markers.py - pkg_resources/_vendor/packaging/metadata.py - pkg_resources/_vendor/packaging/requirements.py - pkg_resources/_vendor/packaging/specifiers.py - pkg_resources/_vendor/packaging/tags.py - pkg_resources/_vendor/packaging/utils.py - pkg_resources/_vendor/packaging/version.py - pkg_resources/_vendor/platformdirs/__init__.py - pkg_resources/_vendor/platformdirs/__main__.py - pkg_resources/_vendor/platformdirs/android.py - pkg_resources/_vendor/platformdirs/api.py - pkg_resources/_vendor/platformdirs/macos.py - pkg_resources/_vendor/platformdirs/unix.py - pkg_resources/_vendor/platformdirs/version.py - pkg_resources/_vendor/platformdirs/windows.py - pkg_resources/_vendor/zipp.py - pkg_resources/extern/__init__.py setuptools/__init__.py setuptools/_core_metadata.py setuptools/_distutils/__init__.py @@ -154,63 +117,6 @@ PY_SRCS( setuptools/_normalization.py setuptools/_path.py setuptools/_reqs.py - setuptools/_vendor/__init__.py - setuptools/_vendor/backports/__init__.py - setuptools/_vendor/backports/tarfile.py - setuptools/_vendor/importlib_metadata/__init__.py - setuptools/_vendor/importlib_metadata/_adapters.py - setuptools/_vendor/importlib_metadata/_collections.py - setuptools/_vendor/importlib_metadata/_compat.py - setuptools/_vendor/importlib_metadata/_functools.py - setuptools/_vendor/importlib_metadata/_itertools.py - setuptools/_vendor/importlib_metadata/_meta.py - setuptools/_vendor/importlib_metadata/_py39compat.py - setuptools/_vendor/importlib_metadata/_text.py - setuptools/_vendor/importlib_resources/__init__.py - setuptools/_vendor/importlib_resources/_adapters.py - setuptools/_vendor/importlib_resources/_common.py - setuptools/_vendor/importlib_resources/_compat.py - setuptools/_vendor/importlib_resources/_itertools.py - setuptools/_vendor/importlib_resources/_legacy.py - setuptools/_vendor/importlib_resources/abc.py - setuptools/_vendor/importlib_resources/readers.py - setuptools/_vendor/importlib_resources/simple.py - setuptools/_vendor/jaraco/__init__.py - setuptools/_vendor/jaraco/context.py - setuptools/_vendor/jaraco/functools/__init__.py - setuptools/_vendor/jaraco/functools/__init__.pyi - setuptools/_vendor/jaraco/text/__init__.py - setuptools/_vendor/more_itertools/__init__.py - setuptools/_vendor/more_itertools/__init__.pyi - setuptools/_vendor/more_itertools/more.py - setuptools/_vendor/more_itertools/more.pyi - setuptools/_vendor/more_itertools/recipes.py - setuptools/_vendor/more_itertools/recipes.pyi - setuptools/_vendor/ordered_set.py - setuptools/_vendor/packaging/__init__.py - setuptools/_vendor/packaging/_elffile.py - setuptools/_vendor/packaging/_manylinux.py - setuptools/_vendor/packaging/_musllinux.py - setuptools/_vendor/packaging/_parser.py - setuptools/_vendor/packaging/_structures.py - setuptools/_vendor/packaging/_tokenizer.py - setuptools/_vendor/packaging/markers.py - setuptools/_vendor/packaging/metadata.py - setuptools/_vendor/packaging/requirements.py - setuptools/_vendor/packaging/specifiers.py - setuptools/_vendor/packaging/tags.py - setuptools/_vendor/packaging/utils.py - setuptools/_vendor/packaging/version.py - setuptools/_vendor/tomli/__init__.py - setuptools/_vendor/tomli/_parser.py - setuptools/_vendor/tomli/_re.py - setuptools/_vendor/tomli/_types.py - setuptools/_vendor/wheel/__init__.py - setuptools/_vendor/wheel/macosx_libfile.py - setuptools/_vendor/wheel/metadata.py - setuptools/_vendor/wheel/util.py - setuptools/_vendor/wheel/wheelfile.py - setuptools/_vendor/zipp.py setuptools/archive_util.py setuptools/build_meta.py setuptools/command/__init__.py @@ -260,7 +166,6 @@ PY_SRCS( setuptools/dist.py setuptools/errors.py setuptools/extension.py - setuptools/extern/__init__.py setuptools/glob.py setuptools/installer.py setuptools/launch.py @@ -283,18 +188,9 @@ RESOURCE_FILES( .dist-info/METADATA .dist-info/entry_points.txt .dist-info/top_level.txt - pkg_resources/_vendor/importlib_resources/py.typed - pkg_resources/_vendor/jaraco/functools/py.typed - pkg_resources/_vendor/more_itertools/py.typed - pkg_resources/_vendor/packaging/py.typed - pkg_resources/_vendor/platformdirs/py.typed + pkg_resources/api_tests.txt + pkg_resources/py.typed setuptools/_distutils/_vendor/packaging/py.typed - setuptools/_vendor/importlib_metadata/py.typed - setuptools/_vendor/importlib_resources/py.typed - setuptools/_vendor/jaraco/functools/py.typed - setuptools/_vendor/more_itertools/py.typed - setuptools/_vendor/packaging/py.typed - setuptools/_vendor/tomli/py.typed setuptools/script.tmpl ) diff --git a/contrib/python/typeguard/.dist-info/METADATA b/contrib/python/typeguard/.dist-info/METADATA new file mode 100644 index 00000000000..6ca2152dd5a --- /dev/null +++ b/contrib/python/typeguard/.dist-info/METADATA @@ -0,0 +1,77 @@ +Metadata-Version: 2.1 +Name: typeguard +Version: 2.13.3 +Summary: Run-time type checker for Python +Home-page: UNKNOWN +Author: Alex Grönholm +Author-email: alex.gronholm@nextday.fi +License: MIT +Project-URL: Documentation, https://typeguard.readthedocs.io/en/latest/ +Project-URL: Change log, https://typeguard.readthedocs.io/en/latest/versionhistory.html +Project-URL: Source code, https://github.com/agronholm/typeguard +Project-URL: Issue tracker, https://github.com/agronholm/typeguard/issues +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Requires-Python: >=3.5.3 +License-File: LICENSE +Provides-Extra: doc +Requires-Dist: sphinx-rtd-theme ; extra == 'doc' +Requires-Dist: sphinx-autodoc-typehints (>=1.2.0) ; extra == 'doc' +Provides-Extra: test +Requires-Dist: pytest ; extra == 'test' +Requires-Dist: typing-extensions ; extra == 'test' +Requires-Dist: mypy ; (platform_python_implementation != "PyPy") and extra == 'test' + +.. image:: https://travis-ci.com/agronholm/typeguard.svg?branch=master + :target: https://travis-ci.com/agronholm/typeguard + :alt: Build Status +.. image:: https://coveralls.io/repos/agronholm/typeguard/badge.svg?branch=master&service=github + :target: https://coveralls.io/github/agronholm/typeguard?branch=master + :alt: Code Coverage +.. image:: https://readthedocs.org/projects/typeguard/badge/?version=latest + :target: https://typeguard.readthedocs.io/en/latest/?badge=latest + +This library provides run-time type checking for functions defined with +`PEP 484 <https://www.python.org/dev/peps/pep-0484/>`_ argument (and return) type annotations. + +Four principal ways to do type checking are provided, each with its pros and cons: + +#. the ``check_argument_types()`` and ``check_return_type()`` functions: + + * debugger friendly (except when running with the pydev debugger with the C extension installed) + * does not work reliably with dynamically defined type hints (e.g. in nested functions) +#. the ``@typechecked`` decorator: + + * automatically type checks yields and sends of returned generators (regular and async) + * adds an extra frame to the call stack for every call to a decorated function +#. the stack profiler hook (``with TypeChecker('packagename'):``) (deprecated): + + * emits warnings instead of raising ``TypeError`` + * requires very few modifications to the code + * multiple TypeCheckers can be stacked/nested + * does not work reliably with dynamically defined type hints (e.g. in nested functions) + * may cause problems with badly behaving debuggers or profilers + * cannot distinguish between an exception being raised and a ``None`` being returned +#. the import hook (``typeguard.importhook.install_import_hook()``): + + * automatically annotates classes and functions with ``@typechecked`` on import + * no code changes required in target modules + * requires imports of modules you need to check to be deferred until after the import hook has + been installed + * may clash with other import hooks + +See the documentation_ for further instructions. + +.. _documentation: https://typeguard.readthedocs.io/en/latest/ + + diff --git a/contrib/python/typeguard/.dist-info/entry_points.txt b/contrib/python/typeguard/.dist-info/entry_points.txt new file mode 100644 index 00000000000..0bde2f50de5 --- /dev/null +++ b/contrib/python/typeguard/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[pytest11] +typeguard = typeguard.pytest_plugin + diff --git a/contrib/python/typeguard/.dist-info/top_level.txt b/contrib/python/typeguard/.dist-info/top_level.txt new file mode 100644 index 00000000000..be5ec23ea20 --- /dev/null +++ b/contrib/python/typeguard/.dist-info/top_level.txt @@ -0,0 +1 @@ +typeguard diff --git a/contrib/python/typeguard/LICENSE b/contrib/python/typeguard/LICENSE new file mode 100644 index 00000000000..07806f8af9d --- /dev/null +++ b/contrib/python/typeguard/LICENSE @@ -0,0 +1,19 @@ +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright (c) Alex Grönholm + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/typeguard/README.rst b/contrib/python/typeguard/README.rst new file mode 100644 index 00000000000..6d2ed01bac4 --- /dev/null +++ b/contrib/python/typeguard/README.rst @@ -0,0 +1,41 @@ +.. image:: https://travis-ci.com/agronholm/typeguard.svg?branch=master + :target: https://travis-ci.com/agronholm/typeguard + :alt: Build Status +.. image:: https://coveralls.io/repos/agronholm/typeguard/badge.svg?branch=master&service=github + :target: https://coveralls.io/github/agronholm/typeguard?branch=master + :alt: Code Coverage +.. image:: https://readthedocs.org/projects/typeguard/badge/?version=latest + :target: https://typeguard.readthedocs.io/en/latest/?badge=latest + +This library provides run-time type checking for functions defined with +`PEP 484 <https://www.python.org/dev/peps/pep-0484/>`_ argument (and return) type annotations. + +Four principal ways to do type checking are provided, each with its pros and cons: + +#. the ``check_argument_types()`` and ``check_return_type()`` functions: + + * debugger friendly (except when running with the pydev debugger with the C extension installed) + * does not work reliably with dynamically defined type hints (e.g. in nested functions) +#. the ``@typechecked`` decorator: + + * automatically type checks yields and sends of returned generators (regular and async) + * adds an extra frame to the call stack for every call to a decorated function +#. the stack profiler hook (``with TypeChecker('packagename'):``) (deprecated): + + * emits warnings instead of raising ``TypeError`` + * requires very few modifications to the code + * multiple TypeCheckers can be stacked/nested + * does not work reliably with dynamically defined type hints (e.g. in nested functions) + * may cause problems with badly behaving debuggers or profilers + * cannot distinguish between an exception being raised and a ``None`` being returned +#. the import hook (``typeguard.importhook.install_import_hook()``): + + * automatically annotates classes and functions with ``@typechecked`` on import + * no code changes required in target modules + * requires imports of modules you need to check to be deferred until after the import hook has + been installed + * may clash with other import hooks + +See the documentation_ for further instructions. + +.. _documentation: https://typeguard.readthedocs.io/en/latest/ diff --git a/contrib/python/typeguard/patches/01-fix-tests.patch b/contrib/python/typeguard/patches/01-fix-tests.patch new file mode 100644 index 00000000000..f012dca6207 --- /dev/null +++ b/contrib/python/typeguard/patches/01-fix-tests.patch @@ -0,0 +1,42 @@ +--- contrib/python/typeguard/tests/test_importhook.py (index) ++++ contrib/python/typeguard/tests/test_importhook.py (working tree) +@@ -8,7 +8,9 @@ import pytest + + from typeguard.importhook import TypeguardFinder, install_import_hook + +-this_dir = Path(__file__).parent ++import yatest.common as yc ++ ++this_dir = Path(yc.test_source_path()) + dummy_module_path = this_dir / 'dummymodule.py' + cached_module_path = Path(cache_from_source(str(dummy_module_path), optimization='typeguard')) + +@@ -29,6 +31,7 @@ def dummymodule(): + sys.path.remove(str(this_dir)) + + ++@pytest.mark.skip + def test_cached_module(dummymodule): + assert cached_module_path.is_file() + +--- contrib/python/typeguard/tests/test_typeguard.py (index) ++++ contrib/python/typeguard/tests/test_typeguard.py (working tree) +@@ -84,1 +84,1 @@ def mock_class(request): +- (Child(), 'test_typeguard.Child'), ++ (Child(), '__tests__.test_typeguard.Child'), +@@ -467,1 +467,1 @@ class TestCheckArgumentTypes: +- r'(test_typeguard\.)?Employee; got tuple instead') ++ r'(__tests__\.test_typeguard\.)?Employee; got tuple instead') +@@ -547,2 +547,2 @@ class TestCheckArgumentTypes: +- assert str(exc.value) == ('type of argument "a" must be test_typeguard.Child or one of ' +- 'its subclasses; got test_typeguard.Parent instead') ++ assert str(exc.value) == ('type of argument "a" must be __tests__.test_typeguard.Child or one of ' ++ 'its subclasses; got __tests__.test_typeguard.Parent instead') +@@ -556,1 +556,1 @@ class TestCheckArgumentTypes: +- '"a" must be a subclass of test_typeguard.Child; got test_typeguard.Parent instead') ++ '"a" must be a subclass of __tests__.test_typeguard.Child; got __tests__.test_typeguard.Parent instead') +@@ -585,2 +585,2 @@ class TestCheckArgumentTypes: +- assert str(exc.value) == ('type of argument "a" must be test_typeguard.Child; ' +- 'got test_typeguard.Parent instead') ++ assert str(exc.value) == ('type of argument "a" must be __tests__.test_typeguard.Child; ' ++ 'got __tests__.test_typeguard.Parent instead') diff --git a/contrib/python/typeguard/patches/02-support-new-typing-extensions.patch b/contrib/python/typeguard/patches/02-support-new-typing-extensions.patch new file mode 100644 index 00000000000..b296e06bd0b --- /dev/null +++ b/contrib/python/typeguard/patches/02-support-new-typing-extensions.patch @@ -0,0 +1,16 @@ +--- contrib/python/typeguard/tests/test_typeguard.py (index) ++++ contrib/python/typeguard/tests/test_typeguard.py (working tree) +@@ -8,11 +8,11 @@ from functools import lru_cache, partial, wraps + from io import BytesIO, StringIO + from typing import ( + AbstractSet, Any, AnyStr, BinaryIO, Callable, Container, Dict, Generator, Generic, Iterable, +- Iterator, List, NamedTuple, Sequence, Set, TextIO, Tuple, Type, TypeVar, Union) ++ Iterator, List, NamedTuple, Sequence, Set, TextIO, Tuple, Type, TypeVar, Union, TypedDict) + from unittest.mock import MagicMock, Mock + + import pytest +-from typing_extensions import Literal, NoReturn, Protocol, TypedDict, runtime_checkable ++from typing_extensions import Literal, NoReturn, Protocol, runtime_checkable + + from typeguard import ( + ForwardRefPolicy, TypeChecker, TypeHintWarning, TypeWarning, check_argument_types, diff --git a/contrib/python/typeguard/patches/03-support-python-3.12.patch b/contrib/python/typeguard/patches/03-support-python-3.12.patch new file mode 100644 index 00000000000..d0432dd0d3f --- /dev/null +++ b/contrib/python/typeguard/patches/03-support-python-3.12.patch @@ -0,0 +1,15 @@ +--- contrib/python/typeguard/typeguard/importhook.py (index) ++++ contrib/python/typeguard/typeguard/importhook.py (working tree) +@@ -29 +29 @@ class TypeguardTransformer(ast.NodeVisitor): +- elif isinstance(child, ast.Expr) and isinstance(child.value, ast.Str): ++ elif isinstance(child, ast.Expr) and isinstance(child.value, ast.Constant): +--- contrib/python/typeguard/tests/test_typeguard.py (index) ++++ contrib/python/typeguard/tests/test_typeguard.py (working tree) +@@ -1227,6 +1227,7 @@ class TestTypeChecked: + def foo(): + pass + ++ @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Fail wint Python 3.12") + @pytest.mark.parametrize('annotation', [TBound, TConstrained], ids=['bound', 'constrained']) + def test_typevar_forwardref(self, annotation): + @typechecked diff --git a/contrib/python/typeguard/patches/04-support-python-3.12.4.patch b/contrib/python/typeguard/patches/04-support-python-3.12.4.patch new file mode 100644 index 00000000000..bed273464aa --- /dev/null +++ b/contrib/python/typeguard/patches/04-support-python-3.12.4.patch @@ -0,0 +1,11 @@ +--- contrib/python/typeguard/typeguard/__init__.py (index) ++++ contrib/python/typeguard/typeguard/__init__.py (working tree) +@@ -265,7 +265,7 @@ def resolve_forwardref(maybe_ref, memo: _TypeCheckMemo): + if sys.version_info < (3, 9, 0): + return evaluate_forwardref(maybe_ref, memo.globals, memo.locals) + else: +- return evaluate_forwardref(maybe_ref, memo.globals, memo.locals, frozenset()) ++ return evaluate_forwardref(maybe_ref, memo.globals, memo.locals, recursive_guard=frozenset()) + + else: + return maybe_ref diff --git a/contrib/python/typeguard/tests/conftest.py b/contrib/python/typeguard/tests/conftest.py new file mode 100644 index 00000000000..2a3132bc4f7 --- /dev/null +++ b/contrib/python/typeguard/tests/conftest.py @@ -0,0 +1,12 @@ +import re +import sys + +version_re = re.compile(r'_py(\d)(\d)\.py$') + + +def pytest_ignore_collect(path, config): + match = version_re.search(path.basename) + if match: + version = tuple(int(x) for x in match.groups()) + if sys.version_info < version: + return True diff --git a/contrib/python/typeguard/tests/dummymodule.py b/contrib/python/typeguard/tests/dummymodule.py new file mode 100644 index 00000000000..7578976a1a7 --- /dev/null +++ b/contrib/python/typeguard/tests/dummymodule.py @@ -0,0 +1,92 @@ +"""Module docstring.""" +from __future__ import absolute_import, division + +from typing import no_type_check, no_type_check_decorator + +from typeguard import typeguard_ignore + + +@no_type_check_decorator +def dummy_decorator(func): + return func + + +def type_checked_func(x: int, y: int) -> int: + return x * y + + +@no_type_check +def non_type_checked_func(x: int, y: str) -> 6: + return 'foo' + + +@dummy_decorator +def non_type_checked_decorated_func(x: int, y: str) -> 6: + return 'foo' + + +@typeguard_ignore +def non_typeguard_checked_func(x: int, y: str) -> 6: + return 'foo' + + +def dynamic_type_checking_func(arg, argtype, return_annotation): + def inner(x: argtype) -> return_annotation: + return str(x) + + return inner(arg) + + +class Metaclass(type): + pass + + +class DummyClass(metaclass=Metaclass): + def type_checked_method(self, x: int, y: int) -> int: + return x * y + + @classmethod + def type_checked_classmethod(cls, x: int, y: int) -> int: + return x * y + + @staticmethod + def type_checked_staticmethod(x: int, y: int) -> int: + return x * y + + @classmethod + def undocumented_classmethod(cls, x, y): + pass + + @staticmethod + def undocumented_staticmethod(x, y): + pass + + @property + def unannotated_property(self): + return None + + +def outer(): + class Inner: + pass + + def create_inner() -> 'Inner': + return Inner() + + return create_inner + + +class Outer: + class Inner: + pass + + def create_inner(self) -> 'Inner': + return Outer.Inner() + + @classmethod + def create_inner_classmethod(cls) -> 'Inner': + return Outer.Inner() + + @staticmethod + def create_inner_staticmethod() -> 'Inner': + return Outer.Inner() diff --git a/contrib/python/typeguard/tests/mypy/negative.py b/contrib/python/typeguard/tests/mypy/negative.py new file mode 100644 index 00000000000..6db0eb2a35b --- /dev/null +++ b/contrib/python/typeguard/tests/mypy/negative.py @@ -0,0 +1,53 @@ +from typeguard import check_argument_types, check_return_type, typechecked, typeguard_ignore + + +@typechecked +def foo(x: int) -> int: + return x + 1 + + +@typechecked +def bar(x: int) -> int: + return str(x) # error: Incompatible return value type (got "str", expected "int") + + +@typeguard_ignore +def non_typeguard_checked_func(x: int) -> int: + return str(x) # error: Incompatible return value type (got "str", expected "int") + + +def returns_str() -> str: + return bar(0) # error: Incompatible return value type (got "int", expected "str") + + +def arg_type(x: int) -> str: + return check_argument_types() # noqa: E501 # error: Incompatible return value type (got "bool", expected "str") + + +def ret_type() -> str: + return check_return_type(False) # noqa: E501 # error: Incompatible return value type (got "bool", expected "str") + + +_ = arg_type(foo) # noqa: E501 # error: Argument 1 to "arg_type" has incompatible type "Callable[[int], int]"; expected "int" +_ = foo("typeguard") # error: Argument 1 to "foo" has incompatible type "str"; expected "int" + + +@typechecked +class MyClass: + def __init__(self, x: int = 0) -> None: + self.x = x + + def add(self, y: int) -> int: + return self.x + y + + +def get_value(c: MyClass) -> int: + return c.x + + +def create_myclass(x: int) -> MyClass: + return MyClass(x) + + +_ = get_value("foo") # noqa: E501 # error: Argument 1 to "get_value" has incompatible type "str"; expected "MyClass" +_ = MyClass(returns_str()) # noqa: E501 # error: Argument 1 to "MyClass" has incompatible type "str"; expected "int" diff --git a/contrib/python/typeguard/tests/mypy/positive.py b/contrib/python/typeguard/tests/mypy/positive.py new file mode 100644 index 00000000000..2f01bebf362 --- /dev/null +++ b/contrib/python/typeguard/tests/mypy/positive.py @@ -0,0 +1,56 @@ +from typing import Callable + +from typeguard import check_argument_types, check_return_type, typechecked + + +@typechecked +def foo(x: str) -> str: + return "hello " + x + + +def takes_callable(f: Callable[[str], str]) -> str: + return f("typeguard") + + +takes_callable(foo) + + +def has_valid_arguments(x: int, y: str) -> bool: + return check_argument_types() + + +def has_valid_return_type(y: str) -> str: + check_return_type(y) + return y + + +@typechecked +class MyClass: + + def __init__(self, x: int) -> None: + self.x = x + + def add(self, y: int) -> int: + return self.x + y + + +def get_value(c: MyClass) -> int: + return c.x + + +@typechecked +def get_value_checked(c: MyClass) -> int: + return c.x + + +def create_myclass(x: int) -> MyClass: + return MyClass(x) + + +@typechecked +def create_myclass_checked(x: int) -> MyClass: + return MyClass(x) + + +get_value(create_myclass(3)) +get_value_checked(create_myclass_checked(1)) diff --git a/contrib/python/typeguard/tests/mypy/test_type_annotations.py b/contrib/python/typeguard/tests/mypy/test_type_annotations.py new file mode 100644 index 00000000000..50ddc50686e --- /dev/null +++ b/contrib/python/typeguard/tests/mypy/test_type_annotations.py @@ -0,0 +1,114 @@ +import os +import platform +import re +import subprocess +from typing import Dict, List + +import pytest + +POSITIVE_FILE = "positive.py" +NEGATIVE_FILE = "negative.py" +LINE_PATTERN = NEGATIVE_FILE + ":([0-9]+):" + +pytestmark = [pytest.mark.skipif(platform.python_implementation() == 'PyPy', + reason='MyPy does not work with PyPy yet')] + + +def get_mypy_cmd(filename: str) -> List[str]: + return ["mypy", "--strict", filename] + + +def get_negative_mypy_output() -> str: + """ + Get the output from running mypy on the negative examples file. + """ + process = subprocess.run( + get_mypy_cmd(NEGATIVE_FILE), stdout=subprocess.PIPE, check=False + ) + output = process.stdout.decode() + assert output + return output + + +def get_expected_errors() -> Dict[int, str]: + """ + Extract the expected errors from comments in the negative examples file. + """ + with open(NEGATIVE_FILE) as f: + lines = f.readlines() + + expected = {} + + for idx, line in enumerate(lines): + line = line.rstrip() + if "# error" in line: + expected[idx + 1] = line[line.index("# error") + 2:] + + # Sanity check. Should update if negative.py changes. + assert len(expected) == 9 + return expected + + +def get_mypy_errors() -> Dict[int, str]: + """ + Extract the errors from running mypy on the negative examples file. + """ + mypy_output = get_negative_mypy_output() + + got = {} + for line in mypy_output.splitlines(): + m = re.match(LINE_PATTERN, line) + if m is None: + continue + got[int(m.group(1))] = line[len(m.group(0)) + 1:] + + return got + + +@pytest.fixture +def chdir_local() -> None: + """ + Change to the local directory. This is so that mypy treats imports from + typeguard as external imports instead of source code (which is handled + differently by mypy). + """ + os.chdir(os.path.dirname(__file__)) + + +@pytest.mark.usefixtures("chdir_local") +def test_positive() -> None: + """ + Run mypy on the positive test file. There should be no errors. + """ + subprocess.check_call(get_mypy_cmd(POSITIVE_FILE)) + + +@pytest.mark.usefixtures("chdir_local") +def test_negative() -> None: + """ + Run mypy on the negative test file. This should fail. The errors from mypy + should match the comments in the file. + """ + got_errors = get_mypy_errors() + expected_errors = get_expected_errors() + + if set(got_errors) != set(expected_errors): + raise RuntimeError( + "Expected error lines {} does not ".format(set(expected_errors)) + + "match mypy error lines {}.".format(set(got_errors)) + ) + + mismatches = [ + (idx, expected_errors[idx], got_errors[idx]) + for idx in expected_errors + if expected_errors[idx] != got_errors[idx] + ] + for (idx, expected, got) in mismatches: + print( + "Line {}".format(idx), + "Expected: {}".format(expected), + "Got: {}".format(got), + sep="\n\t" + ) + if mismatches: + raise RuntimeError("Error messages changed") diff --git a/contrib/python/typeguard/tests/test_importhook.py b/contrib/python/typeguard/tests/test_importhook.py new file mode 100644 index 00000000000..b0f28278da0 --- /dev/null +++ b/contrib/python/typeguard/tests/test_importhook.py @@ -0,0 +1,125 @@ +import sys +import warnings +from importlib import import_module +from importlib.util import cache_from_source +from pathlib import Path + +import pytest + +from typeguard.importhook import TypeguardFinder, install_import_hook + +import yatest.common as yc + +this_dir = Path(yc.test_source_path()) +dummy_module_path = this_dir / 'dummymodule.py' +cached_module_path = Path(cache_from_source(str(dummy_module_path), optimization='typeguard')) + + +@pytest.fixture(scope='module') +def dummymodule(): + if cached_module_path.exists(): + cached_module_path.unlink() + + sys.path.insert(0, str(this_dir)) + try: + with install_import_hook('dummymodule'): + with warnings.catch_warnings(): + warnings.filterwarnings('error', module='typeguard') + module = import_module('dummymodule') + return module + finally: + sys.path.remove(str(this_dir)) + + +@pytest.mark.skip +def test_cached_module(dummymodule): + assert cached_module_path.is_file() + + +def test_type_checked_func(dummymodule): + assert dummymodule.type_checked_func(2, 3) == 6 + + +def test_type_checked_func_error(dummymodule): + pytest.raises(TypeError, dummymodule.type_checked_func, 2, '3').\ + match('"y" must be int; got str instead') + + +def test_non_type_checked_func(dummymodule): + assert dummymodule.non_type_checked_func('bah', 9) == 'foo' + + +def test_non_type_checked_decorated_func(dummymodule): + assert dummymodule.non_type_checked_decorated_func('bah', 9) == 'foo' + + +def test_typeguard_ignored_func(dummymodule): + assert dummymodule.non_typeguard_checked_func('bah', 9) == 'foo' + + +def test_type_checked_method(dummymodule): + instance = dummymodule.DummyClass() + pytest.raises(TypeError, instance.type_checked_method, 'bah', 9).\ + match('"x" must be int; got str instead') + + +def test_type_checked_classmethod(dummymodule): + pytest.raises(TypeError, dummymodule.DummyClass.type_checked_classmethod, 'bah', 9).\ + match('"x" must be int; got str instead') + + +def test_type_checked_staticmethod(dummymodule): + pytest.raises(TypeError, dummymodule.DummyClass.type_checked_classmethod, 'bah', 9).\ + match('"x" must be int; got str instead') + + +@pytest.mark.parametrize('argtype, returntype, error', [ + (int, str, None), + (str, str, '"x" must be str; got int instead'), + (int, int, 'type of the return value must be int; got str instead') +], ids=['correct', 'bad_argtype', 'bad_returntype']) +def test_dynamic_type_checking_func(dummymodule, argtype, returntype, error): + if error: + exc = pytest.raises(TypeError, dummymodule.dynamic_type_checking_func, 4, argtype, + returntype) + exc.match(error) + else: + assert dummymodule.dynamic_type_checking_func(4, argtype, returntype) == '4' + + +def test_class_in_function(dummymodule): + create_inner = dummymodule.outer() + retval = create_inner() + assert retval.__class__.__qualname__ == 'outer.<locals>.Inner' + + +def test_inner_class_method(dummymodule): + retval = dummymodule.Outer().create_inner() + assert retval.__class__.__qualname__ == 'Outer.Inner' + + +def test_inner_class_classmethod(dummymodule): + retval = dummymodule.Outer.create_inner_classmethod() + assert retval.__class__.__qualname__ == 'Outer.Inner' + + +def test_inner_class_staticmethod(dummymodule): + retval = dummymodule.Outer.create_inner_staticmethod() + assert retval.__class__.__qualname__ == 'Outer.Inner' + + +def test_package_name_matching(): + """ + The path finder only matches configured (sub)packages. + """ + packages = ["ham", "spam.eggs"] + dummy_original_pathfinder = None + finder = TypeguardFinder(packages, dummy_original_pathfinder) + + assert finder.should_instrument("ham") + assert finder.should_instrument("ham.eggs") + assert finder.should_instrument("spam.eggs") + + assert not finder.should_instrument("spam") + assert not finder.should_instrument("ha") + assert not finder.should_instrument("spam_eggs") diff --git a/contrib/python/typeguard/tests/test_typeguard.py b/contrib/python/typeguard/tests/test_typeguard.py new file mode 100644 index 00000000000..6e7e9cda8bb --- /dev/null +++ b/contrib/python/typeguard/tests/test_typeguard.py @@ -0,0 +1,1548 @@ +import gc +import sys +import traceback +import warnings +from abc import abstractproperty +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache, partial, wraps +from io import BytesIO, StringIO +from typing import ( + AbstractSet, Any, AnyStr, BinaryIO, Callable, Container, Dict, Generator, Generic, Iterable, + Iterator, List, NamedTuple, Sequence, Set, TextIO, Tuple, Type, TypeVar, Union, TypedDict) +from unittest.mock import MagicMock, Mock + +import pytest +from typing_extensions import Literal, NoReturn, Protocol, runtime_checkable + +from typeguard import ( + ForwardRefPolicy, TypeChecker, TypeHintWarning, TypeWarning, check_argument_types, + check_return_type, check_type, function_name, qualified_name, typechecked) + +try: + from typing import Collection +except ImportError: + # Python 3.6.0+ + Collection = None + +try: + from typing import NewType +except ImportError: + myint = None +else: + myint = NewType("myint", int) + + +TBound = TypeVar('TBound', bound='Parent') +TConstrained = TypeVar('TConstrained', 'Parent', int) +TTypingConstrained = TypeVar('TTypingConstrained', List[int], AbstractSet[str]) +TIntStr = TypeVar('TIntStr', int, str) +TIntCollection = TypeVar('TIntCollection', int, Collection) +TParent = TypeVar('TParent', bound='Parent') +TChild = TypeVar('TChild', bound='Child') +T_Foo = TypeVar('T_Foo') +JSONType = Union[str, int, float, bool, None, List['JSONType'], Dict[str, 'JSONType']] + +DummyDict = TypedDict('DummyDict', {'x': int}, total=False) +issue_42059 = pytest.mark.xfail(bool(DummyDict.__required_keys__), + reason='Fails due to upstream bug BPO-42059') +del DummyDict + +Employee = NamedTuple('Employee', [('name', str), ('id', int)]) + + +class FooGeneric(Generic[T_Foo]): + pass + + +class Parent: + pass + + +class Child(Parent): + def method(self, a: int): + pass + + +class StaticProtocol(Protocol): + def meth(self) -> None: + ... + + +@runtime_checkable +class RuntimeProtocol(Protocol): + def meth(self) -> None: + ... + + +@pytest.fixture(params=[Mock, MagicMock], ids=['mock', 'magicmock']) +def mock_class(request): + return request.param + + +@pytest.mark.parametrize('inputval, expected', [ + (qualified_name, 'function'), + (Child(), '__tests__.test_typeguard.Child'), + (int, 'int') +], ids=['func', 'instance', 'builtintype']) +def test_qualified_name(inputval, expected): + assert qualified_name(inputval) == expected + + +def test_function_name(): + assert function_name(function_name) == 'typeguard.function_name' + + +def test_check_type_no_memo(): + check_type('foo', [1], List[int]) + + +def test_check_type_bytes(): + pytest.raises(TypeError, check_type, 'foo', 7, bytes).\ + match(r'type of foo must be bytes-like; got int instead') + + +def test_check_type_no_memo_fail(): + pytest.raises(TypeError, check_type, 'foo', ['a'], List[int]).\ + match(r'type of foo\[0\] must be int; got str instead') + + +@pytest.mark.parametrize('value', ['bar', b'bar'], ids=['str', 'bytes']) +def test_check_type_anystr(value): + check_type('foo', value, AnyStr) + + +def test_check_type_anystr_fail(): + pytest.raises(TypeError, check_type, 'foo', int, AnyStr).\ + match(r'type of foo must match one of the constraints \(bytes, str\); got type instead') + + +def test_check_return_type(): + def foo() -> int: + assert check_return_type(0) + return 0 + + foo() + + +def test_check_return_type_fail(): + def foo() -> int: + assert check_return_type('foo') + return 1 + + pytest.raises(TypeError, foo).match('type of the return value must be int; got str instead') + + +def test_check_return_notimplemented(): + class Foo: + def __eq__(self, other) -> bool: + assert check_return_type(NotImplemented) + return NotImplemented + + assert Foo().__eq__(1) is NotImplemented + + +def test_check_recursive_type(): + check_type('foo', {'a': [1, 2, 3]}, JSONType) + pytest.raises(TypeError, check_type, 'foo', {'a': (1, 2, 3)}, JSONType, globals=globals()).\ + match(r'type of foo must be one of \(str, int, float, (bool, )?NoneType, ' + r'List\[JSONType\], Dict\[str, JSONType\]\); got dict instead') + + +def test_exec_no_namespace(): + from textwrap import dedent + + exec(dedent(""" + from typeguard import typechecked + + @typechecked + def f() -> None: + pass + + """), {}) + + +class TestCheckArgumentTypes: + def test_any_type(self): + def foo(a: Any): + assert check_argument_types() + + foo('aa') + + def test_mock_value(self, mock_class): + def foo(a: str, b: int, c: dict, d: Any) -> int: + assert check_argument_types() + + foo(mock_class(), mock_class(), mock_class(), mock_class()) + + def test_callable_exact_arg_count(self): + def foo(a: Callable[[int, str], int]): + assert check_argument_types() + + def some_callable(x: int, y: str) -> int: + pass + + foo(some_callable) + + def test_callable_bad_type(self): + def foo(a: Callable[..., int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == 'argument "a" must be a callable' + + def test_callable_too_few_arguments(self): + def foo(a: Callable[[int, str], int]): + assert check_argument_types() + + def some_callable(x: int) -> int: + pass + + exc = pytest.raises(TypeError, foo, some_callable) + assert str(exc.value) == ( + 'callable passed as argument "a" has too few arguments in its declaration; expected 2 ' + 'but 1 argument(s) declared') + + def test_callable_too_many_arguments(self): + def foo(a: Callable[[int, str], int]): + assert check_argument_types() + + def some_callable(x: int, y: str, z: float) -> int: + pass + + exc = pytest.raises(TypeError, foo, some_callable) + assert str(exc.value) == ( + 'callable passed as argument "a" has too many arguments in its declaration; expected ' + '2 but 3 argument(s) declared') + + def test_callable_mandatory_kwonlyargs(self): + def foo(a: Callable[[int, str], int]): + assert check_argument_types() + + def some_callable(x: int, y: str, *, z: float, bar: str) -> int: + pass + + exc = pytest.raises(TypeError, foo, some_callable) + assert str(exc.value) == ( + 'callable passed as argument "a" has mandatory keyword-only arguments in its ' + 'declaration: z, bar') + + def test_callable_class(self): + """ + Test that passing a class as a callable does not count the "self" argument "a"gainst the + ones declared in the Callable specification. + + """ + def foo(a: Callable[[int, str], Any]): + assert check_argument_types() + + class SomeClass: + def __init__(self, x: int, y: str): + pass + + foo(SomeClass) + + def test_callable_plain(self): + def foo(a: Callable): + assert check_argument_types() + + def callback(a): + pass + + foo(callback) + + def test_callable_partial_class(self): + """ + Test that passing a bound method as a callable does not count the "self" argument "a"gainst + the ones declared in the Callable specification. + + """ + def foo(a: Callable[[int], Any]): + assert check_argument_types() + + class SomeClass: + def __init__(self, x: int, y: str): + pass + + foo(partial(SomeClass, y='foo')) + + def test_callable_bound_method(self): + """ + Test that passing a bound method as a callable does not count the "self" argument "a"gainst + the ones declared in the Callable specification. + + """ + def foo(callback: Callable[[int], Any]): + assert check_argument_types() + + foo(Child().method) + + def test_callable_partial_bound_method(self): + """ + Test that passing a bound method as a callable does not count the "self" argument "a"gainst + the ones declared in the Callable specification. + + """ + def foo(callback: Callable[[], Any]): + assert check_argument_types() + + foo(partial(Child().method, 1)) + + def test_callable_defaults(self): + """ + Test that a callable having "too many" arguments don't raise an error if the extra + arguments have default values. + + """ + def foo(callback: Callable[[int, str], Any]): + assert check_argument_types() + + def some_callable(x: int, y: str, z: float = 1.2) -> int: + pass + + foo(some_callable) + + def test_callable_builtin(self): + """ + Test that checking a Callable annotation against a builtin callable does not raise an + error. + + """ + def foo(callback: Callable[[int], Any]): + assert check_argument_types() + + foo([].append) + + def test_dict_bad_type(self): + def foo(a: Dict[str, int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == ( + 'type of argument "a" must be a dict; got int instead') + + def test_dict_bad_key_type(self): + def foo(a: Dict[str, int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, {1: 2}) + assert str(exc.value) == 'type of keys of argument "a" must be str; got int instead' + + def test_dict_bad_value_type(self): + def foo(a: Dict[str, int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, {'x': 'a'}) + assert str(exc.value) == "type of argument \"a\"['x'] must be int; got str instead" + + def test_list_bad_type(self): + def foo(a: List[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == ( + 'type of argument "a" must be a list; got int instead') + + def test_list_bad_element(self): + def foo(a: List[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, [1, 2, 'bb']) + assert str(exc.value) == ( + 'type of argument "a"[2] must be int; got str instead') + + def test_sequence_bad_type(self): + def foo(a: Sequence[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == ( + 'type of argument "a" must be a sequence; got int instead') + + def test_sequence_bad_element(self): + def foo(a: Sequence[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, [1, 2, 'bb']) + assert str(exc.value) == ( + 'type of argument "a"[2] must be int; got str instead') + + def test_abstractset_custom_type(self): + class DummySet(AbstractSet[int]): + def __contains__(self, x: object) -> bool: + return x == 1 + + def __len__(self) -> int: + return 1 + + def __iter__(self) -> Iterator[int]: + yield 1 + + def foo(a: AbstractSet[int]): + assert check_argument_types() + + foo(DummySet()) + + def test_abstractset_bad_type(self): + def foo(a: AbstractSet[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == 'type of argument "a" must be a set; got int instead' + + def test_set_bad_type(self): + def foo(a: Set[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == 'type of argument "a" must be a set; got int instead' + + def test_abstractset_bad_element(self): + def foo(a: AbstractSet[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, {1, 2, 'bb'}) + assert str(exc.value) == ( + 'type of elements of argument "a" must be int; got str instead') + + def test_set_bad_element(self): + def foo(a: Set[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, {1, 2, 'bb'}) + assert str(exc.value) == ( + 'type of elements of argument "a" must be int; got str instead') + + def test_tuple_bad_type(self): + def foo(a: Tuple[int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 5) + assert str(exc.value) == ( + 'type of argument "a" must be a tuple; got int instead') + + def test_tuple_too_many_elements(self): + def foo(a: Tuple[int, str]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, (1, 'aa', 2)) + assert str(exc.value) == ('argument "a" has wrong number of elements (expected 2, got 3 ' + 'instead)') + + def test_tuple_too_few_elements(self): + def foo(a: Tuple[int, str]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, (1,)) + assert str(exc.value) == ('argument "a" has wrong number of elements (expected 2, got 1 ' + 'instead)') + + def test_tuple_bad_element(self): + def foo(a: Tuple[int, str]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, (1, 2)) + assert str(exc.value) == ( + 'type of argument "a"[1] must be str; got int instead') + + def test_tuple_ellipsis_bad_element(self): + def foo(a: Tuple[int, ...]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, (1, 2, 'blah')) + assert str(exc.value) == ( + 'type of argument "a"[2] must be int; got str instead') + + def test_namedtuple(self): + def foo(bar: Employee): + assert check_argument_types() + + foo(Employee('bob', 1)) + + def test_namedtuple_type_mismatch(self): + def foo(bar: Employee): + assert check_argument_types() + + pytest.raises(TypeError, foo, ('bob', 1)).\ + match('type of argument "bar" must be a named tuple of type ' + r'(__tests__\.test_typeguard\.)?Employee; got tuple instead') + + def test_namedtuple_wrong_field_type(self): + def foo(bar: Employee): + assert check_argument_types() + + pytest.raises(TypeError, foo, Employee(2, 1)).\ + match('type of argument "bar".name must be str; got int instead') + + @pytest.mark.parametrize('value', [6, 'aa']) + def test_union(self, value): + def foo(a: Union[str, int]): + assert check_argument_types() + + foo(value) + + def test_union_typing_type(self): + def foo(a: Union[str, Collection]): + assert check_argument_types() + + with pytest.raises(TypeError): + foo(1) + + @pytest.mark.parametrize('value', [6.5, b'aa']) + def test_union_fail(self, value): + def foo(a: Union[str, int]): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, value) + assert str(exc.value) == ( + 'type of argument "a" must be one of (str, int); got {} instead'. + format(value.__class__.__name__)) + + @pytest.mark.parametrize('values', [ + (6, 7), + ('aa', 'bb') + ], ids=['int', 'str']) + def test_typevar_constraints(self, values): + def foo(a: TIntStr, b: TIntStr): + assert check_argument_types() + + foo(*values) + + @pytest.mark.parametrize('value', [ + [6, 7], + {'aa', 'bb'} + ], ids=['int', 'str']) + def test_typevar_collection_constraints(self, value): + def foo(a: TTypingConstrained): + assert check_argument_types() + + foo(value) + + def test_typevar_collection_constraints_fail(self): + def foo(a: TTypingConstrained): + assert check_argument_types() + + pytest.raises(TypeError, foo, {1, 2}).\ + match(r'type of argument "a" must match one of the constraints \(List\[int\], ' + r'AbstractSet\[str\]\); got set instead') + + def test_typevar_constraints_fail(self): + def foo(a: TIntStr, b: TIntStr): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 2.5, 'aa') + assert str(exc.value) == ('type of argument "a" must match one of the constraints ' + '(int, str); got float instead') + + def test_typevar_bound(self): + def foo(a: TParent, b: TParent): + assert check_argument_types() + + foo(Child(), Child()) + + def test_typevar_bound_fail(self): + def foo(a: TChild, b: TChild): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, Parent(), Parent()) + assert str(exc.value) == ('type of argument "a" must be __tests__.test_typeguard.Child or one of ' + 'its subclasses; got __tests__.test_typeguard.Parent instead') + + @pytest.mark.skipif(Type is List, reason='typing.Type could not be imported') + def test_class_bad_subclass(self): + def foo(a: Type[Child]): + assert check_argument_types() + + pytest.raises(TypeError, foo, Parent).match( + '"a" must be a subclass of __tests__.test_typeguard.Child; got __tests__.test_typeguard.Parent instead') + + def test_class_any(self): + def foo(a: Type[Any]): + assert check_argument_types() + + foo(str) + + def test_class_union(self): + def foo(a: Type[Union[str, int]]): + assert check_argument_types() + + foo(str) + foo(int) + pytest.raises(TypeError, foo, tuple).\ + match(r'"a" must match one of the following: \(str, int\); got tuple instead') + + def test_wrapped_function(self): + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + + @decorator + def foo(a: 'Child'): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, Parent()) + assert str(exc.value) == ('type of argument "a" must be __tests__.test_typeguard.Child; ' + 'got __tests__.test_typeguard.Parent instead') + + def test_mismatching_default_type(self): + def foo(a: str = 1): + assert check_argument_types() + + pytest.raises(TypeError, foo).match('type of argument "a" must be str; got int instead') + + def test_implicit_default_none(self): + """ + Test that if the default value is ``None``, a ``None`` argument can be passed. + + """ + def foo(a: str = None): + assert check_argument_types() + + foo() + + def test_generator(self): + """Test that argument type checking works in a generator function too.""" + def generate(a: int): + assert check_argument_types() + yield a + yield a + 1 + + gen = generate(1) + next(gen) + + def test_wrapped_generator_no_return_type_annotation(self): + """Test that return type checking works in a generator function too.""" + @typechecked + def generate(a: int): + yield a + yield a + 1 + + gen = generate(1) + next(gen) + + def test_varargs(self): + def foo(*args: int): + assert check_argument_types() + + foo(1, 2) + + def test_varargs_fail(self): + def foo(*args: int): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, 1, 'a') + exc.match(r'type of argument "args"\[1\] must be int; got str instead') + + def test_kwargs(self): + def foo(**kwargs: int): + assert check_argument_types() + + foo(a=1, b=2) + + def test_kwargs_fail(self): + def foo(**kwargs: int): + assert check_argument_types() + + exc = pytest.raises(TypeError, foo, a=1, b='a') + exc.match(r'type of argument "kwargs"\[\'b\'\] must be int; got str instead') + + def test_generic(self): + def foo(a: FooGeneric[str]): + assert check_argument_types() + + foo(FooGeneric[str]()) + + @pytest.mark.skipif(myint is None, reason='NewType is not present in the typing module') + def test_newtype(self): + def foo(a: myint) -> int: + assert check_argument_types() + return 42 + + assert foo(1) == 42 + exc = pytest.raises(TypeError, foo, "a") + assert str(exc.value) == 'type of argument "a" must be int; got str instead' + + @pytest.mark.skipif(Collection is None, reason='typing.Collection is not available') + def test_collection(self): + def foo(a: Collection): + assert check_argument_types() + + pytest.raises(TypeError, foo, True).match( + 'type of argument "a" must be collections.abc.Collection; got bool instead') + + def test_binary_io(self): + def foo(a: BinaryIO): + assert check_argument_types() + + foo(BytesIO()) + + def test_text_io(self): + def foo(a: TextIO): + assert check_argument_types() + + foo(StringIO()) + + def test_binary_io_fail(self): + def foo(a: TextIO): + assert check_argument_types() + + pytest.raises(TypeError, foo, BytesIO()).match('must be a text based I/O') + + def test_text_io_fail(self): + def foo(a: BinaryIO): + assert check_argument_types() + + pytest.raises(TypeError, foo, StringIO()).match('must be a binary I/O') + + def test_binary_io_real_file(self, tmpdir): + def foo(a: BinaryIO): + assert check_argument_types() + + with tmpdir.join('testfile').open('wb') as f: + foo(f) + + def test_text_io_real_file(self, tmpdir): + def foo(a: TextIO): + assert check_argument_types() + + with tmpdir.join('testfile').open('w') as f: + foo(f) + + def test_recursive_type(self): + def foo(arg: JSONType) -> None: + assert check_argument_types() + + foo({'a': [1, 2, 3]}) + pytest.raises(TypeError, foo, {'a': (1, 2, 3)}).\ + match(r'type of argument "arg" must be one of \(str, int, float, (bool, )?NoneType, ' + r'List\[Union\[str, int, float, (bool, )?NoneType, List\[JSONType\], ' + r'Dict\[str, JSONType\]\]\], ' + r'Dict\[str, Union\[str, int, float, (bool, )?NoneType, List\[JSONType\], ' + r'Dict\[str, JSONType\]\]\]\); got dict instead') + + +class TestTypeChecked: + def test_typechecked(self): + @typechecked + def foo(a: int, b: str) -> str: + return 'abc' + + assert foo(4, 'abc') == 'abc' + + def test_typechecked_always(self): + @typechecked(always=True) + def foo(a: int, b: str) -> str: + return 'abc' + + assert foo(4, 'abc') == 'abc' + + def test_typechecked_arguments_fail(self): + @typechecked + def foo(a: int, b: str) -> str: + return 'abc' + + exc = pytest.raises(TypeError, foo, 4, 5) + assert str(exc.value) == 'type of argument "b" must be str; got int instead' + + def test_typechecked_return_type_fail(self): + @typechecked + def foo(a: int, b: str) -> str: + return 6 + + exc = pytest.raises(TypeError, foo, 4, 'abc') + assert str(exc.value) == 'type of the return value must be str; got int instead' + + def test_typechecked_return_typevar_fail(self): + T = TypeVar('T', int, float) + + @typechecked + def foo(a: T, b: T) -> T: + return 'a' + + pytest.raises(TypeError, foo, 4, 2).\ + match(r'type of the return value must match one of the constraints \(int, float\); ' + r'got str instead') + + def test_typechecked_no_annotations(self, recwarn): + def foo(a, b): + pass + + typechecked(foo) + + func_name = function_name(foo) + assert len(recwarn) == 1 + assert str(recwarn[0].message) == ( + 'no type annotations present -- not typechecking {}'.format(func_name)) + + def test_return_type_none(self): + """Check that a declared return type of None is respected.""" + @typechecked + def foo() -> None: + return 'a' + + exc = pytest.raises(TypeError, foo) + assert str(exc.value) == 'type of the return value must be NoneType; got str instead' + + def test_return_type_magicmock(self, mock_class): + @typechecked + def foo() -> str: + return mock_class() + + foo() + + @pytest.mark.parametrize('typehint', [ + Callable[..., int], + Callable + ], ids=['parametrized', 'unparametrized']) + def test_callable(self, typehint): + @typechecked + def foo(a: typehint): + pass + + def some_callable() -> int: + pass + + foo(some_callable) + + @pytest.mark.parametrize('typehint', [ + List[int], + List, + list, + ], ids=['parametrized', 'unparametrized', 'plain']) + def test_list(self, typehint): + @typechecked + def foo(a: typehint): + pass + + foo([1, 2]) + + @pytest.mark.parametrize('typehint', [ + Dict[str, int], + Dict, + dict + ], ids=['parametrized', 'unparametrized', 'plain']) + def test_dict(self, typehint): + @typechecked + def foo(a: typehint): + pass + + foo({'x': 2}) + + @pytest.mark.parametrize('typehint, value', [ + (Dict, {'x': 2, 6: 4}), + (List, ['x', 6]), + (Sequence, ['x', 6]), + (Set, {'x', 6}), + (AbstractSet, {'x', 6}), + (Tuple, ('x', 6)), + ], ids=['dict', 'list', 'sequence', 'set', 'abstractset', 'tuple']) + def test_unparametrized_types_mixed_values(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + @pytest.mark.parametrize('typehint', [ + Sequence[str], + Sequence + ], ids=['parametrized', 'unparametrized']) + @pytest.mark.parametrize('value', [('a', 'b'), ['a', 'b'], 'abc'], + ids=['tuple', 'list', 'str']) + def test_sequence(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + @pytest.mark.parametrize('typehint', [ + Iterable[str], + Iterable + ], ids=['parametrized', 'unparametrized']) + @pytest.mark.parametrize('value', [('a', 'b'), ['a', 'b'], 'abc'], + ids=['tuple', 'list', 'str']) + def test_iterable(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + @pytest.mark.parametrize('typehint', [ + Container[str], + Container + ], ids=['parametrized', 'unparametrized']) + @pytest.mark.parametrize('value', [('a', 'b'), ['a', 'b'], 'abc'], + ids=['tuple', 'list', 'str']) + def test_container(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + @pytest.mark.parametrize('typehint', [ + AbstractSet[int], + AbstractSet, + Set[int], + Set, + set + ], ids=['abstract_parametrized', 'abstract', 'parametrized', 'unparametrized', 'plain']) + @pytest.mark.parametrize('value', [set(), {6}]) + def test_set(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + @pytest.mark.parametrize('typehint', [ + Tuple[int, int], + Tuple[int, ...], + Tuple, + tuple + ], ids=['parametrized', 'ellipsis', 'unparametrized', 'plain']) + def test_tuple(self, typehint): + @typechecked + def foo(a: typehint): + pass + + foo((1, 2)) + + def test_empty_tuple(self): + @typechecked + def foo(a: Tuple[()]): + pass + + foo(()) + + @pytest.mark.skipif(Type is List, reason='typing.Type could not be imported') + @pytest.mark.parametrize('typehint', [ + Type[Parent], + Type[TypeVar('UnboundType')], # noqa: F821 + Type[TypeVar('BoundType', bound=Parent)], # noqa: F821 + Type, + type + ], ids=['parametrized', 'unbound-typevar', 'bound-typevar', 'unparametrized', 'plain']) + def test_class(self, typehint): + @typechecked + def foo(a: typehint): + pass + + foo(Child) + + @pytest.mark.skipif(Type is List, reason='typing.Type could not be imported') + def test_class_not_a_class(self): + @typechecked + def foo(a: Type[dict]): + pass + + exc = pytest.raises(TypeError, foo, 1) + exc.match('type of argument "a" must be a type; got int instead') + + @pytest.mark.parametrize('typehint, value', [ + (complex, complex(1, 5)), + (complex, 1.0), + (complex, 1), + (float, 1.0), + (float, 1) + ], ids=['complex-complex', 'complex-float', 'complex-int', 'float-float', 'float-int']) + def test_numbers(self, typehint, value): + @typechecked + def foo(a: typehint): + pass + + foo(value) + + def test_coroutine_correct_return_type(self): + @typechecked + async def foo() -> str: + return 'foo' + + coro = foo() + pytest.raises(StopIteration, coro.send, None) + + def test_coroutine_wrong_return_type(self): + @typechecked + async def foo() -> str: + return 1 + + coro = foo() + pytest.raises(TypeError, coro.send, None).\ + match('type of the return value must be str; got int instead') + + def test_bytearray_bytes(self): + """Test that a bytearray is accepted where bytes are expected.""" + @typechecked + def foo(x: bytes) -> None: + pass + + foo(bytearray([1])) + + def test_bytearray_memoryview(self): + """Test that a bytearray is accepted where bytes are expected.""" + @typechecked + def foo(x: bytes) -> None: + pass + + foo(memoryview(b'foo')) + + def test_class_decorator(self): + @typechecked + class Foo: + @staticmethod + def staticmethod() -> int: + return 'foo' + + @classmethod + def classmethod(cls) -> int: + return 'foo' + + def method(self) -> int: + return 'foo' + + @property + def prop(self) -> int: + return 'foo' + + @property + def prop2(self) -> int: + return 'foo' + + @prop2.setter + def prop2(self, value: int) -> None: + pass + + pattern = 'type of the return value must be int; got str instead' + pytest.raises(TypeError, Foo.staticmethod).match(pattern) + pytest.raises(TypeError, Foo.classmethod).match(pattern) + pytest.raises(TypeError, Foo().method).match(pattern) + + with pytest.raises(TypeError) as raises: + Foo().prop + + assert raises.value.args[0] == pattern + + with pytest.raises(TypeError) as raises: + Foo().prop2 + + assert raises.value.args[0] == pattern + + with pytest.raises(TypeError) as raises: + Foo().prop2 = 'foo' + + assert raises.value.args[0] == 'type of argument "value" must be int; got str instead' + + @pytest.mark.parametrize('annotation', [ + Generator[int, str, List[str]], + Generator, + Iterable[int], + Iterable, + Iterator[int], + Iterator + ], ids=['generator', 'bare_generator', 'iterable', 'bare_iterable', 'iterator', + 'bare_iterator']) + def test_generator(self, annotation): + @typechecked + def genfunc() -> annotation: + val1 = yield 2 + val2 = yield 3 + val3 = yield 4 + return [val1, val2, val3] + + gen = genfunc() + with pytest.raises(StopIteration) as exc: + value = next(gen) + while True: + value = gen.send(str(value)) + assert isinstance(value, int) + + assert exc.value.value == ['2', '3', '4'] + + @pytest.mark.parametrize('annotation', [ + Generator[int, str, None], + Iterable[int], + Iterator[int] + ], ids=['generator', 'iterable', 'iterator']) + def test_generator_bad_yield(self, annotation): + @typechecked + def genfunc() -> annotation: + yield 'foo' + + gen = genfunc() + with pytest.raises(TypeError) as exc: + next(gen) + + exc.match('type of value yielded from generator must be int; got str instead') + + def test_generator_bad_send(self): + @typechecked + def genfunc() -> Generator[int, str, None]: + yield 1 + yield 2 + + gen = genfunc() + next(gen) + with pytest.raises(TypeError) as exc: + gen.send(2) + + exc.match('type of value sent to generator must be str; got int instead') + + def test_generator_bad_return(self): + @typechecked + def genfunc() -> Generator[int, str, str]: + yield 1 + return 6 + + gen = genfunc() + next(gen) + with pytest.raises(TypeError) as exc: + gen.send('foo') + + exc.match('type of return value must be str; got int instead') + + def test_return_generator(self): + @typechecked + def genfunc() -> Generator[int, None, None]: + yield 1 + + @typechecked + def foo() -> Generator[int, None, None]: + return genfunc() + + foo() + + def test_builtin_decorator(self): + @typechecked + @lru_cache() + def func(x: int) -> None: + pass + + func(3) + func(3) + pytest.raises(TypeError, func, 'foo').\ + match('type of argument "x" must be int; got str instead') + + # Make sure that @lru_cache is still being used + cache_info = func.__wrapped__.cache_info() + assert cache_info.hits == 1 + + def test_local_class(self): + @typechecked + class LocalClass: + class Inner: + pass + + def create_inner(self) -> 'Inner': + return self.Inner() + + retval = LocalClass().create_inner() + assert isinstance(retval, LocalClass.Inner) + + def test_local_class_async(self): + @typechecked + class LocalClass: + class Inner: + pass + + async def create_inner(self) -> 'Inner': + return self.Inner() + + coro = LocalClass().create_inner() + exc = pytest.raises(StopIteration, coro.send, None) + retval = exc.value.value + assert isinstance(retval, LocalClass.Inner) + + def test_callable_nonmember(self): + class CallableClass: + def __call__(self): + pass + + @typechecked + class LocalClass: + some_callable = CallableClass() + + def test_inherited_class_method(self): + @typechecked + class Parent: + @classmethod + def foo(cls, x: str) -> str: + return cls.__name__ + + @typechecked + class Child(Parent): + pass + + assert Child.foo('bar') == 'Child' + pytest.raises(TypeError, Child.foo, 1) + + def test_class_property(self): + @typechecked + class Foo: + def __init__(self) -> None: + self.foo = 'foo' + + @property + def prop(self) -> int: + """My property.""" + return 4 + + @property + def prop2(self) -> str: + return self.foo + + @prop2.setter + def prop2(self, value: str) -> None: + self.foo = value + + assert Foo.__dict__["prop"].__doc__.strip() == "My property." + f = Foo() + assert f.prop == 4 + assert f.prop2 == 'foo' + f.prop2 = 'bar' + assert f.prop2 == 'bar' + + with pytest.raises(TypeError) as raises: + f.prop2 = 3 + + assert raises.value.args[0] == 'type of argument "value" must be str; got int instead' + + def test_decorator_factory_no_annotations(self): + class CallableClass: + def __call__(self): + pass + + def decorator_factory(): + def decorator(f): + cmd = CallableClass() + return cmd + + return decorator + + with pytest.warns(UserWarning): + @typechecked + @decorator_factory() + def foo(): + pass + + @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Fail wint Python 3.12") + @pytest.mark.parametrize('annotation', [TBound, TConstrained], ids=['bound', 'constrained']) + def test_typevar_forwardref(self, annotation): + @typechecked + def func(x: annotation) -> None: + pass + + func(Parent()) + func(Child()) + pytest.raises(TypeError, func, 'foo') + + @pytest.mark.parametrize('protocol_cls', [RuntimeProtocol, StaticProtocol]) + def test_protocol(self, protocol_cls): + @typechecked + def foo(arg: protocol_cls) -> None: + pass + + class Foo: + def meth(self) -> None: + pass + + foo(Foo()) + + def test_protocol_fail(self): + @typechecked + def foo(arg: RuntimeProtocol) -> None: + pass + + pytest.raises(TypeError, foo, object()).\ + match(r'type of argument "arg" \(object\) is not compatible with the RuntimeProtocol ' + 'protocol') + + def test_noreturn(self): + @typechecked + def foo() -> NoReturn: + pass + + pytest.raises(TypeError, foo).match(r'foo\(\) was declared never to return but it did') + + def test_recursive_type(self): + @typechecked + def foo(arg: JSONType) -> None: + pass + + foo({'a': [1, 2, 3]}) + pytest.raises(TypeError, foo, {'a': (1, 2, 3)}).\ + match(r'type of argument "arg" must be one of \(str, int, float, (bool, )?NoneType, ' + r'List\[Union\[str, int, float, (bool, )?NoneType, List\[JSONType\], ' + r'Dict\[str, JSONType\]\]\], ' + r'Dict\[str, Union\[str, int, float, (bool, )?NoneType, List\[JSONType\], ' + r'Dict\[str, JSONType\]\]\]\); got dict instead') + + def test_literal(self): + from http import HTTPStatus + + @typechecked + def foo(a: Literal[1, True, 'x', b'y', HTTPStatus.ACCEPTED]): + pass + + foo(HTTPStatus.ACCEPTED) + pytest.raises(TypeError, foo, 4).match(r"must be one of \(1, True, 'x', b'y', " + r"<HTTPStatus.ACCEPTED: 202>\); got 4 instead$") + + def test_literal_union(self): + @typechecked + def foo(a: Union[str, Literal[1, 6, 8]]): + pass + + foo(6) + pytest.raises(TypeError, foo, 4).\ + match(r'must be one of \(str, Literal\[1, 6, 8\]\); got int instead$') + + def test_literal_nested(self): + @typechecked + def foo(a: Literal[1, Literal['x', 'a', Literal['z']], 6, 8]): + pass + + foo('z') + pytest.raises(TypeError, foo, 4).match(r"must be one of \(1, 'x', 'a', 'z', 6, 8\); " + r"got 4 instead$") + + def test_literal_illegal_value(self): + @typechecked + def foo(a: Literal[1, 1.1]): + pass + + pytest.raises(TypeError, foo, 4).match(r"Illegal literal value: 1.1$") + + @pytest.mark.parametrize('value, total, error_re', [ + pytest.param({'x': 6, 'y': 'foo'}, True, None, id='correct'), + pytest.param({'y': 'foo'}, True, r'required key\(s\) \("x"\) missing from argument "arg"', + id='missing_x'), + pytest.param({'x': 6, 'y': 3}, True, + 'type of dict item "y" for argument "arg" must be str; got int instead', + id='wrong_y'), + pytest.param({'x': 6}, True, r'required key\(s\) \("y"\) missing from argument "arg"', + id='missing_y_error'), + pytest.param({'x': 6}, False, None, id='missing_y_ok', marks=[issue_42059]), + pytest.param({'x': 'abc'}, False, + 'type of dict item "x" for argument "arg" must be int; got str instead', + id='wrong_x', marks=[issue_42059]), + pytest.param({'x': 6, 'foo': 'abc'}, False, r'extra key\(s\) \("foo"\) in argument "arg"', + id='unknown_key') + ]) + def test_typed_dict(self, value, total, error_re): + DummyDict = TypedDict('DummyDict', {'x': int, 'y': str}, total=total) + + @typechecked + def foo(arg: DummyDict): + pass + + if error_re: + pytest.raises(TypeError, foo, value).match(error_re) + else: + foo(value) + + def test_class_abstract_property(self): + """Regression test for #206.""" + + @typechecked + class Foo: + @abstractproperty + def dummyproperty(self): + pass + + assert isinstance(Foo.dummyproperty, abstractproperty) + + +class TestTypeChecker: + @pytest.fixture + def executor(self): + executor = ThreadPoolExecutor(1) + yield executor + executor.shutdown() + + @pytest.fixture + def checker(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return TypeChecker(__name__) + + @staticmethod + def generatorfunc() -> Generator[int, None, None]: + yield 1 + + @staticmethod + def bad_generatorfunc() -> Generator[int, None, None]: + yield 1 + yield 'foo' + + @staticmethod + def error_function() -> float: + return 1 / 0 + + def test_check_call_args(self, checker: TypeChecker): + def foo(a: int): + pass + + with checker, pytest.warns(TypeWarning) as record: + assert checker.active + foo(1) + foo('x') + + assert not checker.active + foo('x') + + assert len(record) == 1 + warning = record[0].message + assert warning.error == 'type of argument "a" must be int; got str instead' + assert warning.func is foo + assert isinstance(warning.stack, list) + buffer = StringIO() + warning.print_stack(buffer) + assert len(buffer.getvalue()) > 100 + + def test_check_return_value(self, checker: TypeChecker): + def foo() -> int: + return 'x' + + with checker, pytest.warns(TypeWarning) as record: + foo() + + assert len(record) == 1 + assert record[0].message.error == 'type of the return value must be int; got str instead' + + def test_threaded_check_call_args(self, checker: TypeChecker, executor): + def foo(a: int): + pass + + with checker, pytest.warns(TypeWarning) as record: + executor.submit(foo, 1).result() + executor.submit(foo, 'x').result() + + executor.submit(foo, 'x').result() + + assert len(record) == 1 + warning = record[0].message + assert warning.error == 'type of argument "a" must be int; got str instead' + assert warning.func is foo + + def test_double_start(self, checker: TypeChecker): + """Test that the same type checker can't be started twice while running.""" + with checker: + pytest.raises(RuntimeError, checker.start).match('type checker already running') + + def test_nested(self): + """Test that nesting of type checker context managers works as expected.""" + def foo(a: int): + pass + + with warnings.catch_warnings(record=True): + warnings.simplefilter('ignore', DeprecationWarning) + parent = TypeChecker(__name__) + child = TypeChecker(__name__) + + with parent, pytest.warns(TypeWarning) as record: + foo('x') + with child: + foo('x') + + assert len(record) == 3 + + def test_existing_profiler(self, checker: TypeChecker): + """ + Test that an existing profiler function is chained with the type checker and restored after + the block is exited. + + """ + def foo(a: int): + pass + + def profiler(frame, event, arg): + nonlocal profiler_run_count + if event in ('call', 'return'): + profiler_run_count += 1 + + if old_profiler: + old_profiler(frame, event, arg) + + profiler_run_count = 0 + old_profiler = sys.getprofile() + sys.setprofile(profiler) + try: + with checker, pytest.warns(TypeWarning) as record: + foo(1) + foo('x') + + assert sys.getprofile() is profiler + finally: + sys.setprofile(old_profiler) + + assert profiler_run_count + assert len(record) == 1 + + def test_generator(self, checker): + with checker, pytest.warns(None) as record: + gen = self.generatorfunc() + assert next(gen) == 1 + + assert len(record) == 0 + + def test_generator_wrong_yield(self, checker): + with checker, pytest.warns(TypeWarning) as record: + gen = self.bad_generatorfunc() + assert list(gen) == [1, 'foo'] + + assert len(record) == 1 + assert 'type of yielded value must be int; got str instead' in str(record[0].message) + + def test_exception(self, checker): + with checker, pytest.warns(None) as record: + pytest.raises(ZeroDivisionError, self.error_function) + + assert len(record) == 0 + + @pytest.mark.parametrize('policy', [ForwardRefPolicy.WARN, ForwardRefPolicy.GUESS], + ids=['warn', 'guess']) + def test_forward_ref_policy_resolution_fails(self, checker, policy): + def unresolvable_annotation(x: 'OrderedDict'): # noqa + pass + + checker.annotation_policy = policy + gc.collect() # prevent find_function() from finding more than one instance of the function + with checker, pytest.warns(TypeHintWarning) as record: + unresolvable_annotation({}) + + assert len(record) == 1 + assert ("unresolvable_annotation: name 'OrderedDict' is not defined" + in str(record[0].message)) + assert 'x' not in unresolvable_annotation.__annotations__ + + def test_forward_ref_policy_guess(self, checker): + import collections + + def unresolvable_annotation(x: 'OrderedDict'): # noqa + pass + + checker.annotation_policy = ForwardRefPolicy.GUESS + with checker, pytest.warns(TypeHintWarning) as record: + unresolvable_annotation(collections.OrderedDict()) + + assert len(record) == 1 + assert str(record[0].message).startswith("Replaced forward declaration 'OrderedDict' in") + assert unresolvable_annotation.__annotations__['x'] is collections.OrderedDict + + +class TestTracebacks: + def test_short_tracebacks(self): + def foo(a: Callable[..., int]): + assert check_argument_types() + + try: + foo(1) + except TypeError: + _, _, tb = sys.exc_info() + parts = traceback.extract_tb(tb) + typeguard_lines = [part for part in parts + if part.filename.endswith("typeguard/__init__.py")] + assert len(typeguard_lines) == 1 diff --git a/contrib/python/typeguard/tests/test_typeguard_py36.py b/contrib/python/typeguard/tests/test_typeguard_py36.py new file mode 100644 index 00000000000..383f7f3353f --- /dev/null +++ b/contrib/python/typeguard/tests/test_typeguard_py36.py @@ -0,0 +1,189 @@ +import sys +import warnings +from typing import Any, AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Dict + +import pytest +from typing_extensions import Protocol, runtime_checkable + +from typeguard import TypeChecker, typechecked + +try: + from typing import TypedDict +except ImportError: + from typing_extensions import TypedDict + + +@runtime_checkable +class RuntimeProtocol(Protocol): + member: int + + def meth(self) -> None: + ... + + +class TestTypeChecked: + @pytest.mark.parametrize('annotation', [ + AsyncGenerator[int, str], + AsyncIterable[int], + AsyncIterator[int] + ], ids=['generator', 'iterable', 'iterator']) + def test_async_generator(self, annotation): + async def run_generator(): + @typechecked + async def genfunc() -> annotation: + values.append((yield 2)) + values.append((yield 3)) + values.append((yield 4)) + + gen = genfunc() + + value = await gen.asend(None) + with pytest.raises(StopAsyncIteration): + while True: + value = await gen.asend(str(value)) + assert isinstance(value, int) + + values = [] + coro = run_generator() + try: + for elem in coro.__await__(): + print(elem) + except StopAsyncIteration as exc: + values = exc.value + + assert values == ['2', '3', '4'] + + @pytest.mark.parametrize('annotation', [ + AsyncGenerator[int, str], + AsyncIterable[int], + AsyncIterator[int] + ], ids=['generator', 'iterable', 'iterator']) + def test_async_generator_bad_yield(self, annotation): + @typechecked + async def genfunc() -> annotation: + yield 'foo' + + gen = genfunc() + with pytest.raises(TypeError) as exc: + next(gen.__anext__().__await__()) + + exc.match('type of value yielded from generator must be int; got str instead') + + def test_async_generator_bad_send(self): + @typechecked + async def genfunc() -> AsyncGenerator[int, str]: + yield 1 + yield 2 + + gen = genfunc() + pytest.raises(StopIteration, next, gen.__anext__().__await__()) + with pytest.raises(TypeError) as exc: + next(gen.asend(2).__await__()) + + exc.match('type of value sent to generator must be str; got int instead') + + def test_return_async_generator(self): + @typechecked + async def genfunc() -> AsyncGenerator[int, None]: + yield 1 + + @typechecked + def foo() -> AsyncGenerator[int, None]: + return genfunc() + + foo() + + def test_async_generator_iterate(self): + asyncgen = typechecked(asyncgenfunc)() + aiterator = asyncgen.__aiter__() + exc = pytest.raises(StopIteration, aiterator.__anext__().send, None) + assert exc.value.value == 1 + + def test_typeddict_inherited(self): + class ParentDict(TypedDict): + x: int + + class ChildDict(ParentDict, total=False): + y: int + + @typechecked + def foo(arg: ChildDict): + pass + + foo({'x': 1}) + if sys.version_info[:2] != (3, 8): + # TypedDict is unusable for runtime checking on Python 3.8 + pytest.raises(TypeError, foo, {'y': 1}) + + def test_mapping_is_not_typeddict(self): + """Regression test for #216.""" + + class Foo(Dict[str, Any]): + pass + + @typechecked + def foo(arg: Foo): + pass + + foo(Foo({'x': 1})) + + +async def asyncgenfunc() -> AsyncGenerator[int, None]: + yield 1 + + +async def asyncgeniterablefunc() -> AsyncIterable[int]: + yield 1 + + +async def asyncgeniteratorfunc() -> AsyncIterator[int]: + yield 1 + + +class TestTypeChecker: + @pytest.fixture + def checker(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return TypeChecker(__name__) + + @pytest.mark.parametrize('func', [asyncgenfunc, asyncgeniterablefunc, asyncgeniteratorfunc], + ids=['generator', 'iterable', 'iterator']) + def test_async_generator(self, checker, func): + """Make sure that the type checker does not complain about the None return value.""" + with checker, pytest.warns(None) as record: + func() + + assert len(record) == 0 + + def test_callable(self): + class command: + # we need an __annotations__ attribute to trigger the code path + whatever: float + + def __init__(self, function: Callable[[int], int]): + self.function = function + + def __call__(self, arg: int) -> None: + self.function(arg) + + @typechecked + @command + def function(arg: int) -> None: + pass + + function(1) + + +def test_protocol_non_method_members(): + @typechecked + def foo(a: RuntimeProtocol): + pass + + class Foo: + member = 1 + + def meth(self) -> None: + pass + + foo(Foo()) diff --git a/contrib/python/typeguard/tests/ya.make b/contrib/python/typeguard/tests/ya.make new file mode 100644 index 00000000000..0649ac4e9f7 --- /dev/null +++ b/contrib/python/typeguard/tests/ya.make @@ -0,0 +1,22 @@ +PY3TEST() + +PEERDIR( + contrib/python/typeguard + contrib/python/typing-extensions +) + +TEST_SRCS( + conftest.py + dummymodule.py + test_importhook.py + test_typeguard.py + test_typeguard_py36.py +) + +DATA( + arcadia/contrib/python/typeguard/tests +) + +NO_LINT() + +END() diff --git a/contrib/python/typeguard/typeguard/__init__.py b/contrib/python/typeguard/typeguard/__init__.py new file mode 100644 index 00000000000..33d83d4582c --- /dev/null +++ b/contrib/python/typeguard/typeguard/__init__.py @@ -0,0 +1,1258 @@ +__all__ = ('ForwardRefPolicy', 'TypeHintWarning', 'typechecked', 'check_return_type', + 'check_argument_types', 'check_type', 'TypeWarning', 'TypeChecker', + 'typeguard_ignore') + +import collections.abc +import gc +import inspect +import sys +import threading +from collections import OrderedDict +from enum import Enum +from functools import partial, wraps +from inspect import Parameter, isclass, isfunction, isgeneratorfunction +from io import BufferedIOBase, IOBase, RawIOBase, TextIOBase +from traceback import extract_stack, print_stack +from types import CodeType, FunctionType +from typing import ( + IO, TYPE_CHECKING, AbstractSet, Any, AsyncIterable, AsyncIterator, BinaryIO, Callable, Dict, + Generator, Iterable, Iterator, List, NewType, Optional, Sequence, Set, TextIO, Tuple, Type, + TypeVar, Union, get_type_hints, overload) +from unittest.mock import Mock +from warnings import warn +from weakref import WeakKeyDictionary, WeakValueDictionary + +# Python 3.8+ +try: + from typing_extensions import Literal +except ImportError: + try: + from typing import Literal + except ImportError: + Literal = None + +# Python 3.5.4+ / 3.6.2+ +try: + from typing_extensions import NoReturn +except ImportError: + try: + from typing import NoReturn + except ImportError: + NoReturn = None + +# Python 3.6+ +try: + from inspect import isasyncgen, isasyncgenfunction + from typing import AsyncGenerator +except ImportError: + AsyncGenerator = None + + def isasyncgen(obj): + return False + + def isasyncgenfunction(func): + return False + +# Python 3.8+ +try: + from typing import ForwardRef + evaluate_forwardref = ForwardRef._evaluate +except ImportError: + from typing import _ForwardRef as ForwardRef + evaluate_forwardref = ForwardRef._eval_type + +if sys.version_info >= (3, 10): + from typing import is_typeddict +else: + _typed_dict_meta_types = () + if sys.version_info >= (3, 8): + from typing import _TypedDictMeta + _typed_dict_meta_types += (_TypedDictMeta,) + + try: + from typing_extensions import _TypedDictMeta + _typed_dict_meta_types += (_TypedDictMeta,) + except ImportError: + pass + + def is_typeddict(tp) -> bool: + return isinstance(tp, _typed_dict_meta_types) + + +if TYPE_CHECKING: + _F = TypeVar("_F") + + def typeguard_ignore(f: _F) -> _F: + """This decorator is a noop during static type-checking.""" + return f +else: + from typing import no_type_check as typeguard_ignore + + +_type_hints_map = WeakKeyDictionary() # type: Dict[FunctionType, Dict[str, Any]] +_functions_map = WeakValueDictionary() # type: Dict[CodeType, FunctionType] +_missing = object() + +T_CallableOrType = TypeVar('T_CallableOrType', bound=Callable[..., Any]) + +# Lifted from mypy.sharedparse +BINARY_MAGIC_METHODS = { + "__add__", + "__and__", + "__cmp__", + "__divmod__", + "__div__", + "__eq__", + "__floordiv__", + "__ge__", + "__gt__", + "__iadd__", + "__iand__", + "__idiv__", + "__ifloordiv__", + "__ilshift__", + "__imatmul__", + "__imod__", + "__imul__", + "__ior__", + "__ipow__", + "__irshift__", + "__isub__", + "__itruediv__", + "__ixor__", + "__le__", + "__lshift__", + "__lt__", + "__matmul__", + "__mod__", + "__mul__", + "__ne__", + "__or__", + "__pow__", + "__radd__", + "__rand__", + "__rdiv__", + "__rfloordiv__", + "__rlshift__", + "__rmatmul__", + "__rmod__", + "__rmul__", + "__ror__", + "__rpow__", + "__rrshift__", + "__rshift__", + "__rsub__", + "__rtruediv__", + "__rxor__", + "__sub__", + "__truediv__", + "__xor__", +} + + +class ForwardRefPolicy(Enum): + """Defines how unresolved forward references are handled.""" + + ERROR = 1 #: propagate the :exc:`NameError` from :func:`~typing.get_type_hints` + WARN = 2 #: remove the annotation and emit a TypeHintWarning + #: replace the annotation with the argument's class if the qualified name matches, else remove + #: the annotation + GUESS = 3 + + +class TypeHintWarning(UserWarning): + """ + A warning that is emitted when a type hint in string form could not be resolved to an actual + type. + """ + + +class _TypeCheckMemo: + __slots__ = 'globals', 'locals' + + def __init__(self, globals: Dict[str, Any], locals: Dict[str, Any]): + self.globals = globals + self.locals = locals + + +def _strip_annotation(annotation): + if isinstance(annotation, str): + return annotation.strip("'") + else: + return annotation + + +class _CallMemo(_TypeCheckMemo): + __slots__ = 'func', 'func_name', 'arguments', 'is_generator', 'type_hints' + + def __init__(self, func: Callable, frame_locals: Optional[Dict[str, Any]] = None, + args: tuple = None, kwargs: Dict[str, Any] = None, + forward_refs_policy=ForwardRefPolicy.ERROR): + super().__init__(func.__globals__, frame_locals) + self.func = func + self.func_name = function_name(func) + self.is_generator = isgeneratorfunction(func) + signature = inspect.signature(func) + + if args is not None and kwargs is not None: + self.arguments = signature.bind(*args, **kwargs).arguments + else: + assert frame_locals is not None, 'frame must be specified if args or kwargs is None' + self.arguments = frame_locals + + self.type_hints = _type_hints_map.get(func) + if self.type_hints is None: + while True: + if sys.version_info < (3, 5, 3): + frame_locals = dict(frame_locals) + + try: + hints = get_type_hints(func, localns=frame_locals) + except NameError as exc: + if forward_refs_policy is ForwardRefPolicy.ERROR: + raise + + typename = str(exc).split("'", 2)[1] + for param in signature.parameters.values(): + if _strip_annotation(param.annotation) == typename: + break + else: + raise + + func_name = function_name(func) + if forward_refs_policy is ForwardRefPolicy.GUESS: + if param.name in self.arguments: + argtype = self.arguments[param.name].__class__ + stripped = _strip_annotation(param.annotation) + if stripped == argtype.__qualname__: + func.__annotations__[param.name] = argtype + msg = ('Replaced forward declaration {!r} in {} with {!r}' + .format(stripped, func_name, argtype)) + warn(TypeHintWarning(msg)) + continue + + msg = 'Could not resolve type hint {!r} on {}: {}'.format( + param.annotation, function_name(func), exc) + warn(TypeHintWarning(msg)) + del func.__annotations__[param.name] + else: + break + + self.type_hints = OrderedDict() + for name, parameter in signature.parameters.items(): + if name in hints: + annotated_type = hints[name] + + # PEP 428 discourages it by MyPy does not complain + if parameter.default is None: + annotated_type = Optional[annotated_type] + + if parameter.kind == Parameter.VAR_POSITIONAL: + self.type_hints[name] = Tuple[annotated_type, ...] + elif parameter.kind == Parameter.VAR_KEYWORD: + self.type_hints[name] = Dict[str, annotated_type] + else: + self.type_hints[name] = annotated_type + + if 'return' in hints: + self.type_hints['return'] = hints['return'] + + _type_hints_map[func] = self.type_hints + + +def resolve_forwardref(maybe_ref, memo: _TypeCheckMemo): + if isinstance(maybe_ref, ForwardRef): + if sys.version_info < (3, 9, 0): + return evaluate_forwardref(maybe_ref, memo.globals, memo.locals) + else: + return evaluate_forwardref(maybe_ref, memo.globals, memo.locals, recursive_guard=frozenset()) + + else: + return maybe_ref + + +def get_type_name(type_): + name = (getattr(type_, '__name__', None) or getattr(type_, '_name', None) or + getattr(type_, '__forward_arg__', None)) + if name is None: + origin = getattr(type_, '__origin__', None) + name = getattr(origin, '_name', None) + if name is None and not inspect.isclass(type_): + name = type_.__class__.__name__.strip('_') + + args = getattr(type_, '__args__', ()) or getattr(type_, '__values__', ()) + if args != getattr(type_, '__parameters__', ()): + if name == 'Literal': + formatted_args = ', '.join(str(arg) for arg in args) + else: + formatted_args = ', '.join(get_type_name(arg) for arg in args) + + name = '{}[{}]'.format(name, formatted_args) + + module = getattr(type_, '__module__', None) + if module not in (None, 'typing', 'typing_extensions', 'builtins'): + name = module + '.' + name + + return name + + +def find_function(frame) -> Optional[Callable]: + """ + Return a function object from the garbage collector that matches the frame's code object. + + This process is unreliable as several function objects could use the same code object. + Fortunately the likelihood of this happening with the combination of the function objects + having different type annotations is a very rare occurrence. + + :param frame: a frame object + :return: a function object if one was found, ``None`` if not + + """ + func = _functions_map.get(frame.f_code) + if func is None: + for obj in gc.get_referrers(frame.f_code): + if inspect.isfunction(obj): + if func is None: + # The first match was found + func = obj + else: + # A second match was found + return None + + # Cache the result for future lookups + if func is not None: + _functions_map[frame.f_code] = func + else: + raise LookupError('target function not found') + + return func + + +def qualified_name(obj) -> str: + """ + Return the qualified name (e.g. package.module.Type) for the given object. + + Builtins and types from the :mod:`typing` package get special treatment by having the module + name stripped from the generated name. + + """ + type_ = obj if inspect.isclass(obj) else type(obj) + module = type_.__module__ + qualname = type_.__qualname__ + return qualname if module in ('typing', 'builtins') else '{}.{}'.format(module, qualname) + + +def function_name(func: Callable) -> str: + """ + Return the qualified name of the given function. + + Builtins and types from the :mod:`typing` package get special treatment by having the module + name stripped from the generated name. + + """ + # For partial functions and objects with __call__ defined, __qualname__ does not exist + # For functions run in `exec` with a custom namespace, __module__ can be None + module = getattr(func, '__module__', '') or '' + qualname = (module + '.') if module not in ('builtins', '') else '' + return qualname + getattr(func, '__qualname__', repr(func)) + + +def check_callable(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not callable(value): + raise TypeError('{} must be a callable'.format(argname)) + + if getattr(expected_type, "__args__", None): + try: + signature = inspect.signature(value) + except (TypeError, ValueError): + return + + if hasattr(expected_type, '__result__'): + # Python 3.5 + argument_types = expected_type.__args__ + check_args = argument_types is not Ellipsis + else: + # Python 3.6 + argument_types = expected_type.__args__[:-1] + check_args = argument_types != (Ellipsis,) + + if check_args: + # The callable must not have keyword-only arguments without defaults + unfulfilled_kwonlyargs = [ + param.name for param in signature.parameters.values() if + param.kind == Parameter.KEYWORD_ONLY and param.default == Parameter.empty] + if unfulfilled_kwonlyargs: + raise TypeError( + 'callable passed as {} has mandatory keyword-only arguments in its ' + 'declaration: {}'.format(argname, ', '.join(unfulfilled_kwonlyargs))) + + num_mandatory_args = len([ + param.name for param in signature.parameters.values() + if param.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD) and + param.default is Parameter.empty]) + has_varargs = any(param for param in signature.parameters.values() + if param.kind == Parameter.VAR_POSITIONAL) + + if num_mandatory_args > len(argument_types): + raise TypeError( + 'callable passed as {} has too many arguments in its declaration; expected {} ' + 'but {} argument(s) declared'.format(argname, len(argument_types), + num_mandatory_args)) + elif not has_varargs and num_mandatory_args < len(argument_types): + raise TypeError( + 'callable passed as {} has too few arguments in its declaration; expected {} ' + 'but {} argument(s) declared'.format(argname, len(argument_types), + num_mandatory_args)) + + +def check_dict(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not isinstance(value, dict): + raise TypeError('type of {} must be a dict; got {} instead'. + format(argname, qualified_name(value))) + + if expected_type is not dict: + if (hasattr(expected_type, "__args__") and + expected_type.__args__ not in (None, expected_type.__parameters__)): + key_type, value_type = expected_type.__args__ + if key_type is not Any or value_type is not Any: + for k, v in value.items(): + check_type('keys of {}'.format(argname), k, key_type, memo) + check_type('{}[{!r}]'.format(argname, k), v, value_type, memo) + + +def check_typed_dict(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + declared_keys = frozenset(expected_type.__annotations__) + if hasattr(expected_type, '__required_keys__'): + required_keys = expected_type.__required_keys__ + else: # py3.8 and lower + required_keys = declared_keys if expected_type.__total__ else frozenset() + + existing_keys = frozenset(value) + extra_keys = existing_keys - declared_keys + if extra_keys: + keys_formatted = ', '.join('"{}"'.format(key) for key in sorted(extra_keys)) + raise TypeError('extra key(s) ({}) in {}'.format(keys_formatted, argname)) + + missing_keys = required_keys - existing_keys + if missing_keys: + keys_formatted = ', '.join('"{}"'.format(key) for key in sorted(missing_keys)) + raise TypeError('required key(s) ({}) missing from {}'.format(keys_formatted, argname)) + + for key, argtype in get_type_hints(expected_type).items(): + argvalue = value.get(key, _missing) + if argvalue is not _missing: + check_type('dict item "{}" for {}'.format(key, argname), argvalue, argtype, memo) + + +def check_list(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not isinstance(value, list): + raise TypeError('type of {} must be a list; got {} instead'. + format(argname, qualified_name(value))) + + if expected_type is not list: + if hasattr(expected_type, "__args__") and expected_type.__args__ not in \ + (None, expected_type.__parameters__): + value_type = expected_type.__args__[0] + if value_type is not Any: + for i, v in enumerate(value): + check_type('{}[{}]'.format(argname, i), v, value_type, memo) + + +def check_sequence(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not isinstance(value, collections.abc.Sequence): + raise TypeError('type of {} must be a sequence; got {} instead'. + format(argname, qualified_name(value))) + + if hasattr(expected_type, "__args__") and expected_type.__args__ not in \ + (None, expected_type.__parameters__): + value_type = expected_type.__args__[0] + if value_type is not Any: + for i, v in enumerate(value): + check_type('{}[{}]'.format(argname, i), v, value_type, memo) + + +def check_set(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not isinstance(value, AbstractSet): + raise TypeError('type of {} must be a set; got {} instead'. + format(argname, qualified_name(value))) + + if expected_type is not set: + if hasattr(expected_type, "__args__") and expected_type.__args__ not in \ + (None, expected_type.__parameters__): + value_type = expected_type.__args__[0] + if value_type is not Any: + for v in value: + check_type('elements of {}'.format(argname), v, value_type, memo) + + +def check_tuple(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + # Specialized check for NamedTuples + is_named_tuple = False + if sys.version_info < (3, 8, 0): + is_named_tuple = hasattr(expected_type, '_field_types') # deprecated since python 3.8 + else: + is_named_tuple = hasattr(expected_type, '__annotations__') + + if is_named_tuple: + if not isinstance(value, expected_type): + raise TypeError('type of {} must be a named tuple of type {}; got {} instead'. + format(argname, qualified_name(expected_type), qualified_name(value))) + + if sys.version_info < (3, 8, 0): + field_types = expected_type._field_types + else: + field_types = expected_type.__annotations__ + + for name, field_type in field_types.items(): + check_type('{}.{}'.format(argname, name), getattr(value, name), field_type, memo) + + return + elif not isinstance(value, tuple): + raise TypeError('type of {} must be a tuple; got {} instead'. + format(argname, qualified_name(value))) + + if getattr(expected_type, '__tuple_params__', None): + # Python 3.5 + use_ellipsis = expected_type.__tuple_use_ellipsis__ + tuple_params = expected_type.__tuple_params__ + elif getattr(expected_type, '__args__', None): + # Python 3.6+ + use_ellipsis = expected_type.__args__[-1] is Ellipsis + tuple_params = expected_type.__args__[:-1 if use_ellipsis else None] + else: + # Unparametrized Tuple or plain tuple + return + + if use_ellipsis: + element_type = tuple_params[0] + for i, element in enumerate(value): + check_type('{}[{}]'.format(argname, i), element, element_type, memo) + elif tuple_params == ((),): + if value != (): + raise TypeError('{} is not an empty tuple but one was expected'.format(argname)) + else: + if len(value) != len(tuple_params): + raise TypeError('{} has wrong number of elements (expected {}, got {} instead)' + .format(argname, len(tuple_params), len(value))) + + for i, (element, element_type) in enumerate(zip(value, tuple_params)): + check_type('{}[{}]'.format(argname, i), element, element_type, memo) + + +def check_union(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if hasattr(expected_type, '__union_params__'): + # Python 3.5 + union_params = expected_type.__union_params__ + else: + # Python 3.6+ + union_params = expected_type.__args__ + + for type_ in union_params: + try: + check_type(argname, value, type_, memo) + return + except TypeError: + pass + + typelist = ', '.join(get_type_name(t) for t in union_params) + raise TypeError('type of {} must be one of ({}); got {} instead'. + format(argname, typelist, qualified_name(value))) + + +def check_class(argname: str, value, expected_type, memo: _TypeCheckMemo) -> None: + if not isclass(value): + raise TypeError('type of {} must be a type; got {} instead'.format( + argname, qualified_name(value))) + + # Needed on Python 3.7+ + if expected_type is Type: + return + + if getattr(expected_type, '__origin__', None) in (Type, type): + expected_class = expected_type.__args__[0] + else: + expected_class = expected_type + + if expected_class is Any: + return + elif isinstance(expected_class, TypeVar): + check_typevar(argname, value, expected_class, memo, True) + elif getattr(expected_class, '__origin__', None) is Union: + for arg in expected_class.__args__: + try: + check_class(argname, value, arg, memo) + break + except TypeError: + pass + else: + formatted_args = ', '.join(get_type_name(arg) for arg in expected_class.__args__) + raise TypeError('{} must match one of the following: ({}); got {} instead'.format( + argname, formatted_args, qualified_name(value) + )) + elif not issubclass(value, expected_class): + raise TypeError('{} must be a subclass of {}; got {} instead'.format( + argname, qualified_name(expected_class), qualified_name(value))) + + +def check_typevar(argname: str, value, typevar: TypeVar, memo: _TypeCheckMemo, + subclass_check: bool = False) -> None: + value_type = value if subclass_check else type(value) + subject = argname if subclass_check else 'type of ' + argname + + if typevar.__bound__ is not None: + bound_type = resolve_forwardref(typevar.__bound__, memo) + if not issubclass(value_type, bound_type): + raise TypeError( + '{} must be {} or one of its subclasses; got {} instead' + .format(subject, qualified_name(bound_type), qualified_name(value_type))) + elif typevar.__constraints__: + constraints = [resolve_forwardref(c, memo) for c in typevar.__constraints__] + for constraint in constraints: + try: + check_type(argname, value, constraint, memo) + except TypeError: + pass + else: + break + else: + formatted_constraints = ', '.join(get_type_name(constraint) + for constraint in constraints) + raise TypeError('{} must match one of the constraints ({}); got {} instead' + .format(subject, formatted_constraints, qualified_name(value_type))) + + +def check_literal(argname: str, value, expected_type, memo: _TypeCheckMemo): + def get_args(literal): + try: + args = literal.__args__ + except AttributeError: + # Instance of Literal from typing_extensions + args = literal.__values__ + + retval = [] + for arg in args: + if isinstance(arg, Literal.__class__) or getattr(arg, '__origin__', None) is Literal: + # The first check works on py3.6 and lower, the second one on py3.7+ + retval.extend(get_args(arg)) + elif isinstance(arg, (int, str, bytes, bool, type(None), Enum)): + retval.append(arg) + else: + raise TypeError('Illegal literal value: {}'.format(arg)) + + return retval + + final_args = tuple(get_args(expected_type)) + if value not in final_args: + raise TypeError('the value of {} must be one of {}; got {} instead'. + format(argname, final_args, value)) + + +def check_number(argname: str, value, expected_type): + if expected_type is complex and not isinstance(value, (complex, float, int)): + raise TypeError('type of {} must be either complex, float or int; got {} instead'. + format(argname, qualified_name(value.__class__))) + elif expected_type is float and not isinstance(value, (float, int)): + raise TypeError('type of {} must be either float or int; got {} instead'. + format(argname, qualified_name(value.__class__))) + + +def check_io(argname: str, value, expected_type): + if expected_type is TextIO: + if not isinstance(value, TextIOBase): + raise TypeError('type of {} must be a text based I/O object; got {} instead'. + format(argname, qualified_name(value.__class__))) + elif expected_type is BinaryIO: + if not isinstance(value, (RawIOBase, BufferedIOBase)): + raise TypeError('type of {} must be a binary I/O object; got {} instead'. + format(argname, qualified_name(value.__class__))) + elif not isinstance(value, IOBase): + raise TypeError('type of {} must be an I/O object; got {} instead'. + format(argname, qualified_name(value.__class__))) + + +def check_protocol(argname: str, value, expected_type): + # TODO: implement proper compatibility checking and support non-runtime protocols + if getattr(expected_type, '_is_runtime_protocol', False): + if not isinstance(value, expected_type): + raise TypeError('type of {} ({}) is not compatible with the {} protocol'. + format(argname, type(value).__qualname__, expected_type.__qualname__)) + + +# Equality checks are applied to these +origin_type_checkers = { + AbstractSet: check_set, + Callable: check_callable, + collections.abc.Callable: check_callable, + dict: check_dict, + Dict: check_dict, + list: check_list, + List: check_list, + Sequence: check_sequence, + collections.abc.Sequence: check_sequence, + collections.abc.Set: check_set, + set: check_set, + Set: check_set, + tuple: check_tuple, + Tuple: check_tuple, + type: check_class, + Type: check_class, + Union: check_union +} +_subclass_check_unions = hasattr(Union, '__union_set_params__') +if Literal is not None: + origin_type_checkers[Literal] = check_literal + +generator_origin_types = (Generator, collections.abc.Generator, + Iterator, collections.abc.Iterator, + Iterable, collections.abc.Iterable) +asyncgen_origin_types = (AsyncIterator, collections.abc.AsyncIterator, + AsyncIterable, collections.abc.AsyncIterable) +if AsyncGenerator is not None: + asyncgen_origin_types += (AsyncGenerator,) +if hasattr(collections.abc, 'AsyncGenerator'): + asyncgen_origin_types += (collections.abc.AsyncGenerator,) + + +def check_type(argname: str, value, expected_type, memo: Optional[_TypeCheckMemo] = None, *, + globals: Optional[Dict[str, Any]] = None, + locals: Optional[Dict[str, Any]] = None) -> None: + """ + Ensure that ``value`` matches ``expected_type``. + + The types from the :mod:`typing` module do not support :func:`isinstance` or :func:`issubclass` + so a number of type specific checks are required. This function knows which checker to call + for which type. + + :param argname: name of the argument to check; used for error messages + :param value: value to be checked against ``expected_type`` + :param expected_type: a class or generic type instance + :param globals: dictionary of global variables to use for resolving forward references + (defaults to the calling frame's globals) + :param locals: dictionary of local variables to use for resolving forward references + (defaults to the calling frame's locals) + :raises TypeError: if there is a type mismatch + + """ + if expected_type is Any or isinstance(value, Mock): + return + + if expected_type is None: + # Only happens on < 3.6 + expected_type = type(None) + + if memo is None: + frame = sys._getframe(1) + if globals is None: + globals = frame.f_globals + if locals is None: + locals = frame.f_locals + + memo = _TypeCheckMemo(globals, locals) + + expected_type = resolve_forwardref(expected_type, memo) + origin_type = getattr(expected_type, '__origin__', None) + if origin_type is not None: + checker_func = origin_type_checkers.get(origin_type) + if checker_func: + checker_func(argname, value, expected_type, memo) + else: + check_type(argname, value, origin_type, memo) + elif isclass(expected_type): + if issubclass(expected_type, Tuple): + check_tuple(argname, value, expected_type, memo) + elif issubclass(expected_type, (float, complex)): + check_number(argname, value, expected_type) + elif _subclass_check_unions and issubclass(expected_type, Union): + check_union(argname, value, expected_type, memo) + elif isinstance(expected_type, TypeVar): + check_typevar(argname, value, expected_type, memo) + elif issubclass(expected_type, IO): + check_io(argname, value, expected_type) + elif is_typeddict(expected_type): + check_typed_dict(argname, value, expected_type, memo) + elif getattr(expected_type, '_is_protocol', False): + check_protocol(argname, value, expected_type) + else: + expected_type = (getattr(expected_type, '__extra__', None) or origin_type or + expected_type) + + if expected_type is bytes: + # As per https://github.com/python/typing/issues/552 + if not isinstance(value, (bytearray, bytes, memoryview)): + raise TypeError('type of {} must be bytes-like; got {} instead' + .format(argname, qualified_name(value))) + elif not isinstance(value, expected_type): + raise TypeError( + 'type of {} must be {}; got {} instead'. + format(argname, qualified_name(expected_type), qualified_name(value))) + elif isinstance(expected_type, TypeVar): + # Only happens on < 3.6 + check_typevar(argname, value, expected_type, memo) + elif isinstance(expected_type, Literal.__class__): + # Only happens on < 3.7 when using Literal from typing_extensions + check_literal(argname, value, expected_type, memo) + elif expected_type.__class__ is NewType: + # typing.NewType on Python 3.10+ + return check_type(argname, value, expected_type.__supertype__, memo) + elif (isfunction(expected_type) and + getattr(expected_type, "__module__", None) == "typing" and + getattr(expected_type, "__qualname__", None).startswith("NewType.") and + hasattr(expected_type, "__supertype__")): + # typing.NewType on Python 3.9 and below + return check_type(argname, value, expected_type.__supertype__, memo) + + +def check_return_type(retval, memo: Optional[_CallMemo] = None) -> bool: + """ + Check that the return value is compatible with the return value annotation in the function. + + :param retval: the value about to be returned from the call + :return: ``True`` + :raises TypeError: if there is a type mismatch + + """ + if memo is None: + # faster than inspect.currentframe(), but not officially + # supported in all python implementations + frame = sys._getframe(1) + + try: + func = find_function(frame) + except LookupError: + return True # This can happen with the Pydev/PyCharm debugger extension installed + + memo = _CallMemo(func, frame.f_locals) + + if 'return' in memo.type_hints: + if memo.type_hints['return'] is NoReturn: + raise TypeError('{}() was declared never to return but it did'.format(memo.func_name)) + + try: + check_type('the return value', retval, memo.type_hints['return'], memo) + except TypeError as exc: # suppress unnecessarily long tracebacks + # Allow NotImplemented if this is a binary magic method (__eq__() et al) + if retval is NotImplemented and memo.type_hints['return'] is bool: + # This does (and cannot) not check if it's actually a method + func_name = memo.func_name.rsplit('.', 1)[-1] + if len(memo.arguments) == 2 and func_name in BINARY_MAGIC_METHODS: + return True + + raise TypeError(*exc.args) from None + + return True + + +def check_argument_types(memo: Optional[_CallMemo] = None) -> bool: + """ + Check that the argument values match the annotated types. + + Unless both ``args`` and ``kwargs`` are provided, the information will be retrieved from + the previous stack frame (ie. from the function that called this). + + :return: ``True`` + :raises TypeError: if there is an argument type mismatch + + """ + if memo is None: + # faster than inspect.currentframe(), but not officially + # supported in all python implementations + frame = sys._getframe(1) + + try: + func = find_function(frame) + except LookupError: + return True # This can happen with the Pydev/PyCharm debugger extension installed + + memo = _CallMemo(func, frame.f_locals) + + for argname, expected_type in memo.type_hints.items(): + if argname != 'return' and argname in memo.arguments: + value = memo.arguments[argname] + description = 'argument "{}"'.format(argname) + try: + check_type(description, value, expected_type, memo) + except TypeError as exc: # suppress unnecessarily long tracebacks + raise TypeError(*exc.args) from None + + return True + + +class TypeCheckedGenerator: + def __init__(self, wrapped: Generator, memo: _CallMemo): + rtype_args = [] + if hasattr(memo.type_hints['return'], "__args__"): + rtype_args = memo.type_hints['return'].__args__ + + self.__wrapped = wrapped + self.__memo = memo + self.__yield_type = rtype_args[0] if rtype_args else Any + self.__send_type = rtype_args[1] if len(rtype_args) > 1 else Any + self.__return_type = rtype_args[2] if len(rtype_args) > 2 else Any + self.__initialized = False + + def __iter__(self): + return self + + def __next__(self): + return self.send(None) + + def __getattr__(self, name: str) -> Any: + return getattr(self.__wrapped, name) + + def throw(self, *args): + return self.__wrapped.throw(*args) + + def close(self): + self.__wrapped.close() + + def send(self, obj): + if self.__initialized: + check_type('value sent to generator', obj, self.__send_type, memo=self.__memo) + else: + self.__initialized = True + + try: + value = self.__wrapped.send(obj) + except StopIteration as exc: + check_type('return value', exc.value, self.__return_type, memo=self.__memo) + raise + + check_type('value yielded from generator', value, self.__yield_type, memo=self.__memo) + return value + + +class TypeCheckedAsyncGenerator: + def __init__(self, wrapped: AsyncGenerator, memo: _CallMemo): + rtype_args = memo.type_hints['return'].__args__ + self.__wrapped = wrapped + self.__memo = memo + self.__yield_type = rtype_args[0] + self.__send_type = rtype_args[1] if len(rtype_args) > 1 else Any + self.__initialized = False + + def __aiter__(self): + return self + + def __anext__(self): + return self.asend(None) + + def __getattr__(self, name: str) -> Any: + return getattr(self.__wrapped, name) + + def athrow(self, *args): + return self.__wrapped.athrow(*args) + + def aclose(self): + return self.__wrapped.aclose() + + async def asend(self, obj): + if self.__initialized: + check_type('value sent to generator', obj, self.__send_type, memo=self.__memo) + else: + self.__initialized = True + + value = await self.__wrapped.asend(obj) + check_type('value yielded from generator', value, self.__yield_type, memo=self.__memo) + return value + + +@overload +def typechecked(*, always: bool = False) -> Callable[[T_CallableOrType], T_CallableOrType]: + ... + + +@overload +def typechecked(func: T_CallableOrType, *, always: bool = False) -> T_CallableOrType: + ... + + +def typechecked(func=None, *, always=False, _localns: Optional[Dict[str, Any]] = None): + """ + Perform runtime type checking on the arguments that are passed to the wrapped function. + + The return value is also checked against the return annotation if any. + + If the ``__debug__`` global variable is set to ``False``, no wrapping and therefore no type + checking is done, unless ``always`` is ``True``. + + This can also be used as a class decorator. This will wrap all type annotated methods, + including ``@classmethod``, ``@staticmethod``, and ``@property`` decorated methods, + in the class with the ``@typechecked`` decorator. + + :param func: the function or class to enable type checking for + :param always: ``True`` to enable type checks even in optimized mode + + """ + if func is None: + return partial(typechecked, always=always, _localns=_localns) + + if not __debug__ and not always: # pragma: no cover + return func + + if isclass(func): + prefix = func.__qualname__ + '.' + for key, attr in func.__dict__.items(): + if inspect.isfunction(attr) or inspect.ismethod(attr) or inspect.isclass(attr): + if attr.__qualname__.startswith(prefix) and getattr(attr, '__annotations__', None): + setattr(func, key, typechecked(attr, always=always, _localns=func.__dict__)) + elif isinstance(attr, (classmethod, staticmethod)): + if getattr(attr.__func__, '__annotations__', None): + wrapped = typechecked(attr.__func__, always=always, _localns=func.__dict__) + setattr(func, key, type(attr)(wrapped)) + elif isinstance(attr, property): + kwargs = dict(doc=attr.__doc__) + for name in ("fset", "fget", "fdel"): + property_func = kwargs[name] = getattr(attr, name) + if property_func is not None and getattr(property_func, '__annotations__', ()): + kwargs[name] = typechecked( + property_func, always=always, _localns=func.__dict__ + ) + + setattr(func, key, attr.__class__(**kwargs)) + + return func + + if not getattr(func, '__annotations__', None): + warn('no type annotations present -- not typechecking {}'.format(function_name(func))) + return func + + # Find the frame in which the function was declared, for resolving forward references later + if _localns is None: + _localns = sys._getframe(1).f_locals + + # Find either the first Python wrapper or the actual function + python_func = inspect.unwrap(func, stop=lambda f: hasattr(f, '__code__')) + + if not getattr(python_func, '__code__', None): + warn('no code associated -- not typechecking {}'.format(function_name(func))) + return func + + def wrapper(*args, **kwargs): + memo = _CallMemo(python_func, _localns, args=args, kwargs=kwargs) + check_argument_types(memo) + retval = func(*args, **kwargs) + try: + check_return_type(retval, memo) + except TypeError as exc: + raise TypeError(*exc.args) from None + + # If a generator is returned, wrap it if its yield/send/return types can be checked + if inspect.isgenerator(retval) or isasyncgen(retval): + return_type = memo.type_hints.get('return') + if return_type: + origin = getattr(return_type, '__origin__', None) + if origin in generator_origin_types: + return TypeCheckedGenerator(retval, memo) + elif origin is not None and origin in asyncgen_origin_types: + return TypeCheckedAsyncGenerator(retval, memo) + + return retval + + async def async_wrapper(*args, **kwargs): + memo = _CallMemo(python_func, _localns, args=args, kwargs=kwargs) + check_argument_types(memo) + retval = await func(*args, **kwargs) + check_return_type(retval, memo) + return retval + + if inspect.iscoroutinefunction(func): + if python_func.__code__ is not async_wrapper.__code__: + return wraps(func)(async_wrapper) + else: + if python_func.__code__ is not wrapper.__code__: + return wraps(func)(wrapper) + + # the target callable was already wrapped + return func + + +class TypeWarning(UserWarning): + """ + A warning that is emitted when a type check fails. + + :ivar str event: ``call`` or ``return`` + :ivar Callable func: the function in which the violation occurred (the called function if event + is ``call``, or the function where a value of the wrong type was returned from if event is + ``return``) + :ivar str error: the error message contained by the caught :class:`TypeError` + :ivar frame: the frame in which the violation occurred + """ + + __slots__ = ('func', 'event', 'message', 'frame') + + def __init__(self, memo: Optional[_CallMemo], event: str, frame, + exception: Union[str, TypeError]): # pragma: no cover + self.func = memo.func + self.event = event + self.error = str(exception) + self.frame = frame + + if self.event == 'call': + caller_frame = self.frame.f_back + event = 'call to {}() from {}:{}'.format( + function_name(self.func), caller_frame.f_code.co_filename, caller_frame.f_lineno) + else: + event = 'return from {}() at {}:{}'.format( + function_name(self.func), self.frame.f_code.co_filename, self.frame.f_lineno) + + super().__init__('[{thread_name}] {event}: {self.error}'.format( + thread_name=threading.current_thread().name, event=event, self=self)) + + @property + def stack(self): + """Return the stack where the last frame is from the target function.""" + return extract_stack(self.frame) + + def print_stack(self, file: TextIO = None, limit: int = None) -> None: + """ + Print the traceback from the stack frame where the target function was run. + + :param file: an open file to print to (prints to stdout if omitted) + :param limit: the maximum number of stack frames to print + + """ + print_stack(self.frame, limit, file) + + +class TypeChecker: + """ + A type checker that collects type violations by hooking into :func:`sys.setprofile`. + + :param packages: list of top level modules and packages or modules to include for type checking + :param all_threads: ``True`` to check types in all threads created while the checker is + running, ``False`` to only check in the current one + :param forward_refs_policy: how to handle unresolvable forward references in annotations + + .. deprecated:: 2.6 + Use :func:`~.importhook.install_import_hook` instead. This class will be removed in v3.0. + """ + + def __init__(self, packages: Union[str, Sequence[str]], *, all_threads: bool = True, + forward_refs_policy: ForwardRefPolicy = ForwardRefPolicy.ERROR): + assert check_argument_types() + warn('TypeChecker has been deprecated and will be removed in v3.0. ' + 'Use install_import_hook() or the pytest plugin instead.', DeprecationWarning) + self.all_threads = all_threads + self.annotation_policy = forward_refs_policy + self._call_memos = {} # type: Dict[Any, _CallMemo] + self._previous_profiler = None + self._previous_thread_profiler = None + self._active = False + + if isinstance(packages, str): + self._packages = (packages,) + else: + self._packages = tuple(packages) + + @property + def active(self) -> bool: + """Return ``True`` if currently collecting type violations.""" + return self._active + + def should_check_type(self, func: Callable) -> bool: + if not func.__annotations__: + # No point in checking if there are no type hints + return False + elif isasyncgenfunction(func): + # Async generators cannot be supported because the return arg is of an opaque builtin + # type (async_generator_wrapped_value) + return False + else: + # Check types if the module matches any of the package prefixes + return any(func.__module__ == package or func.__module__.startswith(package + '.') + for package in self._packages) + + def start(self): + if self._active: + raise RuntimeError('type checker already running') + + self._active = True + + # Install this instance as the current profiler + self._previous_profiler = sys.getprofile() + sys.setprofile(self) + + # If requested, set this instance as the default profiler for all future threads + # (does not affect existing threads) + if self.all_threads: + self._previous_thread_profiler = threading._profile_hook + threading.setprofile(self) + + def stop(self): + if self._active: + if sys.getprofile() is self: + sys.setprofile(self._previous_profiler) + else: # pragma: no cover + warn('the system profiling hook has changed unexpectedly') + + if self.all_threads: + if threading._profile_hook is self: + threading.setprofile(self._previous_thread_profiler) + else: # pragma: no cover + warn('the threading profiling hook has changed unexpectedly') + + self._active = False + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + def __call__(self, frame, event: str, arg) -> None: # pragma: no cover + if not self._active: + # This happens if all_threads was enabled and a thread was created when the checker was + # running but was then stopped. The thread's profiler callback can't be reset any other + # way but this. + sys.setprofile(self._previous_thread_profiler) + return + + # If an actual profiler is running, don't include the type checking times in its results + if event == 'call': + try: + func = find_function(frame) + except Exception: + func = None + + if func is not None and self.should_check_type(func): + memo = self._call_memos[frame] = _CallMemo( + func, frame.f_locals, forward_refs_policy=self.annotation_policy) + if memo.is_generator: + return_type_hint = memo.type_hints['return'] + if return_type_hint is not None: + origin = getattr(return_type_hint, '__origin__', None) + if origin in generator_origin_types: + # Check the types of the yielded values + memo.type_hints['return'] = return_type_hint.__args__[0] + else: + try: + check_argument_types(memo) + except TypeError as exc: + warn(TypeWarning(memo, event, frame, exc)) + + if self._previous_profiler is not None: + self._previous_profiler(frame, event, arg) + elif event == 'return': + if self._previous_profiler is not None: + self._previous_profiler(frame, event, arg) + + if arg is None: + # a None return value might mean an exception is being raised but we have no way of + # checking + return + + memo = self._call_memos.get(frame) + if memo is not None: + try: + if memo.is_generator: + check_type('yielded value', arg, memo.type_hints['return'], memo) + else: + check_return_type(arg, memo) + except TypeError as exc: + warn(TypeWarning(memo, event, frame, exc)) + + if not memo.is_generator: + del self._call_memos[frame] + elif self._previous_profiler is not None: + self._previous_profiler(frame, event, arg) diff --git a/contrib/python/typeguard/typeguard/importhook.py b/contrib/python/typeguard/typeguard/importhook.py new file mode 100644 index 00000000000..d4e237fe7de --- /dev/null +++ b/contrib/python/typeguard/typeguard/importhook.py @@ -0,0 +1,162 @@ +import ast +import sys +from importlib.abc import MetaPathFinder +from importlib.machinery import SourceFileLoader +from importlib.util import cache_from_source, decode_source +from inspect import isclass +from typing import Iterable, Type +from unittest.mock import patch + + +# The name of this function is magical +def _call_with_frames_removed(f, *args, **kwargs): + return f(*args, **kwargs) + + +def optimized_cache_from_source(path, debug_override=None): + return cache_from_source(path, debug_override, optimization='typeguard') + + +class TypeguardTransformer(ast.NodeVisitor): + def __init__(self) -> None: + self._parents = [] + + def visit_Module(self, node: ast.Module): + # Insert "import typeguard" after any "from __future__ ..." imports + for i, child in enumerate(node.body): + if isinstance(child, ast.ImportFrom) and child.module == '__future__': + continue + elif isinstance(child, ast.Expr) and isinstance(child.value, ast.Constant): + continue # module docstring + else: + node.body.insert(i, ast.Import(names=[ast.alias('typeguard', None)])) + break + + self._parents.append(node) + self.generic_visit(node) + self._parents.pop() + return node + + def visit_ClassDef(self, node: ast.ClassDef): + node.decorator_list.append( + ast.Attribute(ast.Name(id='typeguard', ctx=ast.Load()), 'typechecked', ast.Load()) + ) + self._parents.append(node) + self.generic_visit(node) + self._parents.pop() + return node + + def visit_FunctionDef(self, node: ast.FunctionDef): + # Let the class level decorator handle the methods of a class + if isinstance(self._parents[-1], ast.ClassDef): + return node + + has_annotated_args = any(arg for arg in node.args.args if arg.annotation) + has_annotated_return = bool(node.returns) + if has_annotated_args or has_annotated_return: + node.decorator_list.insert( + 0, + ast.Attribute(ast.Name(id='typeguard', ctx=ast.Load()), 'typechecked', ast.Load()) + ) + + self._parents.append(node) + self.generic_visit(node) + self._parents.pop() + return node + + +class TypeguardLoader(SourceFileLoader): + def source_to_code(self, data, path, *, _optimize=-1): + source = decode_source(data) + tree = _call_with_frames_removed(compile, source, path, 'exec', ast.PyCF_ONLY_AST, + dont_inherit=True, optimize=_optimize) + tree = TypeguardTransformer().visit(tree) + ast.fix_missing_locations(tree) + return _call_with_frames_removed(compile, tree, path, 'exec', + dont_inherit=True, optimize=_optimize) + + def exec_module(self, module): + # Use a custom optimization marker – the import lock should make this monkey patch safe + with patch('importlib._bootstrap_external.cache_from_source', optimized_cache_from_source): + return super().exec_module(module) + + +class TypeguardFinder(MetaPathFinder): + """ + Wraps another path finder and instruments the module with ``@typechecked`` if + :meth:`should_instrument` returns ``True``. + + Should not be used directly, but rather via :func:`~.install_import_hook`. + + .. versionadded:: 2.6 + + """ + + def __init__(self, packages, original_pathfinder): + self.packages = packages + self._original_pathfinder = original_pathfinder + + def find_spec(self, fullname, path=None, target=None): + if self.should_instrument(fullname): + spec = self._original_pathfinder.find_spec(fullname, path, target) + if spec is not None and isinstance(spec.loader, SourceFileLoader): + spec.loader = TypeguardLoader(spec.loader.name, spec.loader.path) + return spec + + return None + + def should_instrument(self, module_name: str) -> bool: + """ + Determine whether the module with the given name should be instrumented. + + :param module_name: full name of the module that is about to be imported (e.g. ``xyz.abc``) + + """ + for package in self.packages: + if module_name == package or module_name.startswith(package + '.'): + return True + + return False + + +class ImportHookManager: + def __init__(self, hook: MetaPathFinder): + self.hook = hook + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self.uninstall() + + def uninstall(self): + try: + sys.meta_path.remove(self.hook) + except ValueError: + pass # already removed + + +def install_import_hook(packages: Iterable[str], *, + cls: Type[TypeguardFinder] = TypeguardFinder) -> ImportHookManager: + """ + Install an import hook that decorates classes and functions with ``@typechecked``. + + This only affects modules loaded **after** this hook has been installed. + + :return: a context manager that uninstalls the hook on exit (or when you call ``.uninstall()``) + + .. versionadded:: 2.6 + + """ + if isinstance(packages, str): + packages = [packages] + + for i, finder in enumerate(sys.meta_path): + if isclass(finder) and finder.__name__ == 'PathFinder' and hasattr(finder, 'find_spec'): + break + else: + raise RuntimeError('Cannot find a PathFinder in sys.meta_path') + + hook = cls(packages, finder) + sys.meta_path.insert(0, hook) + return ImportHookManager(hook) diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/py.typed b/contrib/python/typeguard/typeguard/py.typed index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/importlib_metadata/py.typed +++ b/contrib/python/typeguard/typeguard/py.typed diff --git a/contrib/python/typeguard/typeguard/pytest_plugin.py b/contrib/python/typeguard/typeguard/pytest_plugin.py new file mode 100644 index 00000000000..caa128b5474 --- /dev/null +++ b/contrib/python/typeguard/typeguard/pytest_plugin.py @@ -0,0 +1,30 @@ +import sys + +from typeguard.importhook import install_import_hook + + +def pytest_addoption(parser): + group = parser.getgroup('typeguard') + group.addoption('--typeguard-packages', action='store', + help='comma separated name list of packages and modules to instrument for ' + 'type checking') + + +def pytest_configure(config): + value = config.getoption("typeguard_packages") + if not value: + return + + packages = [pkg.strip() for pkg in value.split(",")] + + already_imported_packages = sorted( + package for package in packages if package in sys.modules + ) + if already_imported_packages: + message = ( + "typeguard cannot check these packages because they " + "are already imported: {}" + ) + raise RuntimeError(message.format(", ".join(already_imported_packages))) + + install_import_hook(packages=packages) diff --git a/contrib/python/typeguard/ya.make b/contrib/python/typeguard/ya.make new file mode 100644 index 00000000000..53dd3e96730 --- /dev/null +++ b/contrib/python/typeguard/ya.make @@ -0,0 +1,30 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(2.13.3) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + typeguard/__init__.py + typeguard/importhook.py + typeguard/pytest_plugin.py +) + +RESOURCE_FILES( + PREFIX contrib/python/typeguard/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt + typeguard/py.typed +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/contrib/python/typing-extensions/py2/tests/ya.make b/contrib/python/typing-extensions/py2/tests/ya.make index fc39868dbf9..a44d7d18099 100644 --- a/contrib/python/typing-extensions/py2/tests/ya.make +++ b/contrib/python/typing-extensions/py2/tests/ya.make @@ -1,7 +1,5 @@ PY2TEST() -SUBSCRIBER(g:python-contrib) - PEERDIR( contrib/python/typing-extensions ) diff --git a/contrib/python/typing-extensions/py2/ya.make b/contrib/python/typing-extensions/py2/ya.make index 646061a04e7..75c0a74e855 100644 --- a/contrib/python/typing-extensions/py2/ya.make +++ b/contrib/python/typing-extensions/py2/ya.make @@ -2,8 +2,6 @@ PY2_LIBRARY() LICENSE(PSF-2.0) -SUBSCRIBER(g:python-contrib) - VERSION(3.10.0.2) NO_LINT() diff --git a/contrib/python/wheel/.dist-info/METADATA b/contrib/python/wheel/.dist-info/METADATA new file mode 100644 index 00000000000..f645dcb673a --- /dev/null +++ b/contrib/python/wheel/.dist-info/METADATA @@ -0,0 +1,66 @@ +Metadata-Version: 2.3 +Name: wheel +Version: 0.45.1 +Summary: A built-package format for Python +Keywords: wheel,packaging +Author-email: Daniel Holth <dholth@fastmail.fm> +Maintainer-email: Alex Grönholm <alex.gronholm@nextday.fi> +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Requires-Dist: pytest >= 6.0.0 ; extra == "test" +Requires-Dist: setuptools >= 65 ; extra == "test" +Project-URL: Changelog, https://wheel.readthedocs.io/en/stable/news.html +Project-URL: Documentation, https://wheel.readthedocs.io/ +Project-URL: Issue Tracker, https://github.com/pypa/wheel/issues +Project-URL: Source, https://github.com/pypa/wheel +Provides-Extra: test + +wheel +===== + +This is a command line tool for manipulating Python wheel files, as defined in +`PEP 427`_. It contains the following functionality: + +* Convert ``.egg`` archives into ``.whl`` +* Unpack wheel archives +* Repack wheel archives +* Add or remove tags in existing wheel archives + +.. _PEP 427: https://www.python.org/dev/peps/pep-0427/ + +Historical note +--------------- + +This project used to contain the implementation of the setuptools_ ``bdist_wheel`` +command, but as of setuptools v70.1, it no longer needs ``wheel`` installed for that to +work. Thus, you should install this **only** if you intend to use the ``wheel`` command +line tool! + +.. _setuptools: https://pypi.org/project/setuptools/ + +Documentation +------------- + +The documentation_ can be found on Read The Docs. + +.. _documentation: https://wheel.readthedocs.io/ + +Code of Conduct +--------------- + +Everyone interacting in the wheel project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. + +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + diff --git a/contrib/python/wheel/.dist-info/entry_points.txt b/contrib/python/wheel/.dist-info/entry_points.txt new file mode 100644 index 00000000000..06c9f69debd --- /dev/null +++ b/contrib/python/wheel/.dist-info/entry_points.txt @@ -0,0 +1,6 @@ +[console_scripts] +wheel=wheel.cli:main + +[distutils.commands] +bdist_wheel=wheel.bdist_wheel:bdist_wheel + diff --git a/contrib/python/wheel/LICENSE.txt b/contrib/python/wheel/LICENSE.txt new file mode 100644 index 00000000000..a31470f14c5 --- /dev/null +++ b/contrib/python/wheel/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2012 Daniel Holth <dholth@fastmail.fm> and contributors + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/wheel/README.rst b/contrib/python/wheel/README.rst new file mode 100644 index 00000000000..ea16e65466a --- /dev/null +++ b/contrib/python/wheel/README.rst @@ -0,0 +1,37 @@ +wheel +===== + +This is a command line tool for manipulating Python wheel files, as defined in +`PEP 427`_. It contains the following functionality: + +* Convert ``.egg`` archives into ``.whl`` +* Unpack wheel archives +* Repack wheel archives +* Add or remove tags in existing wheel archives + +.. _PEP 427: https://www.python.org/dev/peps/pep-0427/ + +Historical note +--------------- + +This project used to contain the implementation of the setuptools_ ``bdist_wheel`` +command, but as of setuptools v70.1, it no longer needs ``wheel`` installed for that to +work. Thus, you should install this **only** if you intend to use the ``wheel`` command +line tool! + +.. _setuptools: https://pypi.org/project/setuptools/ + +Documentation +------------- + +The documentation_ can be found on Read The Docs. + +.. _documentation: https://wheel.readthedocs.io/ + +Code of Conduct +--------------- + +Everyone interacting in the wheel project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. + +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/__init__.py b/contrib/python/wheel/wheel/__init__.py index a773bbbcd7d..3ab8f72d8b2 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/__init__.py +++ b/contrib/python/wheel/wheel/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -__version__ = "0.43.0" +__version__ = "0.45.1" diff --git a/contrib/python/wheel/wheel/__main__.py b/contrib/python/wheel/wheel/__main__.py new file mode 100644 index 00000000000..0be74537494 --- /dev/null +++ b/contrib/python/wheel/wheel/__main__.py @@ -0,0 +1,23 @@ +""" +Wheel command line tool (enable python -m wheel syntax) +""" + +from __future__ import annotations + +import sys + + +def main(): # needed for console script + if __package__ == "": + # To be able to run 'python wheel-0.9.whl/wheel': + import os.path + + path = os.path.dirname(os.path.dirname(__file__)) + sys.path[0:0] = [path] + import wheel.cli + + sys.exit(wheel.cli.main()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/wheel/wheel/_bdist_wheel.py b/contrib/python/wheel/wheel/_bdist_wheel.py new file mode 100644 index 00000000000..88973ebfb88 --- /dev/null +++ b/contrib/python/wheel/wheel/_bdist_wheel.py @@ -0,0 +1,613 @@ +""" +Create a wheel (.whl) distribution. + +A wheel is a built archive format. +""" + +from __future__ import annotations + +import os +import re +import shutil +import stat +import struct +import sys +import sysconfig +import warnings +from email.generator import BytesGenerator, Generator +from email.policy import EmailPolicy +from glob import iglob +from shutil import rmtree +from typing import TYPE_CHECKING, Callable, Iterable, Literal, Sequence, cast +from zipfile import ZIP_DEFLATED, ZIP_STORED + +import setuptools +from setuptools import Command + +from . import __version__ as wheel_version +from .metadata import pkginfo_to_metadata +from .util import log +from .vendored.packaging import tags +from .vendored.packaging import version as _packaging_version +from .wheelfile import WheelFile + +if TYPE_CHECKING: + import types + +# ensure Python logging is configured +try: + __import__("setuptools.logging") +except ImportError: + # setuptools < ?? + from . import _setuptools_logging + + _setuptools_logging.configure() + + +def safe_name(name: str) -> str: + """Convert an arbitrary string to a standard distribution name + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub("[^A-Za-z0-9.]+", "-", name) + + +def safe_version(version: str) -> str: + """ + Convert an arbitrary string to a standard version string + """ + try: + # normalize the version + return str(_packaging_version.Version(version)) + except _packaging_version.InvalidVersion: + version = version.replace(" ", ".") + return re.sub("[^A-Za-z0-9.]+", "-", version) + + +setuptools_major_version = int(setuptools.__version__.split(".")[0]) + +PY_LIMITED_API_PATTERN = r"cp3\d" + + +def _is_32bit_interpreter() -> bool: + return struct.calcsize("P") == 4 + + +def python_tag() -> str: + return f"py{sys.version_info[0]}" + + +def get_platform(archive_root: str | None) -> str: + """Return our platform name 'win32', 'linux_x86_64'""" + result = sysconfig.get_platform() + if result.startswith("macosx") and archive_root is not None: + from .macosx_libfile import calculate_macosx_platform_tag + + result = calculate_macosx_platform_tag(archive_root, result) + elif _is_32bit_interpreter(): + if result == "linux-x86_64": + # pip pull request #3497 + result = "linux-i686" + elif result == "linux-aarch64": + # packaging pull request #234 + # TODO armv8l, packaging pull request #690 => this did not land + # in pip/packaging yet + result = "linux-armv7l" + + return result.replace("-", "_") + + +def get_flag( + var: str, fallback: bool, expected: bool = True, warn: bool = True +) -> bool: + """Use a fallback value for determining SOABI flags if the needed config + var is unset or unavailable.""" + val = sysconfig.get_config_var(var) + if val is None: + if warn: + warnings.warn( + f"Config variable '{var}' is unset, Python ABI tag may be incorrect", + RuntimeWarning, + stacklevel=2, + ) + return fallback + return val == expected + + +def get_abi_tag() -> str | None: + """Return the ABI tag based on SOABI (if available) or emulate SOABI (PyPy2).""" + soabi: str = sysconfig.get_config_var("SOABI") + impl = tags.interpreter_name() + if not soabi and impl in ("cp", "pp") and hasattr(sys, "maxunicode"): + d = "" + m = "" + u = "" + if get_flag("Py_DEBUG", hasattr(sys, "gettotalrefcount"), warn=(impl == "cp")): + d = "d" + + if get_flag( + "WITH_PYMALLOC", + impl == "cp", + warn=(impl == "cp" and sys.version_info < (3, 8)), + ) and sys.version_info < (3, 8): + m = "m" + + abi = f"{impl}{tags.interpreter_version()}{d}{m}{u}" + elif soabi and impl == "cp" and soabi.startswith("cpython"): + # non-Windows + abi = "cp" + soabi.split("-")[1] + elif soabi and impl == "cp" and soabi.startswith("cp"): + # Windows + abi = soabi.split("-")[0] + elif soabi and impl == "pp": + # we want something like pypy36-pp73 + abi = "-".join(soabi.split("-")[:2]) + abi = abi.replace(".", "_").replace("-", "_") + elif soabi and impl == "graalpy": + abi = "-".join(soabi.split("-")[:3]) + abi = abi.replace(".", "_").replace("-", "_") + elif soabi: + abi = soabi.replace(".", "_").replace("-", "_") + else: + abi = None + + return abi + + +def safer_name(name: str) -> str: + return safe_name(name).replace("-", "_") + + +def safer_version(version: str) -> str: + return safe_version(version).replace("-", "_") + + +def remove_readonly( + func: Callable[..., object], + path: str, + excinfo: tuple[type[Exception], Exception, types.TracebackType], +) -> None: + remove_readonly_exc(func, path, excinfo[1]) + + +def remove_readonly_exc(func: Callable[..., object], path: str, exc: Exception) -> None: + os.chmod(path, stat.S_IWRITE) + func(path) + + +class bdist_wheel(Command): + description = "create a wheel distribution" + + supported_compressions = { + "stored": ZIP_STORED, + "deflated": ZIP_DEFLATED, + } + + user_options = [ + ("bdist-dir=", "b", "temporary directory for creating the distribution"), + ( + "plat-name=", + "p", + "platform name to embed in generated filenames " + f"(default: {get_platform(None)})", + ), + ( + "keep-temp", + "k", + "keep the pseudo-installation tree around after " + "creating the distribution archive", + ), + ("dist-dir=", "d", "directory to put final built distributions in"), + ("skip-build", None, "skip rebuilding everything (for testing/debugging)"), + ( + "relative", + None, + "build the archive using relative paths (default: false)", + ), + ( + "owner=", + "u", + "Owner name used when creating a tar file [default: current user]", + ), + ( + "group=", + "g", + "Group name used when creating a tar file [default: current group]", + ), + ("universal", None, "make a universal wheel (default: false)"), + ( + "compression=", + None, + "zipfile compression (one of: {}) (default: 'deflated')".format( + ", ".join(supported_compressions) + ), + ), + ( + "python-tag=", + None, + f"Python implementation compatibility tag (default: '{python_tag()}')", + ), + ( + "build-number=", + None, + "Build number for this particular version. " + "As specified in PEP-0427, this must start with a digit. " + "[default: None]", + ), + ( + "py-limited-api=", + None, + "Python tag (cp32|cp33|cpNN) for abi3 wheel tag (default: false)", + ), + ] + + boolean_options = ["keep-temp", "skip-build", "relative", "universal"] + + def initialize_options(self): + self.bdist_dir: str = None + self.data_dir = None + self.plat_name: str | None = None + self.plat_tag = None + self.format = "zip" + self.keep_temp = False + self.dist_dir: str | None = None + self.egginfo_dir = None + self.root_is_pure: bool | None = None + self.skip_build = None + self.relative = False + self.owner = None + self.group = None + self.universal: bool = False + self.compression: str | int = "deflated" + self.python_tag: str = python_tag() + self.build_number: str | None = None + self.py_limited_api: str | Literal[False] = False + self.plat_name_supplied = False + + def finalize_options(self): + if self.bdist_dir is None: + bdist_base = self.get_finalized_command("bdist").bdist_base + self.bdist_dir = os.path.join(bdist_base, "wheel") + + egg_info = self.distribution.get_command_obj("egg_info") + egg_info.ensure_finalized() # needed for correct `wheel_dist_name` + + self.data_dir = self.wheel_dist_name + ".data" + self.plat_name_supplied = self.plat_name is not None + + try: + self.compression = self.supported_compressions[self.compression] + except KeyError: + raise ValueError(f"Unsupported compression: {self.compression}") from None + + need_options = ("dist_dir", "plat_name", "skip_build") + + self.set_undefined_options("bdist", *zip(need_options, need_options)) + + self.root_is_pure = not ( + self.distribution.has_ext_modules() or self.distribution.has_c_libraries() + ) + + if self.py_limited_api and not re.match( + PY_LIMITED_API_PATTERN, self.py_limited_api + ): + raise ValueError(f"py-limited-api must match '{PY_LIMITED_API_PATTERN}'") + + # Support legacy [wheel] section for setting universal + wheel = self.distribution.get_option_dict("wheel") + if "universal" in wheel: + # please don't define this in your global configs + log.warning( + "The [wheel] section is deprecated. Use [bdist_wheel] instead.", + ) + val = wheel["universal"][1].strip() + if val.lower() in ("1", "true", "yes"): + self.universal = True + + if self.build_number is not None and not self.build_number[:1].isdigit(): + raise ValueError("Build tag (build-number) must start with a digit.") + + @property + def wheel_dist_name(self): + """Return distribution full name with - replaced with _""" + components = ( + safer_name(self.distribution.get_name()), + safer_version(self.distribution.get_version()), + ) + if self.build_number: + components += (self.build_number,) + return "-".join(components) + + def get_tag(self) -> tuple[str, str, str]: + # bdist sets self.plat_name if unset, we should only use it for purepy + # wheels if the user supplied it. + if self.plat_name_supplied: + plat_name = cast(str, self.plat_name) + elif self.root_is_pure: + plat_name = "any" + else: + # macosx contains system version in platform name so need special handle + if self.plat_name and not self.plat_name.startswith("macosx"): + plat_name = self.plat_name + else: + # on macosx always limit the platform name to comply with any + # c-extension modules in bdist_dir, since the user can specify + # a higher MACOSX_DEPLOYMENT_TARGET via tools like CMake + + # on other platforms, and on macosx if there are no c-extension + # modules, use the default platform name. + plat_name = get_platform(self.bdist_dir) + + if _is_32bit_interpreter(): + if plat_name in ("linux-x86_64", "linux_x86_64"): + plat_name = "linux_i686" + if plat_name in ("linux-aarch64", "linux_aarch64"): + # TODO armv8l, packaging pull request #690 => this did not land + # in pip/packaging yet + plat_name = "linux_armv7l" + + plat_name = ( + plat_name.lower().replace("-", "_").replace(".", "_").replace(" ", "_") + ) + + if self.root_is_pure: + if self.universal: + impl = "py2.py3" + else: + impl = self.python_tag + tag = (impl, "none", plat_name) + else: + impl_name = tags.interpreter_name() + impl_ver = tags.interpreter_version() + impl = impl_name + impl_ver + # We don't work on CPython 3.1, 3.0. + if self.py_limited_api and (impl_name + impl_ver).startswith("cp3"): + impl = self.py_limited_api + abi_tag = "abi3" + else: + abi_tag = str(get_abi_tag()).lower() + tag = (impl, abi_tag, plat_name) + # issue gh-374: allow overriding plat_name + supported_tags = [ + (t.interpreter, t.abi, plat_name) for t in tags.sys_tags() + ] + assert ( + tag in supported_tags + ), f"would build wheel with unsupported tag {tag}" + return tag + + def run(self): + build_scripts = self.reinitialize_command("build_scripts") + build_scripts.executable = "python" + build_scripts.force = True + + build_ext = self.reinitialize_command("build_ext") + build_ext.inplace = False + + if not self.skip_build: + self.run_command("build") + + install = self.reinitialize_command("install", reinit_subcommands=True) + install.root = self.bdist_dir + install.compile = False + install.skip_build = self.skip_build + install.warn_dir = False + + # A wheel without setuptools scripts is more cross-platform. + # Use the (undocumented) `no_ep` option to setuptools' + # install_scripts command to avoid creating entry point scripts. + install_scripts = self.reinitialize_command("install_scripts") + install_scripts.no_ep = True + + # Use a custom scheme for the archive, because we have to decide + # at installation time which scheme to use. + for key in ("headers", "scripts", "data", "purelib", "platlib"): + setattr(install, "install_" + key, os.path.join(self.data_dir, key)) + + basedir_observed = "" + + if os.name == "nt": + # win32 barfs if any of these are ''; could be '.'? + # (distutils.command.install:change_roots bug) + basedir_observed = os.path.normpath(os.path.join(self.data_dir, "..")) + self.install_libbase = self.install_lib = basedir_observed + + setattr( + install, + "install_purelib" if self.root_is_pure else "install_platlib", + basedir_observed, + ) + + log.info(f"installing to {self.bdist_dir}") + + self.run_command("install") + + impl_tag, abi_tag, plat_tag = self.get_tag() + archive_basename = f"{self.wheel_dist_name}-{impl_tag}-{abi_tag}-{plat_tag}" + if not self.relative: + archive_root = self.bdist_dir + else: + archive_root = os.path.join( + self.bdist_dir, self._ensure_relative(install.install_base) + ) + + self.set_undefined_options("install_egg_info", ("target", "egginfo_dir")) + distinfo_dirname = ( + f"{safer_name(self.distribution.get_name())}-" + f"{safer_version(self.distribution.get_version())}.dist-info" + ) + distinfo_dir = os.path.join(self.bdist_dir, distinfo_dirname) + self.egg2dist(self.egginfo_dir, distinfo_dir) + + self.write_wheelfile(distinfo_dir) + + # Make the archive + if not os.path.exists(self.dist_dir): + os.makedirs(self.dist_dir) + + wheel_path = os.path.join(self.dist_dir, archive_basename + ".whl") + with WheelFile(wheel_path, "w", self.compression) as wf: + wf.write_files(archive_root) + + # Add to 'Distribution.dist_files' so that the "upload" command works + getattr(self.distribution, "dist_files", []).append( + ( + "bdist_wheel", + "{}.{}".format(*sys.version_info[:2]), # like 3.7 + wheel_path, + ) + ) + + if not self.keep_temp: + log.info(f"removing {self.bdist_dir}") + if not self.dry_run: + if sys.version_info < (3, 12): + rmtree(self.bdist_dir, onerror=remove_readonly) + else: + rmtree(self.bdist_dir, onexc=remove_readonly_exc) + + def write_wheelfile( + self, wheelfile_base: str, generator: str = f"bdist_wheel ({wheel_version})" + ): + from email.message import Message + + msg = Message() + msg["Wheel-Version"] = "1.0" # of the spec + msg["Generator"] = generator + msg["Root-Is-Purelib"] = str(self.root_is_pure).lower() + if self.build_number is not None: + msg["Build"] = self.build_number + + # Doesn't work for bdist_wininst + impl_tag, abi_tag, plat_tag = self.get_tag() + for impl in impl_tag.split("."): + for abi in abi_tag.split("."): + for plat in plat_tag.split("."): + msg["Tag"] = "-".join((impl, abi, plat)) + + wheelfile_path = os.path.join(wheelfile_base, "WHEEL") + log.info(f"creating {wheelfile_path}") + with open(wheelfile_path, "wb") as f: + BytesGenerator(f, maxheaderlen=0).flatten(msg) + + def _ensure_relative(self, path: str) -> str: + # copied from dir_util, deleted + drive, path = os.path.splitdrive(path) + if path[0:1] == os.sep: + path = drive + path[1:] + return path + + @property + def license_paths(self) -> Iterable[str]: + if setuptools_major_version >= 57: + # Setuptools has resolved any patterns to actual file names + return self.distribution.metadata.license_files or () + + files: set[str] = set() + metadata = self.distribution.get_option_dict("metadata") + if setuptools_major_version >= 42: + # Setuptools recognizes the license_files option but does not do globbing + patterns = cast(Sequence[str], self.distribution.metadata.license_files) + else: + # Prior to those, wheel is entirely responsible for handling license files + if "license_files" in metadata: + patterns = metadata["license_files"][1].split() + else: + patterns = () + + if "license_file" in metadata: + warnings.warn( + 'The "license_file" option is deprecated. Use "license_files" instead.', + DeprecationWarning, + stacklevel=2, + ) + files.add(metadata["license_file"][1]) + + if not files and not patterns and not isinstance(patterns, list): + patterns = ("LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*") + + for pattern in patterns: + for path in iglob(pattern): + if path.endswith("~"): + log.debug( + f'ignoring license file "{path}" as it looks like a backup' + ) + continue + + if path not in files and os.path.isfile(path): + log.info( + f'adding license file "{path}" (matched pattern "{pattern}")' + ) + files.add(path) + + return files + + def egg2dist(self, egginfo_path: str, distinfo_path: str): + """Convert an .egg-info directory into a .dist-info directory""" + + def adios(p: str) -> None: + """Appropriately delete directory, file or link.""" + if os.path.exists(p) and not os.path.islink(p) and os.path.isdir(p): + shutil.rmtree(p) + elif os.path.exists(p): + os.unlink(p) + + adios(distinfo_path) + + if not os.path.exists(egginfo_path): + # There is no egg-info. This is probably because the egg-info + # file/directory is not named matching the distribution name used + # to name the archive file. Check for this case and report + # accordingly. + import glob + + pat = os.path.join(os.path.dirname(egginfo_path), "*.egg-info") + possible = glob.glob(pat) + err = f"Egg metadata expected at {egginfo_path} but not found" + if possible: + alt = os.path.basename(possible[0]) + err += f" ({alt} found - possible misnamed archive file?)" + + raise ValueError(err) + + if os.path.isfile(egginfo_path): + # .egg-info is a single file + pkg_info = pkginfo_to_metadata(egginfo_path, egginfo_path) + os.mkdir(distinfo_path) + else: + # .egg-info is a directory + pkginfo_path = os.path.join(egginfo_path, "PKG-INFO") + pkg_info = pkginfo_to_metadata(egginfo_path, pkginfo_path) + + # ignore common egg metadata that is useless to wheel + shutil.copytree( + egginfo_path, + distinfo_path, + ignore=lambda x, y: { + "PKG-INFO", + "requires.txt", + "SOURCES.txt", + "not-zip-safe", + }, + ) + + # delete dependency_links if it is only whitespace + dependency_links_path = os.path.join(distinfo_path, "dependency_links.txt") + with open(dependency_links_path, encoding="utf-8") as dependency_links_file: + dependency_links = dependency_links_file.read().strip() + if not dependency_links: + adios(dependency_links_path) + + pkg_info_path = os.path.join(distinfo_path, "METADATA") + serialization_policy = EmailPolicy( + utf8=True, + mangle_from_=False, + max_line_length=0, + ) + with open(pkg_info_path, "w", encoding="utf-8") as out: + Generator(out, policy=serialization_policy).flatten(pkg_info) + + for license_path in self.license_paths: + filename = os.path.basename(license_path) + shutil.copy(license_path, os.path.join(distinfo_path, filename)) + + adios(egginfo_path) diff --git a/contrib/python/wheel/wheel/_setuptools_logging.py b/contrib/python/wheel/wheel/_setuptools_logging.py new file mode 100644 index 00000000000..a1a2482ba29 --- /dev/null +++ b/contrib/python/wheel/wheel/_setuptools_logging.py @@ -0,0 +1,26 @@ +# copied from setuptools.logging, omitting monkeypatching +from __future__ import annotations + +import logging +import sys + + +def _not_warning(record: logging.LogRecord) -> bool: + return record.levelno < logging.WARNING + + +def configure() -> None: + """ + Configure logging to emit warning and above to stderr + and everything else to stdout. This behavior is provided + for compatibility with distutils.log but may change in + the future. + """ + err_handler = logging.StreamHandler() + err_handler.setLevel(logging.WARNING) + out_handler = logging.StreamHandler(sys.stdout) + out_handler.addFilter(_not_warning) + handlers = err_handler, out_handler + logging.basicConfig( + format="{message}", style="{", handlers=handlers, level=logging.DEBUG + ) diff --git a/contrib/python/wheel/wheel/bdist_wheel.py b/contrib/python/wheel/wheel/bdist_wheel.py new file mode 100644 index 00000000000..dd7b8629e50 --- /dev/null +++ b/contrib/python/wheel/wheel/bdist_wheel.py @@ -0,0 +1,26 @@ +from typing import TYPE_CHECKING +from warnings import warn + +warn( + "The 'wheel' package is no longer the canonical location of the 'bdist_wheel' " + "command, and will be removed in a future release. Please update to setuptools " + "v70.1 or later which contains an integrated version of this command.", + DeprecationWarning, + stacklevel=1, +) + +if TYPE_CHECKING: + from ._bdist_wheel import bdist_wheel as bdist_wheel +else: + try: + # Better integration/compatibility with setuptools: + # in the case new fixes or PEPs are implemented in setuptools + # there is no need to backport them to the deprecated code base. + # This is useful in the case of old packages in the ecosystem + # that are still used but have low maintenance. + from setuptools.command.bdist_wheel import bdist_wheel + except ImportError: + # Only used in the case of old setuptools versions. + # If the user wants to get the latest fixes/PEPs, + # they are encouraged to address the deprecation warning. + from ._bdist_wheel import bdist_wheel as bdist_wheel diff --git a/contrib/python/wheel/wheel/cli/__init__.py b/contrib/python/wheel/wheel/cli/__init__.py new file mode 100644 index 00000000000..6ba1217f5bd --- /dev/null +++ b/contrib/python/wheel/wheel/cli/__init__.py @@ -0,0 +1,155 @@ +""" +Wheel command-line utility. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from argparse import ArgumentTypeError + + +class WheelError(Exception): + pass + + +def unpack_f(args: argparse.Namespace) -> None: + from .unpack import unpack + + unpack(args.wheelfile, args.dest) + + +def pack_f(args: argparse.Namespace) -> None: + from .pack import pack + + pack(args.directory, args.dest_dir, args.build_number) + + +def convert_f(args: argparse.Namespace) -> None: + from .convert import convert + + convert(args.files, args.dest_dir, args.verbose) + + +def tags_f(args: argparse.Namespace) -> None: + from .tags import tags + + names = ( + tags( + wheel, + args.python_tag, + args.abi_tag, + args.platform_tag, + args.build, + args.remove, + ) + for wheel in args.wheel + ) + + for name in names: + print(name) + + +def version_f(args: argparse.Namespace) -> None: + from .. import __version__ + + print(f"wheel {__version__}") + + +def parse_build_tag(build_tag: str) -> str: + if build_tag and not build_tag[0].isdigit(): + raise ArgumentTypeError("build tag must begin with a digit") + elif "-" in build_tag: + raise ArgumentTypeError("invalid character ('-') in build tag") + + return build_tag + + +TAGS_HELP = """\ +Make a new wheel with given tags. Any tags unspecified will remain the same. +Starting the tags with a "+" will append to the existing tags. Starting with a +"-" will remove a tag (use --option=-TAG syntax). Multiple tags can be +separated by ".". The original file will remain unless --remove is given. The +output filename(s) will be displayed on stdout for further processing. +""" + + +def parser(): + p = argparse.ArgumentParser() + s = p.add_subparsers(help="commands") + + unpack_parser = s.add_parser("unpack", help="Unpack wheel") + unpack_parser.add_argument( + "--dest", "-d", help="Destination directory", default="." + ) + unpack_parser.add_argument("wheelfile", help="Wheel file") + unpack_parser.set_defaults(func=unpack_f) + + repack_parser = s.add_parser("pack", help="Repack wheel") + repack_parser.add_argument("directory", help="Root directory of the unpacked wheel") + repack_parser.add_argument( + "--dest-dir", + "-d", + default=os.path.curdir, + help="Directory to store the wheel (default %(default)s)", + ) + repack_parser.add_argument( + "--build-number", help="Build tag to use in the wheel name" + ) + repack_parser.set_defaults(func=pack_f) + + convert_parser = s.add_parser("convert", help="Convert egg or wininst to wheel") + convert_parser.add_argument("files", nargs="*", help="Files to convert") + convert_parser.add_argument( + "--dest-dir", + "-d", + default=os.path.curdir, + help="Directory to store wheels (default %(default)s)", + ) + convert_parser.add_argument("--verbose", "-v", action="store_true") + convert_parser.set_defaults(func=convert_f) + + tags_parser = s.add_parser( + "tags", help="Add or replace the tags on a wheel", description=TAGS_HELP + ) + tags_parser.add_argument("wheel", nargs="*", help="Existing wheel(s) to retag") + tags_parser.add_argument( + "--remove", + action="store_true", + help="Remove the original files, keeping only the renamed ones", + ) + tags_parser.add_argument( + "--python-tag", metavar="TAG", help="Specify an interpreter tag(s)" + ) + tags_parser.add_argument("--abi-tag", metavar="TAG", help="Specify an ABI tag(s)") + tags_parser.add_argument( + "--platform-tag", metavar="TAG", help="Specify a platform tag(s)" + ) + tags_parser.add_argument( + "--build", type=parse_build_tag, metavar="BUILD", help="Specify a build tag" + ) + tags_parser.set_defaults(func=tags_f) + + version_parser = s.add_parser("version", help="Print version and exit") + version_parser.set_defaults(func=version_f) + + help_parser = s.add_parser("help", help="Show this help") + help_parser.set_defaults(func=lambda args: p.print_help()) + + return p + + +def main(): + p = parser() + args = p.parse_args() + if not hasattr(args, "func"): + p.print_help() + else: + try: + args.func(args) + return 0 + except WheelError as e: + print(e, file=sys.stderr) + + return 1 diff --git a/contrib/python/wheel/wheel/cli/convert.py b/contrib/python/wheel/wheel/cli/convert.py new file mode 100644 index 00000000000..61d4775c585 --- /dev/null +++ b/contrib/python/wheel/wheel/cli/convert.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +import os.path +import re +from abc import ABCMeta, abstractmethod +from collections import defaultdict +from collections.abc import Iterator +from email.message import Message +from email.parser import Parser +from email.policy import EmailPolicy +from glob import iglob +from pathlib import Path +from textwrap import dedent +from zipfile import ZipFile + +from .. import __version__ +from ..metadata import generate_requirements +from ..vendored.packaging.tags import parse_tag +from ..wheelfile import WheelFile + +egg_filename_re = re.compile( + r""" + (?P<name>.+?)-(?P<ver>.+?) + (-(?P<pyver>py\d\.\d+) + (-(?P<arch>.+?))? + )?.egg$""", + re.VERBOSE, +) +egg_info_re = re.compile( + r""" + ^(?P<name>.+?)-(?P<ver>.+?) + (-(?P<pyver>py\d\.\d+) + )?.egg-info/""", + re.VERBOSE, +) +wininst_re = re.compile( + r"\.(?P<platform>win32|win-amd64)(?:-(?P<pyver>py\d\.\d))?\.exe$" +) +pyd_re = re.compile(r"\.(?P<abi>[a-z0-9]+)-(?P<platform>win32|win_amd64)\.pyd$") +serialization_policy = EmailPolicy( + utf8=True, + mangle_from_=False, + max_line_length=0, +) +GENERATOR = f"wheel {__version__}" + + +def convert_requires(requires: str, metadata: Message) -> None: + extra: str | None = None + requirements: dict[str | None, list[str]] = defaultdict(list) + for line in requires.splitlines(): + line = line.strip() + if not line: + continue + + if line.startswith("[") and line.endswith("]"): + extra = line[1:-1] + continue + + requirements[extra].append(line) + + for key, value in generate_requirements(requirements): + metadata.add_header(key, value) + + +def convert_pkg_info(pkginfo: str, metadata: Message): + parsed_message = Parser().parsestr(pkginfo) + for key, value in parsed_message.items(): + key_lower = key.lower() + if value == "UNKNOWN": + continue + + if key_lower == "description": + description_lines = value.splitlines() + value = "\n".join( + ( + description_lines[0].lstrip(), + dedent("\n".join(description_lines[1:])), + "\n", + ) + ) + metadata.set_payload(value) + elif key_lower == "home-page": + metadata.add_header("Project-URL", f"Homepage, {value}") + elif key_lower == "download-url": + metadata.add_header("Project-URL", f"Download, {value}") + else: + metadata.add_header(key, value) + + metadata.replace_header("Metadata-Version", "2.4") + + +def normalize(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower().replace("-", "_") + + +class ConvertSource(metaclass=ABCMeta): + name: str + version: str + pyver: str = "py2.py3" + abi: str = "none" + platform: str = "any" + metadata: Message + + @property + def dist_info_dir(self) -> str: + return f"{self.name}-{self.version}.dist-info" + + @abstractmethod + def generate_contents(self) -> Iterator[tuple[str, bytes]]: + pass + + +class EggFileSource(ConvertSource): + def __init__(self, path: Path): + if not (match := egg_filename_re.match(path.name)): + raise ValueError(f"Invalid egg file name: {path.name}") + + # Binary wheels are assumed to be for CPython + self.path = path + self.name = normalize(match.group("name")) + self.version = match.group("ver") + if pyver := match.group("pyver"): + self.pyver = pyver.replace(".", "") + if arch := match.group("arch"): + self.abi = self.pyver.replace("py", "cp") + self.platform = normalize(arch) + + self.metadata = Message() + + def generate_contents(self) -> Iterator[tuple[str, bytes]]: + with ZipFile(self.path, "r") as zip_file: + for filename in sorted(zip_file.namelist()): + # Skip pure directory entries + if filename.endswith("/"): + continue + + # Handle files in the egg-info directory specially, selectively moving + # them to the dist-info directory while converting as needed + if filename.startswith("EGG-INFO/"): + if filename == "EGG-INFO/requires.txt": + requires = zip_file.read(filename).decode("utf-8") + convert_requires(requires, self.metadata) + elif filename == "EGG-INFO/PKG-INFO": + pkginfo = zip_file.read(filename).decode("utf-8") + convert_pkg_info(pkginfo, self.metadata) + elif filename == "EGG-INFO/entry_points.txt": + yield ( + f"{self.dist_info_dir}/entry_points.txt", + zip_file.read(filename), + ) + + continue + + # For any other file, just pass it through + yield filename, zip_file.read(filename) + + +class EggDirectorySource(EggFileSource): + def generate_contents(self) -> Iterator[tuple[str, bytes]]: + for dirpath, _, filenames in os.walk(self.path): + for filename in sorted(filenames): + path = Path(dirpath, filename) + if path.parent.name == "EGG-INFO": + if path.name == "requires.txt": + requires = path.read_text("utf-8") + convert_requires(requires, self.metadata) + elif path.name == "PKG-INFO": + pkginfo = path.read_text("utf-8") + convert_pkg_info(pkginfo, self.metadata) + if name := self.metadata.get("Name"): + self.name = normalize(name) + + if version := self.metadata.get("Version"): + self.version = version + elif path.name == "entry_points.txt": + yield ( + f"{self.dist_info_dir}/entry_points.txt", + path.read_bytes(), + ) + + continue + + # For any other file, just pass it through + yield str(path.relative_to(self.path)), path.read_bytes() + + +class WininstFileSource(ConvertSource): + """ + Handles distributions created with ``bdist_wininst``. + + The egginfo filename has the format:: + + name-ver(-pyver)(-arch).egg-info + + The installer filename has the format:: + + name-ver.arch(-pyver).exe + + Some things to note: + + 1. The installer filename is not definitive. An installer can be renamed + and work perfectly well as an installer. So more reliable data should + be used whenever possible. + 2. The egg-info data should be preferred for the name and version, because + these come straight from the distutils metadata, and are mandatory. + 3. The pyver from the egg-info data should be ignored, as it is + constructed from the version of Python used to build the installer, + which is irrelevant - the installer filename is correct here (even to + the point that when it's not there, any version is implied). + 4. The architecture must be taken from the installer filename, as it is + not included in the egg-info data. + 5. Architecture-neutral installers still have an architecture because the + installer format itself (being executable) is architecture-specific. We + should therefore ignore the architecture if the content is pure-python. + """ + + def __init__(self, path: Path): + self.path = path + self.metadata = Message() + + # Determine the initial architecture and Python version from the file name + # (if possible) + if match := wininst_re.search(path.name): + self.platform = normalize(match.group("platform")) + if pyver := match.group("pyver"): + self.pyver = pyver.replace(".", "") + + # Look for an .egg-info directory and any .pyd files for more precise info + egg_info_found = pyd_found = False + with ZipFile(self.path) as zip_file: + for filename in zip_file.namelist(): + prefix, filename = filename.split("/", 1) + if not egg_info_found and (match := egg_info_re.match(filename)): + egg_info_found = True + self.name = normalize(match.group("name")) + self.version = match.group("ver") + if pyver := match.group("pyver"): + self.pyver = pyver.replace(".", "") + elif not pyd_found and (match := pyd_re.search(filename)): + pyd_found = True + self.abi = match.group("abi") + self.platform = match.group("platform") + + if egg_info_found and pyd_found: + break + + def generate_contents(self) -> Iterator[tuple[str, bytes]]: + dist_info_dir = f"{self.name}-{self.version}.dist-info" + data_dir = f"{self.name}-{self.version}.data" + with ZipFile(self.path, "r") as zip_file: + for filename in sorted(zip_file.namelist()): + # Skip pure directory entries + if filename.endswith("/"): + continue + + # Handle files in the egg-info directory specially, selectively moving + # them to the dist-info directory while converting as needed + prefix, target_filename = filename.split("/", 1) + if egg_info_re.search(target_filename): + basename = target_filename.rsplit("/", 1)[-1] + if basename == "requires.txt": + requires = zip_file.read(filename).decode("utf-8") + convert_requires(requires, self.metadata) + elif basename == "PKG-INFO": + pkginfo = zip_file.read(filename).decode("utf-8") + convert_pkg_info(pkginfo, self.metadata) + elif basename == "entry_points.txt": + yield ( + f"{dist_info_dir}/entry_points.txt", + zip_file.read(filename), + ) + + continue + elif prefix == "SCRIPTS": + target_filename = f"{data_dir}/scripts/{target_filename}" + + # For any other file, just pass it through + yield target_filename, zip_file.read(filename) + + +def convert(files: list[str], dest_dir: str, verbose: bool) -> None: + for pat in files: + for archive in iglob(pat): + path = Path(archive) + if path.suffix == ".egg": + if path.is_dir(): + source: ConvertSource = EggDirectorySource(path) + else: + source = EggFileSource(path) + else: + source = WininstFileSource(path) + + if verbose: + print(f"{archive}...", flush=True, end="") + + dest_path = Path(dest_dir) / ( + f"{source.name}-{source.version}-{source.pyver}-{source.abi}" + f"-{source.platform}.whl" + ) + with WheelFile(dest_path, "w") as wheelfile: + for name_or_zinfo, contents in source.generate_contents(): + wheelfile.writestr(name_or_zinfo, contents) + + # Write the METADATA file + wheelfile.writestr( + f"{source.dist_info_dir}/METADATA", + source.metadata.as_string(policy=serialization_policy).encode( + "utf-8" + ), + ) + + # Write the WHEEL file + wheel_message = Message() + wheel_message.add_header("Wheel-Version", "1.0") + wheel_message.add_header("Generator", GENERATOR) + wheel_message.add_header( + "Root-Is-Purelib", str(source.platform == "any").lower() + ) + tags = parse_tag(f"{source.pyver}-{source.abi}-{source.platform}") + for tag in sorted(tags, key=lambda tag: tag.interpreter): + wheel_message.add_header("Tag", str(tag)) + + wheelfile.writestr( + f"{source.dist_info_dir}/WHEEL", + wheel_message.as_string(policy=serialization_policy).encode( + "utf-8" + ), + ) + + if verbose: + print("OK") diff --git a/contrib/python/wheel/wheel/cli/pack.py b/contrib/python/wheel/wheel/cli/pack.py new file mode 100644 index 00000000000..64469c0c730 --- /dev/null +++ b/contrib/python/wheel/wheel/cli/pack.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import email.policy +import os.path +import re +from email.generator import BytesGenerator +from email.parser import BytesParser + +from wheel.cli import WheelError +from wheel.wheelfile import WheelFile + +DIST_INFO_RE = re.compile(r"^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))\.dist-info$") + + +def pack(directory: str, dest_dir: str, build_number: str | None) -> None: + """Repack a previously unpacked wheel directory into a new wheel file. + + The .dist-info/WHEEL file must contain one or more tags so that the target + wheel file name can be determined. + + :param directory: The unpacked wheel directory + :param dest_dir: Destination directory (defaults to the current directory) + """ + # Find the .dist-info directory + dist_info_dirs = [ + fn + for fn in os.listdir(directory) + if os.path.isdir(os.path.join(directory, fn)) and DIST_INFO_RE.match(fn) + ] + if len(dist_info_dirs) > 1: + raise WheelError(f"Multiple .dist-info directories found in {directory}") + elif not dist_info_dirs: + raise WheelError(f"No .dist-info directories found in {directory}") + + # Determine the target wheel filename + dist_info_dir = dist_info_dirs[0] + name_version = DIST_INFO_RE.match(dist_info_dir).group("namever") + + # Read the tags and the existing build number from .dist-info/WHEEL + wheel_file_path = os.path.join(directory, dist_info_dir, "WHEEL") + with open(wheel_file_path, "rb") as f: + info = BytesParser(policy=email.policy.compat32).parse(f) + tags: list[str] = info.get_all("Tag", []) + existing_build_number = info.get("Build") + + if not tags: + raise WheelError( + f"No tags present in {dist_info_dir}/WHEEL; cannot determine target " + f"wheel filename" + ) + + # Set the wheel file name and add/replace/remove the Build tag in .dist-info/WHEEL + build_number = build_number if build_number is not None else existing_build_number + if build_number is not None: + del info["Build"] + if build_number: + info["Build"] = build_number + name_version += "-" + build_number + + if build_number != existing_build_number: + with open(wheel_file_path, "wb") as f: + BytesGenerator(f, maxheaderlen=0).flatten(info) + + # Reassemble the tags for the wheel file + tagline = compute_tagline(tags) + + # Repack the wheel + wheel_path = os.path.join(dest_dir, f"{name_version}-{tagline}.whl") + with WheelFile(wheel_path, "w") as wf: + print(f"Repacking wheel as {wheel_path}...", end="", flush=True) + wf.write_files(directory) + + print("OK") + + +def compute_tagline(tags: list[str]) -> str: + """Compute a tagline from a list of tags. + + :param tags: A list of tags + :return: A tagline + """ + impls = sorted({tag.split("-")[0] for tag in tags}) + abivers = sorted({tag.split("-")[1] for tag in tags}) + platforms = sorted({tag.split("-")[2] for tag in tags}) + return "-".join([".".join(impls), ".".join(abivers), ".".join(platforms)]) diff --git a/contrib/python/wheel/wheel/cli/tags.py b/contrib/python/wheel/wheel/cli/tags.py new file mode 100644 index 00000000000..88da72e9ec4 --- /dev/null +++ b/contrib/python/wheel/wheel/cli/tags.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import email.policy +import itertools +import os +from collections.abc import Iterable +from email.parser import BytesParser + +from ..wheelfile import WheelFile + + +def _compute_tags(original_tags: Iterable[str], new_tags: str | None) -> set[str]: + """Add or replace tags. Supports dot-separated tags""" + if new_tags is None: + return set(original_tags) + + if new_tags.startswith("+"): + return {*original_tags, *new_tags[1:].split(".")} + + if new_tags.startswith("-"): + return set(original_tags) - set(new_tags[1:].split(".")) + + return set(new_tags.split(".")) + + +def tags( + wheel: str, + python_tags: str | None = None, + abi_tags: str | None = None, + platform_tags: str | None = None, + build_tag: str | None = None, + remove: bool = False, +) -> str: + """Change the tags on a wheel file. + + The tags are left unchanged if they are not specified. To specify "none", + use ["none"]. To append to the previous tags, a tag should start with a + "+". If a tag starts with "-", it will be removed from existing tags. + Processing is done left to right. + + :param wheel: The paths to the wheels + :param python_tags: The Python tags to set + :param abi_tags: The ABI tags to set + :param platform_tags: The platform tags to set + :param build_tag: The build tag to set + :param remove: Remove the original wheel + """ + with WheelFile(wheel, "r") as f: + assert f.filename, f"{f.filename} must be available" + + wheel_info = f.read(f.dist_info_path + "/WHEEL") + info = BytesParser(policy=email.policy.compat32).parsebytes(wheel_info) + + original_wheel_name = os.path.basename(f.filename) + namever = f.parsed_filename.group("namever") + build = f.parsed_filename.group("build") + original_python_tags = f.parsed_filename.group("pyver").split(".") + original_abi_tags = f.parsed_filename.group("abi").split(".") + original_plat_tags = f.parsed_filename.group("plat").split(".") + + tags: list[str] = info.get_all("Tag", []) + existing_build_tag = info.get("Build") + + impls = {tag.split("-")[0] for tag in tags} + abivers = {tag.split("-")[1] for tag in tags} + platforms = {tag.split("-")[2] for tag in tags} + + if impls != set(original_python_tags): + msg = f"Wheel internal tags {impls!r} != filename tags {original_python_tags!r}" + raise AssertionError(msg) + + if abivers != set(original_abi_tags): + msg = f"Wheel internal tags {abivers!r} != filename tags {original_abi_tags!r}" + raise AssertionError(msg) + + if platforms != set(original_plat_tags): + msg = ( + f"Wheel internal tags {platforms!r} != filename tags {original_plat_tags!r}" + ) + raise AssertionError(msg) + + if existing_build_tag != build: + msg = ( + f"Incorrect filename '{build}' " + f"& *.dist-info/WHEEL '{existing_build_tag}' build numbers" + ) + raise AssertionError(msg) + + # Start changing as needed + if build_tag is not None: + build = build_tag + + final_python_tags = sorted(_compute_tags(original_python_tags, python_tags)) + final_abi_tags = sorted(_compute_tags(original_abi_tags, abi_tags)) + final_plat_tags = sorted(_compute_tags(original_plat_tags, platform_tags)) + + final_tags = [ + namever, + ".".join(final_python_tags), + ".".join(final_abi_tags), + ".".join(final_plat_tags), + ] + if build: + final_tags.insert(1, build) + + final_wheel_name = "-".join(final_tags) + ".whl" + + if original_wheel_name != final_wheel_name: + del info["Tag"], info["Build"] + for a, b, c in itertools.product( + final_python_tags, final_abi_tags, final_plat_tags + ): + info["Tag"] = f"{a}-{b}-{c}" + if build: + info["Build"] = build + + original_wheel_path = os.path.join( + os.path.dirname(f.filename), original_wheel_name + ) + final_wheel_path = os.path.join(os.path.dirname(f.filename), final_wheel_name) + + with WheelFile(original_wheel_path, "r") as fin, WheelFile( + final_wheel_path, "w" + ) as fout: + fout.comment = fin.comment # preserve the comment + for item in fin.infolist(): + if item.is_dir(): + continue + if item.filename == f.dist_info_path + "/RECORD": + continue + if item.filename == f.dist_info_path + "/WHEEL": + fout.writestr(item, info.as_bytes()) + else: + fout.writestr(item, fin.read(item)) + + if remove: + os.remove(original_wheel_path) + + return final_wheel_name diff --git a/contrib/python/wheel/wheel/cli/unpack.py b/contrib/python/wheel/wheel/cli/unpack.py new file mode 100644 index 00000000000..d48840e6ec0 --- /dev/null +++ b/contrib/python/wheel/wheel/cli/unpack.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from ..wheelfile import WheelFile + + +def unpack(path: str, dest: str = ".") -> None: + """Unpack a wheel. + + Wheel content will be unpacked to {dest}/{name}-{ver}, where {name} + is the package name and {ver} its version. + + :param path: The path to the wheel. + :param dest: Destination directory (default to current directory). + """ + with WheelFile(path) as wf: + namever = wf.parsed_filename.group("namever") + destination = Path(dest) / namever + print(f"Unpacking to: {destination}...", end="", flush=True) + for zinfo in wf.filelist: + wf.extract(zinfo, destination) + + # Set permissions to the same values as they were set in the archive + # We have to do this manually due to + # https://github.com/python/cpython/issues/59999 + permissions = zinfo.external_attr >> 16 & 0o777 + destination.joinpath(zinfo.filename).chmod(permissions) + + print("OK") diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/macosx_libfile.py b/contrib/python/wheel/wheel/macosx_libfile.py index 8953c3f8051..abdfc9eda1d 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/macosx_libfile.py +++ b/contrib/python/wheel/wheel/macosx_libfile.py @@ -43,6 +43,13 @@ from __future__ import annotations import ctypes import os import sys +from io import BufferedIOBase +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Union + + StrPath = Union[str, os.PathLike[str]] """here the needed const and struct from mach-o header files""" @@ -238,7 +245,7 @@ struct build_version_command { """ -def swap32(x): +def swap32(x: int) -> int: return ( ((x << 24) & 0xFF000000) | ((x << 8) & 0x00FF0000) @@ -247,7 +254,10 @@ def swap32(x): ) -def get_base_class_and_magic_number(lib_file, seek=None): +def get_base_class_and_magic_number( + lib_file: BufferedIOBase, + seek: int | None = None, +) -> tuple[type[ctypes.Structure], int]: if seek is None: seek = lib_file.tell() else: @@ -271,11 +281,11 @@ def get_base_class_and_magic_number(lib_file, seek=None): return BaseClass, magic_number -def read_data(struct_class, lib_file): +def read_data(struct_class: type[ctypes.Structure], lib_file: BufferedIOBase): return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class))) -def extract_macosx_min_system_version(path_to_lib): +def extract_macosx_min_system_version(path_to_lib: str): with open(path_to_lib, "rb") as lib_file: BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0) if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]: @@ -301,7 +311,7 @@ def extract_macosx_min_system_version(path_to_lib): read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch) ] - versions_list = [] + versions_list: list[tuple[int, int, int]] = [] for el in fat_arch_list: try: version = read_mach_header(lib_file, el.offset) @@ -333,7 +343,10 @@ def extract_macosx_min_system_version(path_to_lib): return None -def read_mach_header(lib_file, seek=None): +def read_mach_header( + lib_file: BufferedIOBase, + seek: int | None = None, +) -> tuple[int, int, int] | None: """ This function parses a Mach-O header and extracts information about the minimal macOS version. @@ -380,14 +393,14 @@ def read_mach_header(lib_file, seek=None): continue -def parse_version(version): +def parse_version(version: int) -> tuple[int, int, int]: x = (version & 0xFFFF0000) >> 16 y = (version & 0x0000FF00) >> 8 z = version & 0x000000FF return x, y, z -def calculate_macosx_platform_tag(archive_root, platform_tag): +def calculate_macosx_platform_tag(archive_root: StrPath, platform_tag: str) -> str: """ Calculate proper macosx platform tag basing on files which are included to wheel @@ -420,7 +433,7 @@ def calculate_macosx_platform_tag(archive_root, platform_tag): assert len(base_version) == 2 start_version = base_version - versions_dict = {} + versions_dict: dict[str, tuple[int, int]] = {} for dirpath, _dirnames, filenames in os.walk(archive_root): for filename in filenames: if filename.endswith(".dylib") or filename.endswith(".so"): diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/metadata.py b/contrib/python/wheel/wheel/metadata.py index 341f614ceb3..b8098fa8599 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/metadata.py +++ b/contrib/python/wheel/wheel/metadata.py @@ -11,17 +11,17 @@ import re import textwrap from email.message import Message from email.parser import Parser -from typing import Iterator +from typing import Generator, Iterable, Iterator, Literal -from ..packaging.requirements import Requirement +from .vendored.packaging.requirements import Requirement -def _nonblank(str): +def _nonblank(str: str) -> bool | Literal[""]: return str and not str.startswith("#") @functools.singledispatch -def yield_lines(iterable): +def yield_lines(iterable: Iterable[str]) -> Iterator[str]: r""" Yield valid lines of a string or iterable. >>> list(yield_lines('')) @@ -39,11 +39,13 @@ def yield_lines(iterable): @yield_lines.register(str) -def _(text): +def _(text: str) -> Iterator[str]: return filter(_nonblank, map(str.strip, text.splitlines())) -def split_sections(s): +def split_sections( + s: str | Iterator[str], +) -> Generator[tuple[str | None, list[str]], None, None]: """Split a string or iterable thereof into (section, content) pairs Each ``section`` is a stripped version of the section header ("[section]") and each ``content`` is a list of stripped lines excluding blank lines and @@ -51,7 +53,7 @@ def split_sections(s): header, they're returned in a first ``section`` of ``None``. """ section = None - content = [] + content: list[str] = [] for line in yield_lines(s): if line.startswith("["): if line.endswith("]"): @@ -68,7 +70,7 @@ def split_sections(s): yield section, content -def safe_extra(extra): +def safe_extra(extra: str) -> str: """Convert an arbitrary string to a standard 'extra' name Any runs of non-alphanumeric characters are replaced with a single '_', and the result is always lowercased. @@ -76,7 +78,7 @@ def safe_extra(extra): return re.sub("[^A-Za-z0-9.-]+", "_", extra).lower() -def safe_name(name): +def safe_name(name: str) -> str: """Convert an arbitrary string to a standard distribution name Any runs of non-alphanumeric/. characters are replaced with a single '-'. """ @@ -85,10 +87,10 @@ def safe_name(name): def requires_to_requires_dist(requirement: Requirement) -> str: """Return the version specifier for a requirement in PEP 345/566 fashion.""" - if getattr(requirement, "url", None): + if requirement.url: return " @ " + requirement.url - requires_dist = [] + requires_dist: list[str] = [] for spec in requirement.specifier: requires_dist.append(spec.operator + spec.version) @@ -111,7 +113,7 @@ def convert_requirements(requirements: list[str]) -> Iterator[str]: def generate_requirements( - extras_require: dict[str, list[str]], + extras_require: dict[str | None, list[str]], ) -> Iterator[tuple[str, str]]: """ Convert requirements from a setup()-style dictionary to @@ -131,13 +133,14 @@ def generate_requirements( yield "Provides-Extra", extra if condition: condition = "(" + condition + ") and " - condition += "extra == '%s'" % extra + condition += f"extra == '{extra}'" if condition: condition = " ; " + condition for new_req in convert_requirements(depends): - yield "Requires-Dist", new_req + condition + canonical_req = str(Requirement(new_req + condition)) + yield "Requires-Dist", canonical_req def pkginfo_to_metadata(egg_info_path: str, pkginfo_path: str) -> Message: diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/util.py b/contrib/python/wheel/wheel/util.py index d98d98cb52b..c928aa403b7 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/util.py +++ b/contrib/python/wheel/wheel/util.py @@ -5,15 +5,6 @@ import logging log = logging.getLogger("wheel") -# ensure Python logging is configured -try: - __import__("setuptools.logging") -except ImportError: - # setuptools < ?? - from . import _setuptools_logging - - _setuptools_logging.configure() - def urlsafe_b64encode(data: bytes) -> bytes: """urlsafe_b64encode without padding""" diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/__init__.py b/contrib/python/wheel/wheel/vendored/__init__.py index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/__init__.py +++ b/contrib/python/wheel/wheel/vendored/__init__.py diff --git a/contrib/python/wheel/wheel/vendored/packaging/LICENSE b/contrib/python/wheel/wheel/vendored/packaging/LICENSE new file mode 100644 index 00000000000..6f62d44e4ef --- /dev/null +++ b/contrib/python/wheel/wheel/vendored/packaging/LICENSE @@ -0,0 +1,3 @@ +This software is made available under the terms of *either* of the licenses +found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made +under the terms of *both* these licenses. diff --git a/contrib/python/wheel/wheel/vendored/packaging/LICENSE.APACHE b/contrib/python/wheel/wheel/vendored/packaging/LICENSE.APACHE new file mode 100644 index 00000000000..f433b1a53f5 --- /dev/null +++ b/contrib/python/wheel/wheel/vendored/packaging/LICENSE.APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/contrib/python/wheel/wheel/vendored/packaging/LICENSE.BSD b/contrib/python/wheel/wheel/vendored/packaging/LICENSE.BSD new file mode 100644 index 00000000000..42ce7b75c92 --- /dev/null +++ b/contrib/python/wheel/wheel/vendored/packaging/LICENSE.BSD @@ -0,0 +1,23 @@ +Copyright (c) Donald Stufft and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/__init__.py b/contrib/python/wheel/wheel/vendored/packaging/__init__.py index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/backports/__init__.py +++ b/contrib/python/wheel/wheel/vendored/packaging/__init__.py diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_elffile.py b/contrib/python/wheel/wheel/vendored/packaging/_elffile.py index 6fb19b30bb5..6fb19b30bb5 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_elffile.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_elffile.py diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_manylinux.py b/contrib/python/wheel/wheel/vendored/packaging/_manylinux.py index ad62505f3ff..1f5f4ab3e51 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_manylinux.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_manylinux.py @@ -167,7 +167,7 @@ def _parse_glibc_version(version_str: str) -> Tuple[int, int]: return int(m.group("major")), int(m.group("minor")) -@functools.lru_cache() +@functools.lru_cache def _get_glibc_version() -> Tuple[int, int]: version_str = _glibc_version_string() if version_str is None: diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_musllinux.py b/contrib/python/wheel/wheel/vendored/packaging/_musllinux.py index 86419df9d70..eb4251b5c1e 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_musllinux.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_musllinux.py @@ -28,7 +28,7 @@ def _parse_musl_version(output: str) -> Optional[_MuslVersion]: return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) -@functools.lru_cache() +@functools.lru_cache def _get_musl_version(executable: str) -> Optional[_MuslVersion]: """Detect currently-running musl runtime version. diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_parser.py b/contrib/python/wheel/wheel/vendored/packaging/_parser.py index 684df75457c..513686a2190 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/_parser.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_parser.py @@ -1,6 +1,6 @@ """Handwritten parser of dependency specifiers. -The docstring for each __parse_* function contains ENBF-inspired grammar representing +The docstring for each __parse_* function contains EBNF-inspired grammar representing the implementation. """ diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_structures.py b/contrib/python/wheel/wheel/vendored/packaging/_structures.py index 90a6465f968..90a6465f968 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_structures.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_structures.py diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_tokenizer.py b/contrib/python/wheel/wheel/vendored/packaging/_tokenizer.py index dd0d648d49a..dd0d648d49a 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/_tokenizer.py +++ b/contrib/python/wheel/wheel/vendored/packaging/_tokenizer.py diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/markers.py b/contrib/python/wheel/wheel/vendored/packaging/markers.py index 8b98fca7233..c96d22a5a44 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/markers.py +++ b/contrib/python/wheel/wheel/vendored/packaging/markers.py @@ -14,6 +14,8 @@ from ._parser import ( Op, Value, Variable, +) +from ._parser import ( parse_marker as _parse_marker, ) from ._tokenizer import ParserSyntaxError @@ -69,7 +71,6 @@ def _normalize_extra_values(results: Any) -> Any: def _format_marker( marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True ) -> str: - assert isinstance(marker, (list, tuple, str)) # Sometimes we have a structure like [[...]] which is a single item list diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/requirements.py b/contrib/python/wheel/wheel/vendored/packaging/requirements.py index bdc43a7e98d..bdc43a7e98d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/requirements.py +++ b/contrib/python/wheel/wheel/vendored/packaging/requirements.py diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/specifiers.py b/contrib/python/wheel/wheel/vendored/packaging/specifiers.py index 2d015bab595..6d4066ae277 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/packaging/specifiers.py +++ b/contrib/python/wheel/wheel/vendored/packaging/specifiers.py @@ -364,7 +364,6 @@ class Specifier(BaseSpecifier): return operator_callable def _compare_compatible(self, prospective: Version, spec: str) -> bool: - # Compatible releases have an equivalent combination of >= and ==. That # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to # implement this in terms of the other specifiers instead of @@ -385,7 +384,6 @@ class Specifier(BaseSpecifier): ) def _compare_equal(self, prospective: Version, spec: str) -> bool: - # We need special logic to handle prefix matching if spec.endswith(".*"): # In the case of prefix matching we want to ignore local segment. @@ -429,21 +427,18 @@ class Specifier(BaseSpecifier): return not self._compare_equal(prospective, spec) def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: - # NB: Local version identifiers are NOT permitted in the version # specifier, so local version labels can be universally removed from # the prospective version. return Version(prospective.public) <= Version(spec) def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: - # NB: Local version identifiers are NOT permitted in the version # specifier, so local version labels can be universally removed from # the prospective version. return Version(prospective.public) >= Version(spec) def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: - # Convert our spec to a Version instance, since we'll want to work with # it as a version. spec = Version(spec_str) @@ -468,7 +463,6 @@ class Specifier(BaseSpecifier): return True def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: - # Convert our spec to a Version instance, since we'll want to work with # it as a version. spec = Version(spec_str) diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/tags.py b/contrib/python/wheel/wheel/vendored/packaging/tags.py index 89f1926137d..89f1926137d 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/tags.py +++ b/contrib/python/wheel/wheel/vendored/packaging/tags.py diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/utils.py b/contrib/python/wheel/wheel/vendored/packaging/utils.py index c2c2f75aa80..c2c2f75aa80 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/utils.py +++ b/contrib/python/wheel/wheel/vendored/packaging/utils.py diff --git a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/version.py b/contrib/python/wheel/wheel/vendored/packaging/version.py index 5faab9bd0dc..cda8e99935c 100644 --- a/contrib/python/setuptools/py3/pkg_resources/_vendor/packaging/version.py +++ b/contrib/python/wheel/wheel/vendored/packaging/version.py @@ -452,7 +452,6 @@ class Version(_BaseVersion): def _parse_letter_version( letter: Optional[str], number: Union[str, bytes, SupportsInt, None] ) -> Optional[Tuple[str, int]]: - if letter: # We consider there to be an implicit 0 in a pre-release if there is # not a numeral associated with it. @@ -508,7 +507,6 @@ def _cmpkey( dev: Optional[Tuple[str, int]], local: Optional[LocalType], ) -> CmpKey: - # When we compare a release version, we want to compare it with all of the # trailing zeros removed. So we'll use a reverse the list, drop all the now # leading zeros until we come to something non zero, then take the rest diff --git a/contrib/python/wheel/wheel/vendored/vendor.txt b/contrib/python/wheel/wheel/vendored/vendor.txt new file mode 100644 index 00000000000..14666103a82 --- /dev/null +++ b/contrib/python/wheel/wheel/vendored/vendor.txt @@ -0,0 +1 @@ +packaging==24.0 diff --git a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/wheelfile.py b/contrib/python/wheel/wheel/wheelfile.py index 83a31772bda..0a0f4596c56 100644 --- a/contrib/python/setuptools/py3/setuptools/_vendor/wheel/wheelfile.py +++ b/contrib/python/wheel/wheel/wheelfile.py @@ -7,9 +7,21 @@ import re import stat import time from io import StringIO, TextIOWrapper +from typing import IO, TYPE_CHECKING, Literal from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo -from .util import log, urlsafe_b64decode, urlsafe_b64encode +from wheel.cli import WheelError +from wheel.util import log, urlsafe_b64decode, urlsafe_b64encode + +if TYPE_CHECKING: + from typing import Protocol, Sized, Union + + from typing_extensions import Buffer + + StrPath = Union[str, os.PathLike[str]] + + class SizedBuffer(Sized, Buffer, Protocol): ... + # Non-greedy matching of an optional build number may be too clever (more # invalid wheel filenames will match). Separate regex for .dist-info? @@ -21,7 +33,7 @@ WHEEL_INFO_RE = re.compile( MINIMUM_TIMESTAMP = 315532800 # 1980-01-01 00:00:00 UTC -def get_zipinfo_datetime(timestamp=None): +def get_zipinfo_datetime(timestamp: float | None = None): # Some applications need reproducible .whl files, but they can't do this without # forcing the timestamp of the individual ZipInfo objects. See issue #143. timestamp = int(os.environ.get("SOURCE_DATE_EPOCH", timestamp or time.time())) @@ -36,7 +48,12 @@ class WheelFile(ZipFile): _default_algorithm = hashlib.sha256 - def __init__(self, file, mode="r", compression=ZIP_DEFLATED): + def __init__( + self, + file: StrPath, + mode: Literal["r", "w", "x", "a"] = "r", + compression: int = ZIP_DEFLATED, + ): basename = os.path.basename(file) self.parsed_filename = WHEEL_INFO_RE.match(basename) if not basename.endswith(".whl") or self.parsed_filename is None: @@ -48,7 +65,7 @@ class WheelFile(ZipFile): self.parsed_filename.group("namever") ) self.record_path = self.dist_info_path + "/RECORD" - self._file_hashes = {} + self._file_hashes: dict[str, tuple[None, None] | tuple[int, bytes]] = {} self._file_sizes = {} if mode == "r": # Ignore RECORD and any embedded wheel signatures @@ -89,8 +106,13 @@ class WheelFile(ZipFile): urlsafe_b64decode(hash_sum.encode("ascii")), ) - def open(self, name_or_info, mode="r", pwd=None): - def _update_crc(newdata): + def open( + self, + name_or_info: str | ZipInfo, + mode: Literal["r", "w"] = "r", + pwd: bytes | None = None, + ) -> IO[bytes]: + def _update_crc(newdata: bytes) -> None: eof = ef._eof update_crc_orig(newdata) running_hash.update(newdata) @@ -118,9 +140,9 @@ class WheelFile(ZipFile): return ef - def write_files(self, base_dir): + def write_files(self, base_dir: str): log.info(f"creating '{self.filename}' and adding '{base_dir}' to it") - deferred = [] + deferred: list[tuple[str, str]] = [] for root, dirnames, filenames in os.walk(base_dir): # Sort the directory names so that `os.walk` will walk them in a # defined order on the next iteration. @@ -140,7 +162,12 @@ class WheelFile(ZipFile): for path, arcname in deferred: self.write(path, arcname) - def write(self, filename, arcname=None, compress_type=None): + def write( + self, + filename: str, + arcname: str | None = None, + compress_type: int | None = None, + ) -> None: with open(filename, "rb") as f: st = os.fstat(f.fileno()) data = f.read() @@ -152,7 +179,12 @@ class WheelFile(ZipFile): zinfo.compress_type = compress_type or self.compression self.writestr(zinfo, data, compress_type) - def writestr(self, zinfo_or_arcname, data, compress_type=None): + def writestr( + self, + zinfo_or_arcname: str | ZipInfo, + data: SizedBuffer | str, + compress_type: int | None = None, + ): if isinstance(zinfo_or_arcname, str): zinfo_or_arcname = ZipInfo( zinfo_or_arcname, date_time=get_zipinfo_datetime() @@ -193,7 +225,3 @@ class WheelFile(ZipFile): self.writestr(self.record_path, data.getvalue()) ZipFile.close(self) - - -class WheelError(Exception): - pass diff --git a/contrib/python/wheel/ya.make b/contrib/python/wheel/ya.make new file mode 100644 index 00000000000..15e1b0d4b24 --- /dev/null +++ b/contrib/python/wheel/ya.make @@ -0,0 +1,53 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(0.45.1) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + wheel/__init__.py + wheel/__main__.py + wheel/_bdist_wheel.py + wheel/_setuptools_logging.py + wheel/bdist_wheel.py + wheel/cli/__init__.py + wheel/cli/convert.py + wheel/cli/pack.py + wheel/cli/tags.py + wheel/cli/unpack.py + wheel/macosx_libfile.py + wheel/metadata.py + wheel/util.py + wheel/vendored/__init__.py + wheel/vendored/packaging/__init__.py + wheel/vendored/packaging/_elffile.py + wheel/vendored/packaging/_manylinux.py + wheel/vendored/packaging/_musllinux.py + wheel/vendored/packaging/_parser.py + wheel/vendored/packaging/_structures.py + wheel/vendored/packaging/_tokenizer.py + wheel/vendored/packaging/markers.py + wheel/vendored/packaging/requirements.py + wheel/vendored/packaging/specifiers.py + wheel/vendored/packaging/tags.py + wheel/vendored/packaging/utils.py + wheel/vendored/packaging/version.py + wheel/wheelfile.py +) + +RESOURCE_FILES( + PREFIX contrib/python/wheel/ + .dist-info/METADATA + .dist-info/entry_points.txt + wheel/vendored/packaging/LICENSE + wheel/vendored/packaging/LICENSE.APACHE + wheel/vendored/packaging/LICENSE.BSD + wheel/vendored/vendor.txt +) + +END() diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 4ed479feea4..c150e0c2ec6 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2025011000, Last Updated Fri Jan 10 07:07:01 2025 UTC +# Version 2025011300, Last Updated Mon Jan 13 07:07:01 2025 UTC AAA AARP ABB diff --git a/library/cpp/yt/memory/erased_storage.h b/library/cpp/yt/memory/erased_storage.h index 46ea5cc854d..54870f7e071 100644 --- a/library/cpp/yt/memory/erased_storage.h +++ b/library/cpp/yt/memory/erased_storage.h @@ -2,6 +2,7 @@ #include <concepts> #include <memory> +#include <type_traits> namespace NYT { diff --git a/library/cpp/yt/memory/ref_counted-inl.h b/library/cpp/yt/memory/ref_counted-inl.h index 923b53c822c..04dd629a26b 100644 --- a/library/cpp/yt/memory/ref_counted-inl.h +++ b/library/cpp/yt/memory/ref_counted-inl.h @@ -8,6 +8,8 @@ #include <util/system/sanitizers.h> +#include <stdlib.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// diff --git a/util/system/err.cpp b/util/system/err.cpp index e2e5d8c9fdb..b7fe13aa1c4 100644 --- a/util/system/err.cpp +++ b/util/system/err.cpp @@ -9,6 +9,8 @@ #include <util/stream/printf.h> #include <util/stream/output.h> +#include <cstdlib> + void vwarnx(const char* fmt, va_list args) { Cerr << GetProgramName() << ": "; @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "darwin": { - "md5": "2aeff4a8b56c294648e2a7fe5bcaf2f5", + "md5": "950e88ffc7c851dfbafafb89ba1b1b77", "urls": [ - f"{REGISTRY_ENDPOINT}/7750410363" + f"{REGISTRY_ENDPOINT}/7808696359" ] }, "darwin-arm64": { - "md5": "f4388ea2cd6cf37a66ad839c13356535", + "md5": "9d15442d9bb0a0aa6669cea947fe336b", "urls": [ - f"{REGISTRY_ENDPOINT}/7750409498" + f"{REGISTRY_ENDPOINT}/7808696176" ] }, "linux-aarch64": { - "md5": "8199dfd101c2eeddc2926e0bd8931a66", + "md5": "9250ef732fc3a971ff28a126126b4a76", "urls": [ - f"{REGISTRY_ENDPOINT}/7750408525" + f"{REGISTRY_ENDPOINT}/7808695917" ] }, "win32-clang-cl": { - "md5": "12257bbee911ec403eb302a15c7c22b0", + "md5": "00674a6eda2cf970ab4c73937c808050", "urls": [ - f"{REGISTRY_ENDPOINT}/7750411200" + f"{REGISTRY_ENDPOINT}/7808696543" ] }, "linux": { - "md5": "b921725a0c13639344aec267bef973c0", + "md5": "9a6d64ab467535c82de4351766427700", "urls": [ - f"{REGISTRY_ENDPOINT}/7750412171" + f"{REGISTRY_ENDPOINT}/7808696801" ] } } diff --git a/yql/essentials/providers/common/proto/python/ya.make b/yql/essentials/providers/common/proto/python/ya.make deleted file mode 100644 index 0c5cb1a8521..00000000000 --- a/yql/essentials/providers/common/proto/python/ya.make +++ /dev/null @@ -1,2 +0,0 @@ -PY_PROTOS_FOR(yql/essentials/providers/common/proto) - diff --git a/yql/essentials/providers/common/proto/ya.make b/yql/essentials/providers/common/proto/ya.make index eefd4c6b72b..d7ed8ee96c5 100644 --- a/yql/essentials/providers/common/proto/ya.make +++ b/yql/essentials/providers/common/proto/ya.make @@ -9,8 +9,6 @@ PEERDIR( yql/essentials/protos ) -IF (NOT PY_PROTOS_FOR) - EXCLUDE_TAGS(GO_PROTO) -ENDIF() +EXCLUDE_TAGS(GO_PROTO) END() diff --git a/yql/essentials/providers/common/ya.make b/yql/essentials/providers/common/ya.make index 411038d5b0c..3479cf2ecf4 100644 --- a/yql/essentials/providers/common/ya.make +++ b/yql/essentials/providers/common/ya.make @@ -9,7 +9,7 @@ RECURSE( metrics mkql mkql_simple_file - proto/python + proto provider schema structured_token diff --git a/yt/yt/client/table_client/logical_type.cpp b/yt/yt/client/table_client/logical_type.cpp index 5c31d97df6e..821b726e59e 100644 --- a/yt/yt/client/table_client/logical_type.cpp +++ b/yt/yt/client/table_client/logical_type.cpp @@ -325,9 +325,9 @@ i64 TDecimalLogicalType::GetMemoryUsage() const return sizeof(*this); } -i64 TDecimalLogicalType::GetMemoryUsage(i64 limit) const +i64 TDecimalLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); return sizeof(*this); } @@ -389,17 +389,17 @@ i64 TOptionalLogicalType::GetMemoryUsage() const } } -i64 TOptionalLogicalType::GetMemoryUsage(i64 limit) const +i64 TOptionalLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); if (Element_->GetMetatype() == ELogicalMetatype::Simple) { // NB: See TOptionalLogicalType::GetMemoryUsage(). return 0; - } else if (auto sizeOfThis = static_cast<i64>(sizeof(*this)); sizeOfThis >= limit) { + } else if (auto sizeOfThis = static_cast<i64>(sizeof(*this)); sizeOfThis >= threshold) { return sizeof(*this); } else { - return sizeOfThis + Element_->GetMemoryUsage(limit - sizeOfThis); + return sizeOfThis + Element_->GetMemoryUsage(threshold - sizeOfThis); } } @@ -433,9 +433,9 @@ i64 TSimpleLogicalType::GetMemoryUsage() const return 0; } -i64 TSimpleLogicalType::GetMemoryUsage(i64 limit) const +i64 TSimpleLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); return 0; } @@ -472,14 +472,14 @@ i64 TListLogicalType::GetMemoryUsage() const return sizeof(*this) + Element_->GetMemoryUsage(); } -i64 TListLogicalType::GetMemoryUsage(i64 limit) const +i64 TListLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); - if (auto sizeOfThis = static_cast<i64>(sizeof(*this)); sizeOfThis >= limit) { + if (auto sizeOfThis = static_cast<i64>(sizeof(*this)); sizeOfThis >= threshold) { return sizeOfThis; } else { - return sizeOfThis + Element_->GetMemoryUsage(limit - sizeOfThis); + return sizeOfThis + Element_->GetMemoryUsage(threshold - sizeOfThis); } } @@ -727,16 +727,16 @@ i64 TStructLogicalTypeBase::GetMemoryUsage() const return usage; } -i64 TStructLogicalTypeBase::GetMemoryUsage(i64 limit) const +i64 TStructLogicalTypeBase::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>(sizeof(*this) + sizeof(TStructField) * Fields_.size()); for (const auto& field : Fields_) { - if (usage >= limit) { + if (usage >= threshold) { return usage; } - usage += field.Type->GetMemoryUsage(limit - usage); + usage += field.Type->GetMemoryUsage(threshold - usage); } return usage; } @@ -799,16 +799,16 @@ i64 TTupleLogicalTypeBase::GetMemoryUsage() const return usage; } -i64 TTupleLogicalTypeBase::GetMemoryUsage(i64 limit) const +i64 TTupleLogicalTypeBase::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>(sizeof(*this) + sizeof(TLogicalTypePtr) * Elements_.size()); for (const auto& element : Elements_) { - if (usage >= limit) { + if (usage >= threshold) { return usage; } - usage += element->GetMemoryUsage(limit - usage); + usage += element->GetMemoryUsage(threshold - usage); } return usage; } @@ -867,19 +867,19 @@ i64 TDictLogicalType::GetMemoryUsage() const return sizeof(*this) + Key_->GetMemoryUsage() + Value_->GetMemoryUsage(); } -i64 TDictLogicalType::GetMemoryUsage(i64 limit) const +i64 TDictLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>(sizeof(*this)); - if (usage >= limit) { + if (usage >= threshold) { return usage; } - usage += Key_->GetMemoryUsage(limit - usage); - if (usage >= limit) { + usage += Key_->GetMemoryUsage(threshold - usage); + if (usage >= threshold) { return usage; } - usage += Value_->GetMemoryUsage(limit - usage); + usage += Value_->GetMemoryUsage(threshold - usage); return usage; } @@ -965,15 +965,15 @@ i64 TTaggedLogicalType::GetMemoryUsage() const return sizeof(*this) + GetElement()->GetMemoryUsage(); } -i64 TTaggedLogicalType::GetMemoryUsage(i64 limit) const +i64 TTaggedLogicalType::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>(sizeof(*this)); - if (usage >= limit) { + if (usage >= threshold) { return usage; } - usage += GetElement()->GetMemoryUsage(limit - usage); + usage += GetElement()->GetMemoryUsage(threshold - usage); return usage; } diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h index 2c9a6afbfa1..93858896329 100644 --- a/yt/yt/client/table_client/logical_type.h +++ b/yt/yt/client/table_client/logical_type.h @@ -72,7 +72,7 @@ public: Y_FORCE_INLINE const TTaggedLogicalType& UncheckedAsTaggedTypeRef() const; virtual i64 GetMemoryUsage() const = 0; - virtual i64 GetMemoryUsage(i64 limit) const = 0; + virtual i64 GetMemoryUsage(i64 threshold) const = 0; virtual int GetTypeComplexity() const = 0; // This function doesn't validate children of current node. @@ -177,7 +177,7 @@ public: TDecimalLogicalType(int precision, int scale); i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -206,7 +206,7 @@ public: Y_FORCE_INLINE bool IsElementNullable() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -227,7 +227,7 @@ public: Y_FORCE_INLINE ESimpleLogicalValueType GetElement() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -247,7 +247,7 @@ public: Y_FORCE_INLINE const TLogicalTypePtr& GetElement() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -308,7 +308,7 @@ public: Y_FORCE_INLINE const std::vector<TStructField>& GetFields() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -328,7 +328,7 @@ public: Y_FORCE_INLINE const std::vector<TLogicalTypePtr>& GetElements() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -385,7 +385,7 @@ public: Y_FORCE_INLINE const TLogicalTypePtr& GetValue() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; @@ -407,7 +407,7 @@ public: Y_FORCE_INLINE const TLogicalTypePtr& GetElement() const; i64 GetMemoryUsage() const override; - i64 GetMemoryUsage(i64 limit) const override; + i64 GetMemoryUsage(i64 threshold) const override; int GetTypeComplexity() const override; void ValidateNode(const TWalkContext& context) const override; bool IsNullable() const override; diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index 46cfbee1b23..4b826077e03 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -282,9 +282,9 @@ i64 TColumnSchema::GetMemoryUsage() const (Group_ ? Group_->size() : 0); } -i64 TColumnSchema::GetMemoryUsage(i64 limit) const +i64 TColumnSchema::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>( sizeof(TColumnSchema) + @@ -295,11 +295,11 @@ i64 TColumnSchema::GetMemoryUsage(i64 limit) const (Aggregate_ ? Aggregate_->size() : 0) + (Group_ ? Group_->size() : 0)); - if (usage >= limit) { + if (usage >= threshold) { return usage; } - return usage + LogicalType_->GetMemoryUsage(limit - usage); + return usage + LogicalType_->GetMemoryUsage(threshold - usage); } bool TColumnSchema::IsOfV1Type() const @@ -1459,17 +1459,17 @@ i64 TTableSchema::GetMemoryUsage() const return usage; } -i64 TTableSchema::GetMemoryUsage(i64 limit) const +i64 TTableSchema::GetMemoryUsage(i64 threshold) const { - YT_ASSERT(limit > 0); + YT_ASSERT(threshold > 0); auto usage = static_cast<i64>(sizeof(TTableSchema)); for (const auto& column : Columns()) { - if (usage >= limit) { + if (usage >= threshold) { return usage; } - usage += column.GetMemoryUsage(limit - usage); + usage += column.GetMemoryUsage(threshold - usage); } return usage; } @@ -1613,19 +1613,19 @@ void PrintTo(const TTableSchema& tableSchema, std::ostream* os) TTableSchemaTruncatedFormatter::TTableSchemaTruncatedFormatter( const TTableSchemaPtr& schema, - i64 memoryLimit) + i64 memoryThreshold) : Schema_(schema.Get()) - , Limit_(memoryLimit) + , Threshold_(memoryThreshold) { - YT_ASSERT(Limit_ >= 0); + YT_ASSERT(Threshold_ >= 0); } void TTableSchemaTruncatedFormatter::operator()(TStringBuilderBase* builder) const { if (!Schema_) { builder->AppendString(ToString(nullptr)); - } else if (Limit_ > 0 && Schema_->GetMemoryUsage(Limit_) <= Limit_) { - builder->AppendFormat("%v", *Schema_); + } else if (Threshold_ > 0 && Schema_->GetMemoryUsage(Threshold_) < Threshold_) { + FormatValue(builder, *Schema_, "%v"); } else { builder->AppendString("<schema memory usage is over the logging threshold>"); } diff --git a/yt/yt/client/table_client/schema.h b/yt/yt/client/table_client/schema.h index a74ed5f5d70..5feae632bf9 100644 --- a/yt/yt/client/table_client/schema.h +++ b/yt/yt/client/table_client/schema.h @@ -166,7 +166,7 @@ public: EValueType GetWireType() const; i64 GetMemoryUsage() const; - i64 GetMemoryUsage(i64 limit) const; + i64 GetMemoryUsage(i64 threshold) const; // Check if column has plain old v1 type. bool IsOfV1Type() const; @@ -398,8 +398,7 @@ public: void Load(TStreamLoadContext& context); i64 GetMemoryUsage() const; - - i64 GetMemoryUsage(i64 limit) const; + i64 GetMemoryUsage(i64 threshold) const; private: struct TColumnInfo @@ -469,18 +468,18 @@ class TTableSchemaTruncatedFormatter { public: // NB: #schema is allowed to be |nullptr|. - TTableSchemaTruncatedFormatter(const TTableSchemaPtr& schema, i64 memoryLimit); + TTableSchemaTruncatedFormatter(const TTableSchemaPtr& schema, i64 memoryThreshold); void operator()(TStringBuilderBase* builder) const; private: const TTableSchema* const Schema_ = nullptr; - const i64 Limit_ = 0; + const i64 Threshold_ = 0; }; TFormatterWrapper<TTableSchemaTruncatedFormatter> MakeTableSchemaTruncatedFormatter( const TTableSchemaPtr& schema, - i64 memoryLimit); + i64 memoryThreshold); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/unittests/ypath_ut.cpp b/yt/yt/client/unittests/ypath_ut.cpp index bc74d5f7cc1..93a3c199d1a 100644 --- a/yt/yt/client/unittests/ypath_ut.cpp +++ b/yt/yt/client/unittests/ypath_ut.cpp @@ -19,6 +19,7 @@ #include <yt/yt/core/ytree/tree_visitor.h> #include <yt/yt/core/ytree/ypath_client.h> #include <yt/yt/core/ytree/ypath_service.h> +#include <yt/yt/core/ytree/helpers.h> #include <util/string/vector.h> diff --git a/yt/yt/core/concurrency/fair_share_action_queue-inl.h b/yt/yt/core/concurrency/fair_share_action_queue-inl.h index 9619f33276a..e4a4feeec61 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue-inl.h +++ b/yt/yt/core/concurrency/fair_share_action_queue-inl.h @@ -17,8 +17,14 @@ public: const TString& threadName, const std::vector<TString>& queueNames, const THashMap<TString, std::vector<TString>>& bucketToQueues, + NThreading::TThreadOptions threadOptions, NProfiling::IRegistryPtr registry) - : Queue_(CreateFairShareActionQueue(threadName, queueNames, bucketToQueues, std::move(registry))) + : Queue_(CreateFairShareActionQueue( + threadName, + queueNames, + bucketToQueues, + threadOptions, + std::move(registry))) { } const IInvokerPtr& GetInvoker(EQueue queue) override @@ -41,6 +47,7 @@ template <typename EQueue, typename EBucket> IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue( const TString& threadName, const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues, + NThreading::TThreadOptions threadOptions, NProfiling::IRegistryPtr registry) { std::vector<TString> queueNames; @@ -55,7 +62,12 @@ IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueu stringBucket.push_back(ToString(queue)); } } - return New<TEnumIndexedFairShareActionQueue<EQueue>>(threadName, queueNames, stringBuckets, std::move(registry)); + return New<TEnumIndexedFairShareActionQueue<EQueue>>( + threadName, + queueNames, + stringBuckets, + threadOptions, + std::move(registry)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_action_queue.cpp b/yt/yt/core/concurrency/fair_share_action_queue.cpp index db018103bc0..ef005510284 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue.cpp +++ b/yt/yt/core/concurrency/fair_share_action_queue.cpp @@ -19,6 +19,7 @@ namespace NYT::NConcurrency { using namespace NProfiling; using namespace NYPath; using namespace NYTree; +using namespace NThreading; //////////////////////////////////////////////////////////////////////////////// @@ -30,6 +31,7 @@ public: const TString& threadName, const std::vector<TString>& queueNames, const THashMap<TString, std::vector<TString>>& bucketToQueues, + TThreadOptions threadOptions, NProfiling::IRegistryPtr registry) : ShutdownCookie_(RegisterShutdownCallback( Format("FairShareActionQueue(%v)", threadName), @@ -94,8 +96,17 @@ public: YT_VERIFY(QueueIndexToBucketQueueIndex_[queueIndex] != -1); } - Queue_ = New<TFairShareInvokerQueue>(CallbackEventCount_, std::move(bucketDescriptions), std::move(registry)); - Thread_ = New<TFairShareQueueSchedulerThread>(Queue_, CallbackEventCount_, threadName, threadName); + Queue_ = New<TFairShareInvokerQueue>( + CallbackEventCount_, + std::move(bucketDescriptions), + std::move(registry)); + + Thread_ = New<TFairShareQueueSchedulerThread>( + Queue_, + CallbackEventCount_, + threadName, + threadName, + threadOptions); } ~TFairShareActionQueue() @@ -141,7 +152,7 @@ public: } private: - const TIntrusivePtr<NThreading::TEventCount> CallbackEventCount_ = New<NThreading::TEventCount>(); + const TIntrusivePtr<TEventCount> CallbackEventCount_ = New<TEventCount>(); const TShutdownCookie ShutdownCookie_; const IInvokerPtr ShutdownInvoker_ = GetShutdownInvoker(); @@ -171,9 +182,15 @@ IFairShareActionQueuePtr CreateFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, const THashMap<TString, std::vector<TString>>& bucketToQueues, + TThreadOptions threadOptions, NProfiling::IRegistryPtr registry) { - return New<TFairShareActionQueue>(threadName, queueNames, bucketToQueues, std::move(registry)); + return New<TFairShareActionQueue>( + threadName, + queueNames, + bucketToQueues, + threadOptions, + std::move(registry)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_action_queue.h b/yt/yt/core/concurrency/fair_share_action_queue.h index a1f8d4e2b13..195bde54614 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue.h +++ b/yt/yt/core/concurrency/fair_share_action_queue.h @@ -8,6 +8,8 @@ #include <yt/yt/core/profiling/public.h> +#include <yt/yt/core/threading/thread.h> + namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// @@ -28,6 +30,7 @@ IFairShareActionQueuePtr CreateFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, const THashMap<TString, std::vector<TString>>& bucketToQueues = {}, + NThreading::TThreadOptions threadOptions = {}, NProfiling::IRegistryPtr registry = {}); //////////////////////////////////////////////////////////////////////////////// @@ -47,6 +50,7 @@ template <typename EQueue, typename EBucket = EQueue> IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue( const TString& threadName, const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues = {}, + NThreading::TThreadOptions threadOptions = {}, NProfiling::IRegistryPtr registry = {}); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.cpp b/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.cpp index fed943be936..c21ac16eb56 100644 --- a/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.cpp +++ b/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.cpp @@ -10,11 +10,13 @@ TFairShareQueueSchedulerThread::TFairShareQueueSchedulerThread( TFairShareInvokerQueuePtr queue, TIntrusivePtr<NThreading::TEventCount> callbackEventCount, const TString& threadGroupName, - const TString& threadName) + const TString& threadName, + NThreading::TThreadOptions options) : TSchedulerThread( std::move(callbackEventCount), threadGroupName, - threadName) + threadName, + options) , Queue_(std::move(queue)) { } diff --git a/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.h b/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.h index 5716b472ad3..7dc802a3a6e 100644 --- a/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.h +++ b/yt/yt/core/concurrency/fair_share_queue_scheduler_thread.h @@ -16,7 +16,8 @@ public: TFairShareInvokerQueuePtr queue, TIntrusivePtr<NThreading::TEventCount> callbackEventCount, const TString& threadGroupName, - const TString& threadName); + const TString& threadName, + NThreading::TThreadOptions options = {}); protected: const TFairShareInvokerQueuePtr Queue_; diff --git a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp index c0e4612e03d..92fa3e12122 100644 --- a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp +++ b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp @@ -1047,7 +1047,6 @@ TFiberSchedulerThread::TFiberSchedulerThread( , ThreadGroupName_(std::move(threadGroupName)) { } - void TFiberSchedulerThread::ThreadMain() { // Hold this strongly. diff --git a/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp index d13026ce8d7..955c3b3adc5 100644 --- a/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp +++ b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp @@ -144,7 +144,11 @@ TEST_F(TTestFairShareActionQueue, TestProfiling) THashMap<EBuckets, std::vector<EQueues>> bucketToQueues{}; bucketToQueues[EBuckets::Bucket1] = {EQueues::Queue1, EQueues::Queue2}; bucketToQueues[EBuckets::Bucket2] = {EQueues::Queue3}; - auto queue = CreateEnumIndexedFairShareActionQueue<EQueues>("ActionQueue", bucketToQueues, registry); + auto queue = CreateEnumIndexedFairShareActionQueue<EQueues>( + "ActionQueue", + bucketToQueues, + /*threadOptions*/ {}, + registry); auto config = CreateExporterConfig(); auto exporter = New<TSolomonExporter>(config, registry); diff --git a/yt/yt/core/misc/collection_helpers-inl.h b/yt/yt/core/misc/collection_helpers-inl.h index 0bfa6d4ec22..1afff7460ed 100644 --- a/yt/yt/core/misc/collection_helpers-inl.h +++ b/yt/yt/core/misc/collection_helpers-inl.h @@ -140,7 +140,7 @@ void MergeFrom(TTarget* target, const TSource& source) } template <class TMap, class TKeySet> -TKeySet DropMissingKeys(TMap&& map, const TKeySet& set) +TKeySet DropAndReturnMissingKeys(TMap&& map, const TKeySet& set) { TKeySet dropped; for (auto it = map.begin(); it != map.end(); ) { @@ -154,6 +154,18 @@ TKeySet DropMissingKeys(TMap&& map, const TKeySet& set) return dropped; } +template <class TMap, class TKeySet> +void DropMissingKeys(TMap&& map, const TKeySet& set) +{ + for (auto it = map.begin(); it != map.end(); ) { + if (!set.contains(it->first)) { + map.erase(it++); + } else { + ++it; + } + } +} + template <class TMap, class TKey> auto GetIteratorOrCrash(TMap&& map, const TKey& key) { diff --git a/yt/yt/core/misc/collection_helpers.h b/yt/yt/core/misc/collection_helpers.h index f5f316ee30b..4695096e258 100644 --- a/yt/yt/core/misc/collection_helpers.h +++ b/yt/yt/core/misc/collection_helpers.h @@ -40,7 +40,10 @@ template <class TSource, class TTarget> void MergeFrom(TTarget* target, const TSource& source); template <class TMap, class TKeySet> -TKeySet DropMissingKeys(TMap&& map, const TKeySet& set); +[[nodiscard]] TKeySet DropAndReturnMissingKeys(TMap&& map, const TKeySet& set); + +template <class TMap, class TKeySet> +void DropMissingKeys(TMap&& map, const TKeySet& set); /*! * This function is supposed to replace a frequent pattern diff --git a/yt/yt/core/rpc/unittests/lib/common.h b/yt/yt/core/rpc/unittests/lib/common.h index f6b83454f7c..9d2f5fca815 100644 --- a/yt/yt/core/rpc/unittests/lib/common.h +++ b/yt/yt/core/rpc/unittests/lib/common.h @@ -59,6 +59,8 @@ #include <yt/yt/core/ytree/convert.h> #include <yt/yt/core/ytree/helpers.h> +#include <yt/yt/build/ya_version.h> + #include <library/cpp/testing/common/env.h> #include <library/cpp/testing/common/network.h> diff --git a/yt/yt/core/rpc/unittests/rpc_ut.cpp b/yt/yt/core/rpc/unittests/rpc_ut.cpp index da19d392a61..e3ce7376ee0 100644 --- a/yt/yt/core/rpc/unittests/rpc_ut.cpp +++ b/yt/yt/core/rpc/unittests/rpc_ut.cpp @@ -533,10 +533,10 @@ TYPED_TEST(TNotGrpcTest, TrackedRegularAttachments) // Attachment allocator proactively allocate slice of 4 KB. // See NYT::NBus::TPacketDecoder::TChunkedMemoryTrackingAllocator::Allocate. // default stub = 4096. - // header + body = 103 bytes. + // header + body = 79 bytes. // attachments = 22 bytes. - // sum is 4221 bytes. - EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 4221 + 32768); + // sum is 4219 bytes. + EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 4197 + 32768 + std::ssize(GetRpcUserAgent())); EXPECT_EQ(3u, attachments.size()); EXPECT_EQ("Hello_", StringFromSharedRef(attachments[0])); EXPECT_EQ("from_", StringFromSharedRef(attachments[1])); @@ -602,7 +602,7 @@ TYPED_TEST(TNotGrpcTest, Compression) // attachmentStrings[1].size() = 36 * 2 bytes from decoder. // attachmentStrings[2].size() = 90 * 2 bytes from decoder. // sum is 4584 bytes. - EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 4584 + 32768); + EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 4562 + 32768 + std::ssize(GetRpcUserAgent())); EXPECT_TRUE(rsp->message() == message); EXPECT_GE(rsp->GetResponseMessage().Size(), static_cast<size_t>(2)); const auto& serializedResponseBody = SerializeProtoToRefWithCompression(*rsp, responseCodecId); @@ -831,9 +831,9 @@ TYPED_TEST(TNotGrpcTest, MemoryTracking) { auto rpcUsage = memoryUsageTracker->GetTotalUsage(); - // 1292468 = 32768 + 1228800 = 32768 + 4096 * 300 + 300 * 103 (header + body). + // 1285268 = 32768 + 1252500 = 32768 + 4096 * 300 + 300 * 79 (header + body). // 32768 - socket buffers, 4096 - default size per request. - EXPECT_GE(rpcUsage, 1292468); + EXPECT_GE(rpcUsage, 1285268 + 300 * std::ssize(GetRpcUserAgent())); } } @@ -850,10 +850,10 @@ TYPED_TEST(TNotGrpcTest, MemoryTrackingMultipleConnections) } { - // 11059200 / 300 = 36974 = 32768 + 4096 + 103 (header + body). + // 11082900 / 300 = 36974 = 32768 + 4096 + 79 (header + body). // 4 KB - stub for request. // See NYT::NBus::TPacketDecoder::TChunkedMemoryTrackingAllocator::Allocate. - EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 11090100); + EXPECT_GE(memoryUsageTracker->GetTotalUsage(), 11082900 + 300 * std::ssize(GetRpcUserAgent())); } } @@ -907,9 +907,9 @@ TYPED_TEST(TNotGrpcTest, MemoryOvercommit) // Attachment allocator proactively allocate slice of 4 KB. // See NYT::NBus::TPacketDecoder::TChunkedMemoryTrackingAllocator::Allocate. // default stub = 4096. - // header + body = 103 bytes. + // header + body = 79 bytes. // attachments = 6_KB kbytes. - EXPECT_GE(rpcUsage, 32768 + 4096 + 6144 + 103); + EXPECT_GE(rpcUsage, 32768 + 4096 + 6144 + 79 + std::ssize(GetRpcUserAgent())); } } diff --git a/yt/yt/core/tracing/trace_context.cpp b/yt/yt/core/tracing/trace_context.cpp index 8a186701d3e..2544f0147ab 100644 --- a/yt/yt/core/tracing/trace_context.cpp +++ b/yt/yt/core/tracing/trace_context.cpp @@ -10,6 +10,7 @@ #include <yt/yt/core/misc/protobuf_helpers.h> #include <yt/yt/core/ytree/convert.h> +#include <yt/yt/core/ytree/helpers.h> #include <yt/yt_proto/yt/core/tracing/proto/tracing_ext.pb.h> diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 79dfe4b3cce..41e2da5dfa3 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -300,6 +300,7 @@ SRCS( ytree/ypath_service.cpp ytree/yson_struct.cpp ytree/yson_struct_detail.cpp + ytree/yson_struct_update.cpp json/config.cpp json/json_callbacks.cpp diff --git a/yt/yt/core/yson/attribute_consumer.cpp b/yt/yt/core/yson/attribute_consumer.cpp index 10f30d27308..6e0c2f3ef60 100644 --- a/yt/yt/core/yson/attribute_consumer.cpp +++ b/yt/yt/core/yson/attribute_consumer.cpp @@ -3,6 +3,8 @@ namespace NYT::NYson { +using namespace NYTree; + //////////////////////////////////////////////////////////////////////////////// TAttributeFragmentConsumer::TAttributeFragmentConsumer(IAsyncYsonConsumer* underlyingConsumer) @@ -121,7 +123,7 @@ void TAttributeFragmentConsumer::Finish() TAttributeValueConsumer::TAttributeValueConsumer( IAsyncYsonConsumer* underlyingConsumer, - TString key) + IAttributeDictionary::TKey key) : UnderlyingConsumer_(underlyingConsumer) , Key_(std::move(key)) { } diff --git a/yt/yt/core/yson/attribute_consumer.h b/yt/yt/core/yson/attribute_consumer.h index 1239476c3cc..4ac8a1e54a2 100644 --- a/yt/yt/core/yson/attribute_consumer.h +++ b/yt/yt/core/yson/attribute_consumer.h @@ -3,6 +3,8 @@ #include "public.h" #include "async_consumer.h" +#include <yt/yt/core/ytree/attributes.h> + namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// @@ -54,7 +56,7 @@ class TAttributeValueConsumer public: TAttributeValueConsumer( IAsyncYsonConsumer* underlyingConsumer, - TString key); + NYTree::IAttributeDictionary::TKey key); void OnStringScalar(TStringBuf value) override; void OnInt64Scalar(i64 value) override; @@ -76,7 +78,7 @@ public: private: IAsyncYsonConsumer* const UnderlyingConsumer_; - const TString Key_; + const NYTree::IAttributeDictionary::TKey Key_; bool Empty_ = true; void ProduceKeyIfNeeded(); diff --git a/yt/yt/core/yson/config.cpp b/yt/yt/core/yson/config.cpp index eab523b4f88..0bf5b228fd5 100644 --- a/yt/yt/core/yson/config.cpp +++ b/yt/yt/core/yson/config.cpp @@ -9,7 +9,7 @@ void TProtobufInteropConfig::Register(TRegistrar registrar) registrar.Parameter("default_enum_yson_storage_type", &TThis::DefaultEnumYsonStorageType) .Default(EEnumYsonStorageType::String); registrar.Parameter("utf8_check", &TThis::Utf8Check) - .Default(EUtf8Check::Disable); + .Default(EUtf8Check::ThrowOnFail); registrar.Parameter("force_snake_case_names", &TThis::ForceSnakeCaseNames) .Default(false); } diff --git a/yt/yt/core/ytree/attribute_filter-inl.h b/yt/yt/core/ytree/attribute_filter-inl.h new file mode 100644 index 00000000000..a319270ad12 --- /dev/null +++ b/yt/yt/core/ytree/attribute_filter-inl.h @@ -0,0 +1,19 @@ +#ifndef ATTRIBUTE_FILER_INL_H_ +#error "Direct inclusion of this file is not allowed, include attribute_filter.h" +// For the sake of sane code completion. +#include "attribute_filter.h" +#endif + +namespace NYT::NYTree { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +TAttributeFilter::TAttributeFilter(std::initializer_list<T> keys) + : Keys({keys.begin(), keys.end()}) + , Universal(false) +{ } + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/attribute_filter.cpp b/yt/yt/core/ytree/attribute_filter.cpp index 796507ccf25..a8e72c13932 100644 --- a/yt/yt/core/ytree/attribute_filter.cpp +++ b/yt/yt/core/ytree/attribute_filter.cpp @@ -197,20 +197,21 @@ std::unique_ptr<IHeterogenousFilterConsumer> CreateFilteringConsumerImpl( //////////////////////////////////////////////////////////////////////////////// -TAttributeFilter::TAttributeFilter(std::vector<TString> keys, std::vector<TYPath> paths) +TAttributeFilter::TAttributeFilter(std::vector<IAttributeDictionary::TKey> keys, std::vector<TYPath> paths) : Keys(std::move(keys)) , Paths(std::move(paths)) , Universal(false) { } -TAttributeFilter& TAttributeFilter::operator =(std::vector<TString> keys) -{ - Keys = std::move(keys); - Paths = {}; - Universal = false; +TAttributeFilter::TAttributeFilter(std::initializer_list<TString> keys) + : Keys({keys.begin(), keys.end()}) + , Universal(false) +{ } - return *this; -} +TAttributeFilter::TAttributeFilter(const std::vector<TString>& keys) + : Keys({keys.begin(), keys.end()}) + , Universal(false) +{ } TAttributeFilter::operator bool() const { @@ -448,12 +449,12 @@ void Deserialize(TAttributeFilter& filter, const INodePtr& node) filter.Universal = false; filter.Keys.clear(); if (auto keysNode = mapNode->FindChild("keys")) { - filter.Keys = ConvertTo<std::vector<TString>>(keysNode); + filter.Keys = ConvertTo<std::vector<IAttributeDictionary::TKey>>(keysNode); } filter.Paths.clear(); if (auto pathsNode = mapNode->FindChild("paths")) { - filter.Paths = ConvertTo<std::vector<TString>>(pathsNode); + filter.Paths = ConvertTo<std::vector<TYPath>>(pathsNode); } break; @@ -461,7 +462,7 @@ void Deserialize(TAttributeFilter& filter, const INodePtr& node) case ENodeType::List: { // Compatibility mode with HTTP clients that specify attribute keys as string lists. filter.Universal = false; - filter.Keys = ConvertTo<std::vector<TString>>(node); + filter.Keys = ConvertTo<std::vector<IAttributeDictionary::TKey>>(node); filter.Paths = {}; break; } diff --git a/yt/yt/core/ytree/attribute_filter.h b/yt/yt/core/ytree/attribute_filter.h index 461d024fd37..679ca466761 100644 --- a/yt/yt/core/ytree/attribute_filter.h +++ b/yt/yt/core/ytree/attribute_filter.h @@ -1,13 +1,12 @@ #pragma once #include "public.h" +#include "attributes.h" #include <yt/yt/core/ypath/public.h> #include <yt/yt/core/yson/public.h> -#include <any> - namespace NYT::NYTree { //////////////////////////////////////////////////////////////////////////////// @@ -67,7 +66,7 @@ namespace NYT::NYTree { struct TAttributeFilter { //! Whitelist of top-level keys to be returned. - std::vector<TString> Keys; + std::vector<IAttributeDictionary::TKey> Keys; std::vector<NYPath::TYPath> Paths; //! If true, filter is universal, i.e. behavior depends on service's own policy; @@ -78,10 +77,17 @@ struct TAttributeFilter TAttributeFilter() = default; //! Creates a non-universal filter from given keys and paths. - //! This constructor is intentionally non-explicit so that common idiom attributeFilter = {"foo", "bar"} works. - TAttributeFilter(std::vector<TString> keys, std::vector<NYPath::TYPath> paths = {}); + /*implicit*/ TAttributeFilter( + std::vector<IAttributeDictionary::TKey> keys, + std::vector<NYPath::TYPath> paths = {}); + + //! Creates a non-universal filter from a given list of keys. + template <class T> + TAttributeFilter(std::initializer_list<T> keys); - TAttributeFilter& operator =(std::vector<TString> keys); + // TODO(babenko): switch to std::string + TAttributeFilter(std::initializer_list<TString> keys); + TAttributeFilter(const std::vector<TString>& keys); //! Returns true for non-universal filter and false otherwise. explicit operator bool() const; @@ -95,11 +101,11 @@ struct TAttributeFilter void ValidateKeysOnly(TStringBuf context = "this context") const; //! Returns true if #key appears in Keys or "/#key" appears in Paths using linear search. - bool AdmitsKeySlow(TStringBuf key) const; + bool AdmitsKeySlow(IAttributeDictionary::TKeyView key) const; // std::nullopt stands for "take the whole attribute without path filtration" (i.e. an equivalent of {""}). using TPathFilter = std::optional<std::vector<TYPath>>; - using TKeyToFilter = THashMap<TString, TPathFilter>; + using TKeyToFilter = THashMap<IAttributeDictionary::TKey, TPathFilter>; //! Normalization procedure removes redundant keys or paths and returns a mapping of form //! top-level key -> list of YPaths inside this top-level key (i.e. with /<key> stripped off) or @@ -161,3 +167,7 @@ void FormatValue( //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NYTree + +#define ATTRIBUTE_FILER_INL_H_ +#include "attribute_filter-inl.h" +#undef ATTRIBUTE_FILER_INL_H_ diff --git a/yt/yt/core/ytree/attributes-inl.h b/yt/yt/core/ytree/attributes-inl.h index 2ced0193d81..fc096d59dbf 100644 --- a/yt/yt/core/ytree/attributes-inl.h +++ b/yt/yt/core/ytree/attributes-inl.h @@ -4,8 +4,6 @@ #include "attributes.h" #endif -// #include "attribute_consumer.h" -// #include "serialize.h" #include "convert.h" #include <library/cpp/yt/error/error_attributes.h> @@ -17,7 +15,7 @@ namespace NYT { namespace NYTree { template <class T> -T IAttributeDictionary::Get(TStringBuf key) const +T IAttributeDictionary::Get(TKeyView key) const { auto yson = GetYson(key); try { @@ -30,7 +28,7 @@ T IAttributeDictionary::Get(TStringBuf key) const } template <class T> -T IAttributeDictionary::GetAndRemove(const TString& key) +T IAttributeDictionary::GetAndRemove(TKeyView key) { auto result = Get<T>(key); Remove(key); @@ -38,13 +36,13 @@ T IAttributeDictionary::GetAndRemove(const TString& key) } template <class T> -T IAttributeDictionary::Get(TStringBuf key, const T& defaultValue) const +T IAttributeDictionary::Get(TKeyView key, const T& defaultValue) const { return Find<T>(key).value_or(defaultValue); } template <class T> -T IAttributeDictionary::GetAndRemove(const TString& key, const T& defaultValue) +T IAttributeDictionary::GetAndRemove(TKeyView key, const T& defaultValue) { auto result = Find<T>(key); if (result) { @@ -56,7 +54,7 @@ T IAttributeDictionary::GetAndRemove(const TString& key, const T& defaultValue) } template <class T> -typename TOptionalTraits<T>::TOptional IAttributeDictionary::Find(TStringBuf key) const +typename TOptionalTraits<T>::TOptional IAttributeDictionary::Find(TKeyView key) const { auto yson = FindYson(key); if (!yson) { @@ -72,7 +70,7 @@ typename TOptionalTraits<T>::TOptional IAttributeDictionary::Find(TStringBuf key } template <class T> -typename TOptionalTraits<T>::TOptional IAttributeDictionary::FindAndRemove(const TString& key) +typename TOptionalTraits<T>::TOptional IAttributeDictionary::FindAndRemove(TKeyView key) { auto result = Find<T>(key); if (result) { @@ -82,7 +80,7 @@ typename TOptionalTraits<T>::TOptional IAttributeDictionary::FindAndRemove(const } template <class T> -void IAttributeDictionary::Set(const TString& key, const T& value) +void IAttributeDictionary::Set(TKeyView key, const T& value) { auto yson = ConvertToYsonString(value, NYson::EYsonFormat::Binary); SetYson(key, yson); diff --git a/yt/yt/core/ytree/attributes.cpp b/yt/yt/core/ytree/attributes.cpp index 58b88e72c8a..96f69f33c8e 100644 --- a/yt/yt/core/ytree/attributes.cpp +++ b/yt/yt/core/ytree/attributes.cpp @@ -9,7 +9,7 @@ using namespace NYson; //////////////////////////////////////////////////////////////////////////////// -TYsonString IAttributeDictionary::GetYson(TStringBuf key) const +TYsonString IAttributeDictionary::GetYson(TKeyView key) const { auto result = FindYson(key); if (!result) { @@ -18,7 +18,7 @@ TYsonString IAttributeDictionary::GetYson(TStringBuf key) const return result; } -TYsonString IAttributeDictionary::GetYsonAndRemove(const TString& key) +TYsonString IAttributeDictionary::GetYsonAndRemove(TKeyView key) { auto result = GetYson(key); Remove(key); diff --git a/yt/yt/core/ytree/attributes.h b/yt/yt/core/ytree/attributes.h index 0c99076f19a..15457ad4e59 100644 --- a/yt/yt/core/ytree/attributes.h +++ b/yt/yt/core/ytree/attributes.h @@ -13,25 +13,26 @@ namespace NYT::NYTree { struct IAttributeDictionary : public TRefCounted { - using TKey = TString; + using TKey = std::string; + using TKeyView = TStringBuf; using TValue = NYson::TYsonString; using TKeyValuePair = std::pair<TKey, TValue>; //! Returns the list of all keys in the dictionary. - virtual std::vector<TString> ListKeys() const = 0; + virtual std::vector<TKey> ListKeys() const = 0; //! Returns the list of all key-value pairs in the dictionary. virtual std::vector<TKeyValuePair> ListPairs() const = 0; //! Returns the value of the attribute (null indicates that the attribute is not found). - virtual NYson::TYsonString FindYson(TStringBuf key) const = 0; + virtual TValue FindYson(TKeyView key) const = 0; //! Sets the value of the attribute. - virtual void SetYson(const TString& key, const NYson::TYsonString& value) = 0; + virtual void SetYson(TKeyView key, const TValue& value) = 0; //! Removes the attribute. //! Returns |true| if the attribute was removed or |false| if there is no attribute with this key. - virtual bool Remove(const TString& key) = 0; + virtual bool Remove(TKeyView key) = 0; // Extension methods @@ -39,44 +40,44 @@ struct IAttributeDictionary void Clear(); //! Returns the value of the attribute (throws an exception if the attribute is not found). - NYson::TYsonString GetYson(TStringBuf key) const; + TValue GetYson(TKeyView key) const; //! Same as #GetYson but removes the value. - NYson::TYsonString GetYsonAndRemove(const TString& key); + TValue GetYsonAndRemove(TKeyView key); //! Finds the attribute and deserializes its value. //! Throws if no such value is found. template <class T> - T Get(TStringBuf key) const; + T Get(TKeyView key) const; //! Same as #Get but removes the value. template <class T> - T GetAndRemove(const TString& key); + T GetAndRemove(TKeyView key); //! Finds the attribute and deserializes its value. //! Uses default value if no such attribute is found. template <class T> - T Get(TStringBuf key, const T& defaultValue) const; + T Get(TKeyView key, const T& defaultValue) const; //! Same as #Get but removes the value if it exists. template <class T> - T GetAndRemove(const TString& key, const T& defaultValue); + T GetAndRemove(TKeyView key, const T& defaultValue); //! Finds the attribute and deserializes its value. //! Returns null if no such attribute is found. template <class T> - typename TOptionalTraits<T>::TOptional Find(TStringBuf key) const; + typename TOptionalTraits<T>::TOptional Find(TKeyView key) const; //! Same as #Find but removes the value if it exists. template <class T> - typename TOptionalTraits<T>::TOptional FindAndRemove(const TString& key); + typename TOptionalTraits<T>::TOptional FindAndRemove(TKeyView key); //! Returns |true| iff the given key is present. - bool Contains(TStringBuf key) const; + bool Contains(TKeyView key) const; //! Sets the attribute with a serialized value. template <class T> - void Set(const TString& key, const T& value); + void Set(TKeyView key, const T& value); //! Constructs an instance from a map node (by serializing the values). static IAttributeDictionaryPtr FromMap(const IMapNodePtr& node); diff --git a/yt/yt/core/ytree/convert-inl.h b/yt/yt/core/ytree/convert-inl.h index 6d1294f6465..8dd191066e5 100644 --- a/yt/yt/core/ytree/convert-inl.h +++ b/yt/yt/core/ytree/convert-inl.h @@ -8,7 +8,6 @@ #include "default_building_consumer.h" #include "serialize.h" #include "tree_builder.h" -#include "helpers.h" #include <yt/yt/core/ypath/token.h> @@ -158,7 +157,10 @@ INodePtr ConvertToNode( template <class T> IAttributeDictionaryPtr ConvertToAttributes(const T& value) { - auto attributes = CreateEphemeralAttributes(); + // Forward declaration. + IAttributeDictionaryPtr CreateEphemeralAttributes(std::optional<int> ysonNestingLevelLimit); + + auto attributes = CreateEphemeralAttributes(std::nullopt); TAttributeConsumer consumer(attributes.Get()); Serialize(value, &consumer); return attributes; diff --git a/yt/yt/core/ytree/helpers.cpp b/yt/yt/core/ytree/helpers.cpp index b90c8ebd184..a97708fd2ce 100644 --- a/yt/yt/core/ytree/helpers.cpp +++ b/yt/yt/core/ytree/helpers.cpp @@ -5,6 +5,8 @@ #include <yt/yt/core/misc/error.h> +#include <library/cpp/yt/memory/leaky_ref_counted_singleton.h> + namespace NYT::NYTree { using namespace NYson; @@ -55,9 +57,9 @@ public: : NestingLevelLimit_(ysonNestingLevelLimit) { } - std::vector<TString> ListKeys() const override + std::vector<TKey> ListKeys() const override { - std::vector<TString> keys; + std::vector<TKey> keys; keys.reserve(Map_.size()); for (const auto& [key, value] : Map_) { keys.push_back(key); @@ -75,13 +77,13 @@ public: return pairs; } - TYsonString FindYson(TStringBuf key) const override + TValue FindYson(TKeyView key) const override { auto it = Map_.find(key); return it == Map_.end() ? TYsonString() : it->second; } - void SetYson(const TString& key, const TYsonString& value) override + void SetYson(TKeyView key, const TValue& value) override { YT_ASSERT(value.GetType() == EYsonType::Node); if (NestingLevelLimit_) { @@ -90,13 +92,13 @@ public: Map_[key] = value; } - bool Remove(const TString& key) override + bool Remove(TKeyView key) override { return Map_.erase(key) > 0; } private: - THashMap<TString, TYsonString> Map_; + THashMap<TKey, TYsonString, THash<std::string_view>, TEqualTo<std::string_view>> Map_; std::optional<int> NestingLevelLimit_; }; @@ -111,7 +113,7 @@ class TEmptyAttributeDictionary : public IAttributeDictionary { public: - std::vector<TString> ListKeys() const override + std::vector<TKey> ListKeys() const override { return {}; } @@ -121,30 +123,29 @@ public: return {}; } - TYsonString FindYson(TStringBuf /*key*/) const override + TValue FindYson(TKeyView /*key*/) const override { return {}; } - void SetYson(const TString& /*key*/, const TYsonString& /*value*/) override + void SetYson(TKeyView /*key*/, const TValue& /*value*/) override { YT_ABORT(); } - bool Remove(const TString& /*key*/) override + bool Remove(TKeyView /*key*/) override { return false; } + +private: + DECLARE_LEAKY_REF_COUNTED_SINGLETON_FRIEND() + TEmptyAttributeDictionary() = default; }; const IAttributeDictionary& EmptyAttributes() { - struct TSingleton - { - IAttributeDictionaryPtr EmptyAttributes = New<TEmptyAttributeDictionary>(); - }; - - return *LeakySingleton<TSingleton>()->EmptyAttributes; + return *LeakyRefCountedSingleton<TEmptyAttributeDictionary>(); } //////////////////////////////////////////////////////////////////////////////// @@ -157,7 +158,7 @@ public: : Underlying_(underlying) { } - std::vector<TString> ListKeys() const override + std::vector<TKey> ListKeys() const override { auto guard = ReaderGuard(Lock_); return Underlying_->ListKeys(); @@ -169,19 +170,19 @@ public: return Underlying_->ListPairs(); } - NYson::TYsonString FindYson(TStringBuf key) const override + TValue FindYson(TKeyView key) const override { auto guard = ReaderGuard(Lock_); return Underlying_->FindYson(key); } - void SetYson(const TString& key, const NYson::TYsonString& value) override + void SetYson(TKeyView key, const TValue& value) override { auto guard = WriterGuard(Lock_); Underlying_->SetYson(key, value); } - bool Remove(const TString& key) override + bool Remove(TKeyView key) override { auto guard = WriterGuard(Lock_); return Underlying_->Remove(key); @@ -300,7 +301,7 @@ void TAttributeDictionarySerializer::LoadNonNull(TStreamLoadContext& context, co //////////////////////////////////////////////////////////////////////////////// -void ValidateYTreeKey(TStringBuf key) +void ValidateYTreeKey(IAttributeDictionary::TKeyView key) { Y_UNUSED(key); // XXX(vvvv): Disabled due to existing data with empty keys, see https://st.yandex-team.ru/YQL-2640 diff --git a/yt/yt/core/ytree/helpers.h b/yt/yt/core/ytree/helpers.h index 2d5b4fc8ec4..a51a2375969 100644 --- a/yt/yt/core/ytree/helpers.h +++ b/yt/yt/core/ytree/helpers.h @@ -1,6 +1,7 @@ #pragma once #include "public.h" +#include "attributes.h" #include <yt/yt/core/misc/serialize.h> @@ -46,15 +47,14 @@ struct TAttributeDictionarySerializer //////////////////////////////////////////////////////////////////////////////// -void ValidateYTreeKey(TStringBuf key); +void ValidateYTreeKey(IAttributeDictionary::TKeyView key); void ValidateYPathResolutionDepth(TYPathBuf path, int depth); //! Helps implementing IAttributeDictionary::ListPairs by delegating to //! IAttributeDictionary::ListKeys and IAttributeDictionary::FindYson for those not capable //! of providing a custom efficient implementation. -std::vector<std::pair<TString, NYson::TYsonString>> ListAttributesPairs(const IAttributeDictionary& attributes); - +std::vector<std::pair<IAttributeDictionary::TKey, NYson::TYsonString>> ListAttributesPairs(const IAttributeDictionary& attributes); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/unittests/attributes_ut.cpp b/yt/yt/core/ytree/unittests/attributes_ut.cpp index b53fafdaf6f..f0f4c9ff9ec 100644 --- a/yt/yt/core/ytree/unittests/attributes_ut.cpp +++ b/yt/yt/core/ytree/unittests/attributes_ut.cpp @@ -2,6 +2,7 @@ #include <yt/yt/core/ytree/attributes.h> #include <yt/yt/core/ytree/convert.h> +#include <yt/yt/core/ytree/helpers.h> #include <yt/yt/core/yson/string.h> diff --git a/yt/yt/core/ytree/unittests/ya.make b/yt/yt/core/ytree/unittests/ya.make index 1bc54b95aa9..172b7702944 100644 --- a/yt/yt/core/ytree/unittests/ya.make +++ b/yt/yt/core/ytree/unittests/ya.make @@ -14,6 +14,7 @@ SRCS( tree_builder_ut.cpp lazy_ypath_service_ut.cpp yson_schema_ut.cpp + yson_struct_update_ut.cpp yson_struct_ut.cpp ytree_fluent_ut.cpp ytree_ut.cpp diff --git a/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp new file mode 100644 index 00000000000..4ace5c047c7 --- /dev/null +++ b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp @@ -0,0 +1,175 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/ytree/convert.h> +#include <yt/yt/core/ytree/ypath_client.h> +#include <yt/yt/core/ytree/yson_struct.h> +#include <yt/yt/core/ytree/yson_struct_update.h> + +namespace NYT::NYTree { +namespace { + +using namespace NYson; + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_STRUCT(TMapperSpec) + +struct TMapperSpec + : public TYsonStruct +{ + std::string Command; + + REGISTER_YSON_STRUCT(TMapperSpec); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("command", &TThis::Command) + .Default("cat"); + } +}; + +DEFINE_REFCOUNTED_TYPE(TMapperSpec) + +DECLARE_REFCOUNTED_STRUCT(TSpecWithPool) + +struct TSpecWithPool + : public TYsonStruct +{ +public: + std::string Pool; + int NonUpdatable; + + REGISTER_YSON_STRUCT(TSpecWithPool); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("pool", &TThis::Pool); + registrar.Parameter("non_updatable", &TThis::NonUpdatable) + .Default(10); + } +}; + +DEFINE_REFCOUNTED_TYPE(TSpecWithPool) + +DECLARE_REFCOUNTED_STRUCT(TSpecBase) + +// TODO(coteeq): Validate +// static constexpr auto BadMaxFailedJobCount = 42; + +struct TSpecBase + : public TSpecWithPool +{ +public: + int MaxFailedJobCount; + TMapperSpecPtr Mapper; + + REGISTER_YSON_STRUCT(TSpecBase); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("max_failed_job_count", &TThis::MaxFailedJobCount) + .Default(10) + .GreaterThan(0) + .LessThan(1000); + + registrar.Parameter("mapper", &TThis::Mapper) + .DefaultNew(); + } +}; + +DEFINE_REFCOUNTED_TYPE(TSpecBase) + +TEST(TUpdateYsonStructTest, Simple) +{ + auto oldSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{pool=pool;}"))); + auto newSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{pool=new_pool;}"))); + + std::string updatedPool; + + auto configurator = NYsonStructUpdate::TConfigurator<TSpecWithPool>(); + { + configurator.Field("pool", &TSpecBase::Pool) + .Updater(BIND([&] (const std::string& newPool) { + updatedPool = newPool; + })); + } + + NYsonStructUpdate::Update(configurator, oldSpec, newSpec); + + EXPECT_EQ(updatedPool, "new_pool"); +} + +TEST(TUpdateYsonStructTest, NonUpdatable) +{ + auto oldSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{non_updatable=42;pool=pool}"))); + auto newSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{non_updatable=43;pool=pool}"))); + + std::string updatedPool; + + auto configurator = NYsonStructUpdate::TConfigurator<TSpecWithPool>(); + { + configurator.Field("pool", &TSpecBase::Pool) + .Updater(BIND([&] (const std::string& newPool) { + updatedPool = newPool; + })); + } + + EXPECT_ANY_THROW({ + NYsonStructUpdate::Update(configurator, oldSpec, newSpec); + }); +} + + +TEST(TUpdateYsonStructTest, Inherited) +{ + auto oldSpec = ConvertTo<TSpecBasePtr>(TYsonString(TString("{pool=pool;}"))); + auto newSpec = ConvertTo<TSpecBasePtr>(TYsonString(TString("{pool=new_pool;}"))); + + std::string updatedPool; + + auto configurator = NYsonStructUpdate::TConfigurator<TSpecBase>(); + { + NYsonStructUpdate::TConfigurator<TSpecWithPool> parentRegistrar = configurator; + parentRegistrar.Field("pool", &TSpecBase::Pool) + .Updater(BIND([&] (const std::string& newPool) { + updatedPool = newPool; + })); + } + + NYsonStructUpdate::Update(configurator, oldSpec, newSpec); + + EXPECT_EQ(updatedPool, "new_pool"); +} + +TEST(TUpdateYsonStructTest, Nested) +{ + auto oldSpec = ConvertTo<TSpecBasePtr>(TYsonString(TString("{pool=pool; mapper={command=cat};}"))); + auto newSpec = ConvertTo<TSpecBasePtr>(TYsonString(TString("{pool=new_pool;mapper={command=sort};}"))); + + std::string updatedPool; + std::string updatedCommand; + + auto configurator = NYsonStructUpdate::TConfigurator<TSpecBase>(); + { + NYsonStructUpdate::TConfigurator<TSpecWithPool> parentRegistrar = configurator; + parentRegistrar.Field("pool", &TSpecBase::Pool) + .Updater(BIND([&] (const std::string& newPool) { + updatedPool = newPool; + })); + } + configurator.Field("mapper", &TSpecBase::Mapper) + .NestedUpdater(BIND([&](NYsonStructUpdate::TConfigurator<TMapperSpec> configurator) { + configurator.Field("command", &TMapperSpec::Command) + .Updater(BIND([&] (const std::string& newCommand) { + updatedCommand = newCommand; + })); + })); + + NYsonStructUpdate::Update(configurator, oldSpec, newSpec); + + EXPECT_EQ(updatedPool, "new_pool"); + EXPECT_EQ(updatedCommand, "sort"); +} + +} // namespace +} // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/ypath_detail.cpp b/yt/yt/core/ytree/ypath_detail.cpp index a0b5d421f6a..f9f2ee05d71 100644 --- a/yt/yt/core/ytree/ypath_detail.cpp +++ b/yt/yt/core/ytree/ypath_detail.cpp @@ -280,9 +280,9 @@ TSupportsAttributes::TCombinedAttributeDictionary::TCombinedAttributeDictionary( : Owner_(owner) { } -std::vector<TString> TSupportsAttributes::TCombinedAttributeDictionary::ListKeys() const +auto TSupportsAttributes::TCombinedAttributeDictionary::ListKeys() const -> std::vector<TKey> { - std::vector<TString> keys; + std::vector<TKey> keys; auto* provider = Owner_->GetBuiltinAttributeProvider(); if (provider) { @@ -305,7 +305,7 @@ std::vector<TString> TSupportsAttributes::TCombinedAttributeDictionary::ListKeys return keys; } -std::vector<IAttributeDictionary::TKeyValuePair> TSupportsAttributes::TCombinedAttributeDictionary::ListPairs() const +auto TSupportsAttributes::TCombinedAttributeDictionary::ListPairs() const -> std::vector<TKeyValuePair> { std::vector<TKeyValuePair> pairs; @@ -334,7 +334,7 @@ std::vector<IAttributeDictionary::TKeyValuePair> TSupportsAttributes::TCombinedA return pairs; } -TYsonString TSupportsAttributes::TCombinedAttributeDictionary::FindYson(TStringBuf key) const +auto TSupportsAttributes::TCombinedAttributeDictionary::FindYson(TKeyView key) const -> TValue { auto* provider = Owner_->GetBuiltinAttributeProvider(); if (provider) { @@ -354,7 +354,7 @@ TYsonString TSupportsAttributes::TCombinedAttributeDictionary::FindYson(TStringB return customAttributes->FindYson(key); } -void TSupportsAttributes::TCombinedAttributeDictionary::SetYson(const TString& key, const TYsonString& value) +void TSupportsAttributes::TCombinedAttributeDictionary::SetYson(TKeyView key, const TYsonString& value) { auto* provider = Owner_->GetBuiltinAttributeProvider(); if (provider) { @@ -377,7 +377,7 @@ void TSupportsAttributes::TCombinedAttributeDictionary::SetYson(const TString& k customAttributes->SetYson(key, value); } -bool TSupportsAttributes::TCombinedAttributeDictionary::Remove(const TString& key) +bool TSupportsAttributes::TCombinedAttributeDictionary::Remove(TKeyView key) { auto* provider = Owner_->GetBuiltinAttributeProvider(); if (provider) { diff --git a/yt/yt/core/ytree/ypath_detail.h b/yt/yt/core/ytree/ypath_detail.h index 8615d5ce259..852f8bae936 100644 --- a/yt/yt/core/ytree/ypath_detail.h +++ b/yt/yt/core/ytree/ypath_detail.h @@ -322,11 +322,11 @@ private: public: explicit TCombinedAttributeDictionary(TSupportsAttributes* owner); - std::vector<TString> ListKeys() const override; + std::vector<TKey> ListKeys() const override; std::vector<TKeyValuePair> ListPairs() const override; - NYson::TYsonString FindYson(TStringBuf key) const override; - void SetYson(const TString& key, const NYson::TYsonString& value) override; - bool Remove(const TString& key) override; + TValue FindYson(TKeyView key) const override; + void SetYson(TKeyView key, const TValue& value) override; + bool Remove(TKeyView key) override; private: TSupportsAttributes* const Owner_; diff --git a/yt/yt/core/ytree/yson_struct.cpp b/yt/yt/core/ytree/yson_struct.cpp index b4481ea422d..d101854ca0f 100644 --- a/yt/yt/core/ytree/yson_struct.cpp +++ b/yt/yt/core/ytree/yson_struct.cpp @@ -151,6 +151,11 @@ bool TYsonStructBase::IsEqual(const TYsonStructBase& rhs) const return Meta_->CompareStructs(this, &rhs); } +const IYsonStructMeta* TYsonStructBase::GetMeta() const +{ + return Meta_; +} + //////////////////////////////////////////////////////////////////////////////// void TYsonStruct::InitializeRefCounted() diff --git a/yt/yt/core/ytree/yson_struct.h b/yt/yt/core/ytree/yson_struct.h index bc42b7073ec..4d917baf7ab 100644 --- a/yt/yt/core/ytree/yson_struct.h +++ b/yt/yt/core/ytree/yson_struct.h @@ -116,6 +116,8 @@ public: // a member method. bool IsEqual(const TYsonStructBase& rhs) const; + const IYsonStructMeta* GetMeta() const; + private: template <class TValue> friend class TYsonStructParameter; diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h index 3a4cdea34fa..11ed3dfdd5d 100644 --- a/yt/yt/core/ytree/yson_struct_detail-inl.h +++ b/yt/yt/core/ytree/yson_struct_detail-inl.h @@ -18,6 +18,23 @@ namespace NYT::NYTree { //////////////////////////////////////////////////////////////////////////////// +template <class TStruct, class TValue> +struct TTypedYsonStructField + : public ITypeErasedYsonStructField +{ + TYsonStructField<TStruct, TValue> Field; +}; + +template <class TStruct, class TValue> +ITypeErasedYsonStructFieldPtr CreateTypeErasedYsonStructField(TYsonStructField<TStruct, TValue> field) +{ + auto erasedField = New<TTypedYsonStructField<TStruct, TValue>>(); + erasedField->Field = field; + return erasedField; +} + +//////////////////////////////////////////////////////////////////////////////// + namespace NPrivate { //////////////////////////////////////////////////////////////////////////////// @@ -722,6 +739,15 @@ TValue& TYsonFieldAccessor<TStruct, TValue>::GetValue(const TYsonStructBase* sou return TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(source)->*Field_; } +template <class TStruct, class TValue> +bool TYsonFieldAccessor<TStruct, TValue>::HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const +{ + if (auto typedThat = DynamicPointerCast<TTypedYsonStructField<TStruct, TValue>>(erasedField)) { + return typedThat->Field == Field_; + } + return false; +} + //////////////////////////////////////////////////////////////////////////////// template <class TStruct, class TValue> @@ -730,6 +756,12 @@ TUniversalYsonParameterAccessor<TStruct, TValue>::TUniversalYsonParameterAccesso { } template <class TStruct, class TValue> +bool TUniversalYsonParameterAccessor<TStruct, TValue>::HoldsField(ITypeErasedYsonStructFieldPtr /*field*/) const +{ + YT_UNIMPLEMENTED(); +} + +template <class TStruct, class TValue> TValue& TUniversalYsonParameterAccessor<TStruct, TValue>::GetValue(const TYsonStructBase* source) { return Accessor_(TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(source)); @@ -1030,6 +1062,19 @@ int TYsonStructParameter<TValue>::GetFieldIndex() const return FieldIndex_; } +template <class TValue> +const TValue& TYsonStructParameter<TValue>::GetValue(const TYsonStructBase* source) const +{ + YT_VERIFY(FieldAccessor_); + return FieldAccessor_->GetValue(source); +} + +template <class TValue> +bool TYsonStructParameter<TValue>::HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const +{ + return FieldAccessor_ && FieldAccessor_->HoldsField(erasedField); +} + //////////////////////////////////////////////////////////////////////////////// // Standard postprocessors diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h index 3ed99df7ec2..9cbcdfcfdc2 100644 --- a/yt/yt/core/ytree/yson_struct_detail.h +++ b/yt/yt/core/ytree/yson_struct_detail.h @@ -47,6 +47,21 @@ struct TYsonSourceTraits template <class TStruct, class TValue> using TYsonStructField = TValue(TStruct::*); +// This is intended to be used as an equality-only comparator of YsonStruct fields. +// dynamic_cast is used to compare generic #ITypeErasedYsonStructField to +// a concrete #TTypedYsonStructField (see -inl.h). +struct ITypeErasedYsonStructField + : public TRefCounted +{ }; + +DECLARE_REFCOUNTED_STRUCT(ITypeErasedYsonStructField); +DEFINE_REFCOUNTED_TYPE(ITypeErasedYsonStructField); + +template <class TStruct, class TValue> +ITypeErasedYsonStructFieldPtr CreateTypeErasedYsonStructField(TYsonStructField<TStruct, TValue> field); + +//////////////////////////////////////////////////////////////////////////////// + struct TLoadParameterOptions { NYPath::TYPath Path; @@ -92,6 +107,8 @@ struct IYsonStructParameter virtual bool CompareParameter(const TYsonStructBase* lhsSelf, const TYsonStructBase* rhsSelf) const = 0; virtual int GetFieldIndex() const = 0; + + virtual bool HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const = 0; }; DECLARE_REFCOUNTED_STRUCT(IYsonStructParameter) @@ -211,6 +228,7 @@ template <class TValue> struct IYsonFieldAccessor { virtual TValue& GetValue(const TYsonStructBase* source) = 0; + virtual bool HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const = 0; virtual ~IYsonFieldAccessor() = default; }; @@ -222,6 +240,7 @@ class TYsonFieldAccessor { public: explicit TYsonFieldAccessor(TYsonStructField<TStruct, TValue> field); + bool HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const override; TValue& GetValue(const TYsonStructBase* source) override; private: @@ -236,6 +255,7 @@ class TUniversalYsonParameterAccessor { public: explicit TUniversalYsonParameterAccessor(std::function<TValue&(TStruct*)> field); + bool HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const override; TValue& GetValue(const TYsonStructBase* source) override; private: @@ -288,6 +308,9 @@ public: virtual int GetFieldIndex() const override; + const TValue& GetValue(const TYsonStructBase* source) const; + bool HoldsField(ITypeErasedYsonStructFieldPtr erasedField) const override; + // Mark as optional. Field will be default-initialized if `init` is true, initialization is skipped otherwise. TYsonStructParameter& Optional(bool init = true); // Set default value. It will be copied during instance initialization. diff --git a/yt/yt/core/ytree/yson_struct_update-inl.h b/yt/yt/core/ytree/yson_struct_update-inl.h new file mode 100644 index 00000000000..c8dceb91163 --- /dev/null +++ b/yt/yt/core/ytree/yson_struct_update-inl.h @@ -0,0 +1,194 @@ +#ifndef YSON_STRUCT_UPDATE_INL_H_ +#error "Direct inclusion of this file is not allowed, include yson_struct_update.h" +// For the sake of sane code completion. +#include "yson_struct_update.h" +#endif + +namespace NYT::NYTree::NYsonStructUpdate { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +struct TRegisteredFieldDirectory + : public TRefCounted +{ + template <class TStruct> + static TRegisteredFieldDirectoryPtr Create() + { + static auto meta = GetMeta<TStruct>(); + + auto directory = New<TRegisteredFieldDirectory>(); + directory->Meta = meta; + return directory; + } + + THashMap<IYsonStructParameterPtr, IFieldRegistrarPtr> ParameterToFieldRegistrar; + const IYsonStructMeta* Meta; + +private: + template <class TStruct> + static const IYsonStructMeta* GetMeta() { + return New<TStruct>()->GetMeta(); + } +}; + +DEFINE_REFCOUNTED_TYPE(TRegisteredFieldDirectory); + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +struct TUnwrapYsonStructIntrusivePtr +{ }; + +template <class T> +struct TUnwrapYsonStructIntrusivePtr<TIntrusivePtr<T>> +{ + using TStruct = T; +}; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +template <CYsonStructDerived TStruct> +TConfigurator<TStruct>::TConfigurator(NDetail::TRegisteredFieldDirectoryPtr registeredFields) + : RegisteredFields_(registeredFields) +{ + // NB: Initialize TRegisteredFieldDirectory with + // the youngest child in the hierarchy. + if (!registeredFields) { + RegisteredFields_ = NDetail::TRegisteredFieldDirectory::Create<TStruct>(); + } +} + +template <CYsonStructDerived TStruct> +template <class TValue> +TFieldRegistrar<TValue>& TConfigurator<TStruct>::Field(const TString& name, TYsonStructField<TStruct, TValue> field) +{ + IYsonStructParameterPtr parameter; + + try { + parameter = RegisteredFields_->Meta->GetParameter(name); + } catch (const std::exception& ex) { + YT_ABORT(); + } + YT_VERIFY(parameter->HoldsField(CreateTypeErasedYsonStructField(field))); + + auto fieldRegistrar = New<TFieldRegistrar<TValue>>(); + RegisteredFields_->ParameterToFieldRegistrar.emplace(parameter, fieldRegistrar); + return *fieldRegistrar; +} + +template <CYsonStructDerived TStruct> +template <class TAncestor> +TConfigurator<TStruct>::operator TConfigurator<TAncestor>() const +{ + static_assert(std::derived_from<TStruct, TAncestor> && std::derived_from<TAncestor, TYsonStructBase>); + return TConfigurator<TAncestor>(RegisteredFields_); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class TValue> +TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&, const TValue&)> updater) +{ + VerifyEmpty(); + Updater_ = updater; + return *this; +} + +template <class TValue> +TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&)> updater) +{ + VerifyEmpty(); + Updater_ = BIND([updater = std::move(updater)] (const TValue& /*old*/, const TValue& new_) { + updater(std::move(new_)); + }); + return *this; +} + +// NB(coteeq): Little note on the weirdness of the signature: +// +// #TValue could be either YsonStruct or not. In case it is YsonStruct, I want +// to provide #NestedUpdater, that will statically check that TValue is +// actually an YsonStruct. +// On the other hand, if TValue is not an YsonStruct, I do not want this method +// to be callable (or to even exist). +// Thirdly, the signature of this method requires me to specify +// TConfigurator<TValue> that will complain if TValue is not CYsonStructDerived. +// +// So I made this method templated with <CYsonStructDerived TOtherValue>. This +// way, TConfigurator<TOtherValue> always makes sense, I just need to static_assert +// that TOtherValue == TValue. Last thing is that nested YsonStruct is always +// written as TIntrusivePtr<TMyStruct>, so I need to unwrap that to TMyStruct. +template <class TValue> +template <CYsonStructDerived TUnwrappedValue> +TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::NestedUpdater( + TCallback<void(TConfigurator<TUnwrappedValue>)> registerCallback) +{ + static_assert( + std::is_same_v< + TUnwrappedValue, + typename NDetail::TUnwrapYsonStructIntrusivePtr<TValue>::TStruct>); + + VerifyEmpty(); + TConfigurator<TUnwrappedValue> registrar; + registerCallback(registrar); + Updater_ = BIND([registrar = std::move(registrar)] (const TValue& old, const TValue& new_) { + Update(registrar, old, new_); + }); + return *this; +} + +template <class TValue> +void TFieldRegistrar<TValue>::DoUpdate( + IYsonStructParameterPtr parameter, + TYsonStructBase* old, + TYsonStructBase* new_) const +{ + if (!Updater_) { + return; + } + + auto typedParameter = DynamicPointerCast<TYsonStructParameter<TValue>>(parameter); + YT_VERIFY(typedParameter); + Updater_( + typedParameter->GetValue(old), + typedParameter->GetValue(new_)); +} + +template <class TValue> +void TFieldRegistrar<TValue>::VerifyEmpty() const +{ + YT_VERIFY(!Updater_); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class TStruct> +void Update( + const TConfigurator<TStruct>& registrar, + TIntrusivePtr<TStruct> old, + TIntrusivePtr<TStruct> new_) +{ + const auto* meta = old->GetMeta(); + const auto& parameterToFieldRegistrar = registrar.RegisteredFields_->ParameterToFieldRegistrar; + for (const auto& [name, parameter] : meta->GetParameterMap()) { + if (parameter->CompareParameter(old.Get(), new_.Get())) { + continue; + } + + if (auto fieldDescIter = parameterToFieldRegistrar.find(parameter); + fieldDescIter != parameterToFieldRegistrar.end() + ) { + fieldDescIter->second->DoUpdate(parameter, old.Get(), new_.Get()); + } else { + THROW_ERROR_EXCEPTION("Field %Qv is not marked as updatable, but was changed", name); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree::NYsonStructUpdate diff --git a/yt/yt/core/ytree/yson_struct_update.cpp b/yt/yt/core/ytree/yson_struct_update.cpp new file mode 100644 index 00000000000..a2315860ae8 --- /dev/null +++ b/yt/yt/core/ytree/yson_struct_update.cpp @@ -0,0 +1 @@ +#include "yson_struct_update.h" diff --git a/yt/yt/core/ytree/yson_struct_update.h b/yt/yt/core/ytree/yson_struct_update.h new file mode 100644 index 00000000000..2444329254d --- /dev/null +++ b/yt/yt/core/ytree/yson_struct_update.h @@ -0,0 +1,101 @@ +#pragma once + +#include "yson_struct.h" +#include "yson_struct_detail.h" + +namespace NYT::NYTree::NYsonStructUpdate { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +DECLARE_REFCOUNTED_STRUCT(TRegisteredFieldDirectory); + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +struct IFieldRegistrar + : public TRefCounted +{ + virtual void DoUpdate( + IYsonStructParameterPtr parameter, + TYsonStructBase* old, + TYsonStructBase* new_) const = 0; +}; + +DECLARE_REFCOUNTED_STRUCT(IFieldRegistrar); +DEFINE_REFCOUNTED_TYPE(IFieldRegistrar); + +//////////////////////////////////////////////////////////////////////////////// + +template <class TValue> +struct TFieldRegistrar; + +//////////////////////////////////////////////////////////////////////////////// + +template <CYsonStructDerived TStruct> +struct TConfigurator +{ + explicit TConfigurator(NDetail::TRegisteredFieldDirectoryPtr state = {}); + + template <class TValue> + TFieldRegistrar<TValue>& Field(const TString& name, TYsonStructField<TStruct, TValue> field); + + // Converts to a registrar of a base class + template <class TAncestor> + operator TConfigurator<TAncestor>() const; + +private: + NDetail::TRegisteredFieldDirectoryPtr RegisteredFields_; + + template <class TUpdateStruct> + friend void Update( + const TConfigurator<TUpdateStruct>& registrar, + TIntrusivePtr<TUpdateStruct> old, + TIntrusivePtr<TUpdateStruct> new_); +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class TValue> +struct TFieldRegistrar + : public IFieldRegistrar +{ + // Registers updater that accepts old and new values as arguments. + TFieldRegistrar& Updater(TCallback<void(const TValue&, const TValue&)> updater); + + // Registers updater that accepts only new value as an argument. + TFieldRegistrar& Updater(TCallback<void(const TValue&)> updater); + + // Registers nested YsonStruct to be updated recursively. + template <CYsonStructDerived TUnwrappedValue> + TFieldRegistrar& NestedUpdater( + TCallback<void(TConfigurator<TUnwrappedValue>)> registerCb); + + void DoUpdate( + IYsonStructParameterPtr parameter, + TYsonStructBase* old, + TYsonStructBase* new_) const override; + +private: + void VerifyEmpty() const; + + TCallback<void(const TValue&, const TValue&)> Updater_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class TStruct> +void Update( + const TConfigurator<TStruct>& registrar, + TIntrusivePtr<TStruct> old, + TIntrusivePtr<TStruct> new_); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree::NYsonStructUpdate + +#define YSON_STRUCT_UPDATE_INL_H_ +#include "yson_struct_update-inl.h" +#undef YSON_STRUCT_UPDATE_INL_H_ diff --git a/yt/yt/library/numeric/algorithm_helpers-inl.h b/yt/yt/library/numeric/algorithm_helpers-inl.h index 980fccb5c34..4fa0417a65d 100644 --- a/yt/yt/library/numeric/algorithm_helpers-inl.h +++ b/yt/yt/library/numeric/algorithm_helpers-inl.h @@ -4,6 +4,8 @@ #include "algorithm_helpers.h" #endif +#include <stdlib.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/ytprof/integration/test_http.py b/yt/yt/library/ytprof/integration/test_http.py index 2e274ac1056..5d3b5ae17bd 100644 --- a/yt/yt/library/ytprof/integration/test_http.py +++ b/yt/yt/library/ytprof/integration/test_http.py @@ -90,16 +90,16 @@ def test_spinlock_profile(running_example): if yatest.common.context.build_type != "profile": pytest.skip() - fetch_data(running_example, "lock?d=1") - fetch_data(running_example, "lock?d=1&frac=1") + fetch_data(running_example, "spinlock/lock?d=1") + fetch_data(running_example, "spinlock/lock?d=1&frac=1") def test_block_profile(running_example): if yatest.common.context.build_type != "profile": pytest.skip() - fetch_data(running_example, "block?d=1") - fetch_data(running_example, "block?d=1&frac=1") + fetch_data(running_example, "spinlock/block?d=1") + fetch_data(running_example, "spinlock/block?d=1&frac=1") def test_binary_handler(running_example): |