diff options
author | mikhnenko <mikhnenko@yandex-team.com> | 2024-03-27 10:35:27 +0300 |
---|---|---|
committer | mikhnenko <mikhnenko@yandex-team.com> | 2024-03-27 10:47:39 +0300 |
commit | 9b902baa4a858f2176c82aa0b20f88232f0da0d8 (patch) | |
tree | 7165a551c2244c4b3c28479ac3a3f6d62346ec89 /contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends | |
parent | a1c989e67e438005fa0c34ed0e910536c8941862 (diff) | |
download | ydb-9b902baa4a858f2176c82aa0b20f88232f0da0d8.tar.gz |
Update libcxx to 10 Oct 2023 dc129d6f715cf83a2072fc8de8b4e4c70bca6935
97ce40d276e44357a49b7a945af841896126dca8
Diffstat (limited to 'contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends')
8 files changed, 128 insertions, 39 deletions
diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h index 8fe26797bf..c8a071af82 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h @@ -16,7 +16,7 @@ #include <__atomic/memory_order.h> #include <__config> #include <__functional/operations.h> -#include <__iterator/iterator_traits.h> +#include <__iterator/concepts.h> #include <__type_traits/is_execution_policy.h> #include <__utility/pair.h> #include <__utility/terminate_on_exception.h> @@ -67,7 +67,7 @@ template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate> _LIBCPP_HIDE_FROM_ABI bool __pstl_any_of(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { return std::__terminate_on_exception([&] { return std::__parallel_or( __first, __last, [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { @@ -76,7 +76,7 @@ __pstl_any_of(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __la }); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { return std::__simd_or(__first, __last - __first, __pred); } else { return std::any_of(__first, __last, __pred); diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h index 5e5e0a23bf..8b531887c7 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h @@ -12,7 +12,7 @@ #include <__algorithm/fill.h> #include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__iterator/concepts.h> #include <__type_traits/is_execution_policy.h> #include <__utility/terminate_on_exception.h> @@ -37,7 +37,7 @@ template <class _ExecutionPolicy, class _ForwardIterator, class _Tp> _LIBCPP_HIDE_FROM_ABI void __pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { std::__terminate_on_exception([&] { __par_backend::__parallel_for( __first, __last, [&__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { @@ -46,7 +46,7 @@ __pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last }); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { std::__simd_fill_n(__first, __last - __first, __value); } else { std::fill(__first, __last, __value); diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h index 3fa49549e6..91610c0408 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h @@ -14,6 +14,7 @@ #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/pair.h> @@ -93,7 +94,7 @@ template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate> _LIBCPP_HIDE_FROM_ABI _ForwardIterator __pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { return std::__terminate_on_exception([&] { return std::__parallel_find( __first, @@ -106,7 +107,7 @@ __pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __l true); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { using __diff_t = __iter_diff_t<_ForwardIterator>; return std::__simd_first(__first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) { return __pred(__iter[__i]); diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h index 36d0ac238e..f6f22fdd87 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h @@ -12,7 +12,7 @@ #include <__algorithm/for_each.h> #include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__iterator/concepts.h> #include <__type_traits/is_execution_policy.h> #include <__utility/terminate_on_exception.h> @@ -37,7 +37,7 @@ template <class _ExecutionPolicy, class _ForwardIterator, class _Functor> _LIBCPP_HIDE_FROM_ABI void __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { std::__terminate_on_exception([&] { std::__par_backend::__parallel_for( __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { @@ -46,7 +46,7 @@ __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __ }); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { std::__simd_walk_1(__first, __last - __first, __func); } else { std::for_each(__first, __last, __func); diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h index bab6a3639b..50b6e0b1d0 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h @@ -9,8 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#include <__algorithm/inplace_merge.h> #include <__algorithm/lower_bound.h> #include <__algorithm/max.h> +#include <__algorithm/merge.h> #include <__algorithm/upper_bound.h> #include <__atomic/atomic.h> #include <__config> @@ -21,7 +23,6 @@ #include <__memory/construct_at.h> #include <__memory/unique_ptr.h> #include <__numeric/reduce.h> -#include <__utility/exception_guard.h> #include <__utility/move.h> #include <__utility/pair.h> #include <__utility/terminate_on_exception.h> @@ -57,14 +58,11 @@ struct __chunk_partitions { ptrdiff_t __first_chunk_size_; }; -[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI __chunk_partitions __partition_chunks(ptrdiff_t __size); +[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI __chunk_partitions __partition_chunks(ptrdiff_t __size) noexcept; template <class _RandomAccessIterator, class _Functor> _LIBCPP_HIDE_FROM_ABI void -__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) { - auto __partitions = __libdispatch::__partition_chunks(__last - __first); - - // Perform the chunked execution. +__dispatch_parallel_for(__chunk_partitions __partitions, _RandomAccessIterator __first, _Functor __func) { __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; auto __index = @@ -75,6 +73,13 @@ __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fun }); } +template <class _RandomAccessIterator, class _Functor> +_LIBCPP_HIDE_FROM_ABI void +__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) { + return __libdispatch::__dispatch_parallel_for( + __libdispatch::__partition_chunks(__last - __first), std::move(__first), std::move(__func)); +} + template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIteratorOut> struct __merge_range { __merge_range(_RandomAccessIterator1 __mid1, _RandomAccessIterator2 __mid2, _RandomAccessIteratorOut __result) @@ -220,11 +225,92 @@ _LIBCPP_HIDE_FROM_ABI _Value __parallel_transform_reduce( }); } -// TODO: parallelize this template <class _RandomAccessIterator, class _Comp, class _LeafSort> _LIBCPP_HIDE_FROM_ABI void __parallel_stable_sort( _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) { - __leaf_sort(__first, __last, __comp); + const auto __size = __last - __first; + auto __partitions = __libdispatch::__partition_chunks(__size); + + if (__partitions.__chunk_count_ == 0) + return; + + if (__partitions.__chunk_count_ == 1) + return __leaf_sort(__first, __last, __comp); + + using _Value = __iter_value_type<_RandomAccessIterator>; + + auto __destroy = [__size](_Value* __ptr) { + std::destroy_n(__ptr, __size); + std::allocator<_Value>().deallocate(__ptr, __size); + }; + + // TODO: use __uninitialized_buffer + unique_ptr<_Value[], decltype(__destroy)> __values(std::allocator<_Value>().allocate(__size), __destroy); + + return std::__terminate_on_exception([&] { + // Initialize all elements to a moved-from state + // TODO: Don't do this - this can be done in the first merge - see https://llvm.org/PR63928 + std::__construct_at(__values.get(), std::move(*__first)); + for (__iter_diff_t<_RandomAccessIterator> __i = 1; __i != __size; ++__i) { + std::__construct_at(__values.get() + __i, std::move(__values.get()[__i - 1])); + } + *__first = std::move(__values.get()[__size - 1]); + + __libdispatch::__dispatch_parallel_for( + __partitions, + __first, + [&__leaf_sort, &__comp](_RandomAccessIterator __chunk_first, _RandomAccessIterator __chunk_last) { + __leaf_sort(std::move(__chunk_first), std::move(__chunk_last), __comp); + }); + + bool __objects_are_in_buffer = false; + do { + const auto __old_chunk_size = __partitions.__chunk_size_; + if (__partitions.__chunk_count_ % 2 == 1) { + auto __inplace_merge_chunks = [&__comp, &__partitions](auto __first_chunk_begin) { + std::inplace_merge( + __first_chunk_begin, + __first_chunk_begin + __partitions.__first_chunk_size_, + __first_chunk_begin + __partitions.__first_chunk_size_ + __partitions.__chunk_size_, + __comp); + }; + if (__objects_are_in_buffer) + __inplace_merge_chunks(__values.get()); + else + __inplace_merge_chunks(__first); + __partitions.__first_chunk_size_ += 2 * __partitions.__chunk_size_; + } else { + __partitions.__first_chunk_size_ += __partitions.__chunk_size_; + } + + __partitions.__chunk_size_ *= 2; + __partitions.__chunk_count_ /= 2; + + auto __merge_chunks = [__partitions, __old_chunk_size, &__comp](auto __from_first, auto __to_first) { + __libdispatch::__dispatch_parallel_for( + __partitions, + __from_first, + [__old_chunk_size, &__from_first, &__to_first, &__comp](auto __chunk_first, auto __chunk_last) { + std::merge(std::make_move_iterator(__chunk_first), + std::make_move_iterator(__chunk_last - __old_chunk_size), + std::make_move_iterator(__chunk_last - __old_chunk_size), + std::make_move_iterator(__chunk_last), + __to_first + (__chunk_first - __from_first), + __comp); + }); + }; + + if (__objects_are_in_buffer) + __merge_chunks(__values.get(), __first); + else + __merge_chunks(__first, __values.get()); + __objects_are_in_buffer = !__objects_are_in_buffer; + } while (__partitions.__chunk_count_ > 1); + + if (__objects_are_in_buffer) { + std::move(__values.get(), __values.get() + __size, __first); + } + }); } _LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h index d5be1e302d..c4b28e9502 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h @@ -12,7 +12,7 @@ #include <__algorithm/merge.h> #include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__iterator/concepts.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> #include <__utility/terminate_on_exception.h> @@ -39,9 +39,9 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_merge( _ForwardOutIterator __result, _Comp __comp) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value && - __has_random_access_iterator_category<_ForwardOutIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { return std::__terminate_on_exception([&] { __par_backend::__parallel_merge( __first1, diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h index ef25ff0238..0259d8a84b 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__algorithm/transform.h> #include <__config> +#include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> @@ -43,8 +44,8 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( _ForwardOutIterator __result, _UnaryOperation __op) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value && - __has_random_access_iterator_category<_ForwardOutIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { std::__terminate_on_exception([&] { std::__par_backend::__parallel_for( __first, __last, [__op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { @@ -54,8 +55,8 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( }); return __result + (__last - __first); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value && - __has_random_access_iterator_category<_ForwardOutIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { return std::__simd_walk_2( __first, __last - __first, @@ -90,9 +91,9 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( _ForwardOutIterator __result, _BinaryOperation __op) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value && - __has_random_access_iterator_category<_ForwardOutIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { std::__terminate_on_exception([&] { std::__par_backend::__parallel_for( __first1, @@ -109,9 +110,9 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( }); return __result + (__last1 - __first1); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value && - __has_random_access_iterator_category<_ForwardOutIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { return std::__simd_walk_3( __first1, __last1 - __first1, diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h index c51c312d93..2afe5c7d10 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -11,6 +11,7 @@ #include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> +#include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/transform_reduce.h> #include <__type_traits/is_arithmetic.h> @@ -106,8 +107,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( _BinaryOperation1 __reduce, _BinaryOperation2 __transform) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { return std::__terminate_on_exception([&] { return __par_backend::__parallel_transform_reduce( __first1, @@ -130,8 +131,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( }); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { return std::__simd_transform_reduce( __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) { return __transform(__first1[__i], __first2[__i]); @@ -156,7 +157,7 @@ _LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( _BinaryOperation __reduce, _UnaryOperation __transform) { if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { return std::__terminate_on_exception([&] { return __par_backend::__parallel_transform_reduce( std::move(__first), @@ -175,7 +176,7 @@ _LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( }); }); } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category<_ForwardIterator>::value) { + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { return std::__simd_transform_reduce( __last - __first, std::move(__init), |