diff options
author | hiddenpath <hiddenpath@yandex-team.com> | 2024-02-21 23:16:42 +0300 |
---|---|---|
committer | hiddenpath <hiddenpath@yandex-team.com> | 2024-02-21 23:33:25 +0300 |
commit | 9052eb5cc304b8da8885fc4e3364ebddc16945f3 (patch) | |
tree | 3c252f6161dd0745c7732d74c9304c000645ab47 /contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends | |
parent | f5eb715f103692e7c7536e13bef3f281fd78e5e7 (diff) | |
download | ydb-9052eb5cc304b8da8885fc4e3364ebddc16945f3.tar.gz |
Update libcxx to llvmorg-17.0.6
Update libcxx to llvmorg-17.0.6
c871ef572c71b4fef22d4a9e65bcebc57e625aea
Diffstat (limited to 'contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends')
13 files changed, 1274 insertions, 0 deletions
diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backend.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backend.h new file mode 100644 index 0000000000..e54f331b94 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backend.h @@ -0,0 +1,59 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H + +#include <__config> + +/* + + // _Functor takes a subrange for [__first, __last) that should be executed in serial + template <class _RandomAccessIterator, class _Functor> + void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func); + + template <class _Iterator, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduction> + _Tp __parallel_transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduction); + + // Cancel the execution of other jobs - they aren't needed anymore + void __cancel_execution(); + + template <class _RandomAccessIterator1, + class _RandomAccessIterator2, + class _RandomAccessIterator3, + class _Compare, + class _LeafMerge> + void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __outit, + _Compare __comp, + _LeafMerge __leaf_merge); + + template <class _RandomAccessIterator, class _Comp, class _LeafSort> + void __parallel_stable_sort(_RandomAccessIterator __first, + _RandomAccessIterator __last, + _Comp __comp, + _LeafSort __leaf_sort); + + TODO: Document the parallel backend +*/ + +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h new file mode 100644 index 0000000000..8fe26797bf --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_ANY_OF_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_ANY_OF_H + +#include <__algorithm/any_of.h> +#include <__algorithm/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__atomic/atomic.h> +#include <__atomic/memory_order.h> +#include <__config> +#include <__functional/operations.h> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/pair.h> +#include <__utility/terminate_on_exception.h> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Index, class _Brick> +_LIBCPP_HIDE_FROM_ABI bool __parallel_or(_Index __first, _Index __last, _Brick __f) { + std::atomic<bool> __found(false); + __par_backend::__parallel_for(__first, __last, [__f, &__found](_Index __i, _Index __j) { + if (!__found.load(std::memory_order_relaxed) && __f(__i, __j)) { + __found.store(true, std::memory_order_relaxed); + __par_backend::__cancel_execution(); + } + }); + return __found; +} + +// TODO: check whether __simd_first() can be used here +template <class _Index, class _DifferenceType, class _Pred> +_LIBCPP_HIDE_FROM_ABI bool __simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept { + _DifferenceType __block_size = 4 < __n ? 4 : __n; + const _Index __last = __first + __n; + while (__last != __first) { + int32_t __flag = 1; + _PSTL_PRAGMA_SIMD_REDUCTION(& : __flag) + for (_DifferenceType __i = 0; __i < __block_size; ++__i) + if (__pred(*(__first + __i))) + __flag = 0; + if (!__flag) + return true; + + __first += __block_size; + if (__last - __first >= __block_size << 1) { + // Double the block _Size. Any unnecessary iterations can be amortized against work done so far. + __block_size <<= 1; + } else { + __block_size = __last - __first; + } + } + return false; +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate> +_LIBCPP_HIDE_FROM_ABI bool +__pstl_any_of(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + return std::__terminate_on_exception([&] { + return std::__parallel_or( + __first, __last, [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + return std::__pstl_any_of<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __pred); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + return std::__simd_or(__first, __last - __first, __pred); + } else { + return std::any_of(__first, __last, __pred); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_ANY_OF_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h new file mode 100644 index 0000000000..ea2210a4a7 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H + +#include <__config> +#include <cstddef> + +#if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) +# include <__algorithm/pstl_backends/cpu_backends/serial.h> +#elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) +# include <__algorithm/pstl_backends/cpu_backends/thread.h> +#elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +# include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> +#else +# error "Invalid CPU backend choice" +#endif + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +struct __cpu_backend_tag {}; + +inline constexpr size_t __lane_size = 64; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h new file mode 100644 index 0000000000..5e5e0a23bf --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FILL_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FILL_H + +#include <__algorithm/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/terminate_on_exception.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Index, class _DifferenceType, class _Tp> +_LIBCPP_HIDE_FROM_ABI _Index __simd_fill_n(_Index __first, _DifferenceType __n, const _Tp& __value) noexcept { + _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED + _PSTL_PRAGMA_SIMD + for (_DifferenceType __i = 0; __i < __n; ++__i) + __first[__i] = __value; + return __first + __n; +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + std::__terminate_on_exception([&] { + __par_backend::__parallel_for( + __first, __last, [&__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + std::__pstl_fill<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __value); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + std::__simd_fill_n(__first, __last - __first, __value); + } else { + std::fill(__first, __last, __value); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FILL_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h new file mode 100644 index 0000000000..3fa49549e6 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FIND_IF_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FIND_IF_H + +#include <__algorithm/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__atomic/atomic.h> +#include <__config> +#include <__functional/operations.h> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/pair.h> +#include <__utility/terminate_on_exception.h> +#include <cstddef> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Index, class _Brick, class _Compare> +_LIBCPP_HIDE_FROM_ABI _Index +__parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool __b_first) { + typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType; + const _DifferenceType __n = __last - __first; + _DifferenceType __initial_dist = __b_first ? __n : -1; + std::atomic<_DifferenceType> __extremum(__initial_dist); + // TODO: find out what is better here: parallel_for or parallel_reduce + __par_backend::__parallel_for(__first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) { + // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of + // why using a shared variable scales fairly well in this situation. + if (__comp(__i - __first, __extremum)) { + _Index __res = __f(__i, __j); + // If not '__last' returned then we found what we want so put this to extremum + if (__res != __j) { + const _DifferenceType __k = __res - __first; + for (_DifferenceType __old = __extremum; __comp(__k, __old); __old = __extremum) { + __extremum.compare_exchange_weak(__old, __k); + } + } + } + }); + return __extremum != __initial_dist ? __first + __extremum : __last; +} + +template <class _Index, class _DifferenceType, class _Compare> +_LIBCPP_HIDE_FROM_ABI _Index +__simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept { + // Experiments show good block sizes like this + const _DifferenceType __block_size = 8; + alignas(__lane_size) _DifferenceType __lane[__block_size] = {0}; + while (__end - __begin >= __block_size) { + _DifferenceType __found = 0; + _PSTL_PRAGMA_SIMD_REDUCTION(| : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size; ++__i) { + const _DifferenceType __t = __comp(__first, __i); + __lane[__i - __begin] = __t; + __found |= __t; + } + if (__found) { + _DifferenceType __i; + // This will vectorize + for (__i = 0; __i < __block_size; ++__i) { + if (__lane[__i]) { + break; + } + } + return __first + __begin + __i; + } + __begin += __block_size; + } + + // Keep remainder scalar + while (__begin != __end) { + if (__comp(__first, __begin)) { + return __first + __begin; + } + ++__begin; + } + return __first + __end; +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate> +_LIBCPP_HIDE_FROM_ABI _ForwardIterator +__pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + return std::__terminate_on_exception([&] { + return std::__parallel_find( + __first, + __last, + [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + return std::__pstl_find_if<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __pred); + }, + less<>{}, + true); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + using __diff_t = __iter_diff_t<_ForwardIterator>; + return std::__simd_first(__first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) { + return __pred(__iter[__i]); + }); + } else { + return std::find_if(__first, __last, __pred); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FIND_IF_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h new file mode 100644 index 0000000000..36d0ac238e --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H + +#include <__algorithm/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/terminate_on_exception.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Iterator, class _DifferenceType, class _Function> +_LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept { + _PSTL_PRAGMA_SIMD + for (_DifferenceType __i = 0; __i < __n; ++__i) + __f(__first[__i]); + + return __first + __n; +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _Functor> +_LIBCPP_HIDE_FROM_ABI void +__pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + std::__terminate_on_exception([&] { + std::__par_backend::__parallel_for( + __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __func); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + std::__simd_walk_1(__first, __last - __first, __func); + } else { + std::for_each(__first, __last, __func); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h new file mode 100644 index 0000000000..bab6a3639b --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h @@ -0,0 +1,241 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H + +#include <__algorithm/lower_bound.h> +#include <__algorithm/max.h> +#include <__algorithm/upper_bound.h> +#include <__atomic/atomic.h> +#include <__config> +#include <__exception/terminate.h> +#include <__iterator/iterator_traits.h> +#include <__iterator/move_iterator.h> +#include <__memory/allocator.h> +#include <__memory/construct_at.h> +#include <__memory/unique_ptr.h> +#include <__numeric/reduce.h> +#include <__utility/exception_guard.h> +#include <__utility/move.h> +#include <__utility/pair.h> +#include <__utility/terminate_on_exception.h> +#include <cstddef> +#include <new> + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend { +inline namespace __libdispatch { + +// ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do +// we. +// TODO: Do we want to add [[_Clang::__callback__(__func, __context, __)]]? +_LIBCPP_EXPORTED_FROM_ABI void +__dispatch_apply(size_t __chunk_count, void* __context, void (*__func)(void* __context, size_t __chunk)) noexcept; + +template <class _Func> +_LIBCPP_HIDE_FROM_ABI void __dispatch_apply(size_t __chunk_count, _Func __func) noexcept { + __libdispatch::__dispatch_apply(__chunk_count, &__func, [](void* __context, size_t __chunk) { + (*static_cast<_Func*>(__context))(__chunk); + }); +} + +struct __chunk_partitions { + ptrdiff_t __chunk_count_; // includes the first chunk + ptrdiff_t __chunk_size_; + ptrdiff_t __first_chunk_size_; +}; + +[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI __chunk_partitions __partition_chunks(ptrdiff_t __size); + +template <class _RandomAccessIterator, class _Functor> +_LIBCPP_HIDE_FROM_ABI void +__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) { + auto __partitions = __libdispatch::__partition_chunks(__last - __first); + + // Perform the chunked execution. + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + __func(__first + __index, __first + __index + __this_chunk_size); + }); +} + +template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIteratorOut> +struct __merge_range { + __merge_range(_RandomAccessIterator1 __mid1, _RandomAccessIterator2 __mid2, _RandomAccessIteratorOut __result) + : __mid1_(__mid1), __mid2_(__mid2), __result_(__result) {} + + _RandomAccessIterator1 __mid1_; + _RandomAccessIterator2 __mid2_; + _RandomAccessIteratorOut __result_; +}; + +template <typename _RandomAccessIterator1, + typename _RandomAccessIterator2, + typename _RandomAccessIterator3, + typename _Compare, + typename _LeafMerge> +_LIBCPP_HIDE_FROM_ABI void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __result, + _Compare __comp, + _LeafMerge __leaf_merge) { + __chunk_partitions __partitions = + __libdispatch::__partition_chunks(std::max<ptrdiff_t>(__last1 - __first1, __last2 - __first2)); + + if (__partitions.__chunk_count_ == 0) + return; + + if (__partitions.__chunk_count_ == 1) { + __leaf_merge(__first1, __last1, __first2, __last2, __result, __comp); + return; + } + + using __merge_range_t = __merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3>; + auto const __n_ranges = __partitions.__chunk_count_ + 1; + + // TODO: use __uninitialized_buffer + auto __destroy = [=](__merge_range_t* __ptr) { + std::destroy_n(__ptr, __n_ranges); + std::allocator<__merge_range_t>().deallocate(__ptr, __n_ranges); + }; + unique_ptr<__merge_range_t[], decltype(__destroy)> __ranges( + std::allocator<__merge_range_t>().allocate(__n_ranges), __destroy); + + // TODO: Improve the case where the smaller range is merged into just a few (or even one) chunks of the larger case + std::__terminate_on_exception([&] { + __merge_range_t* __r = __ranges.get(); + std::__construct_at(__r++, __first1, __first2, __result); + + bool __iterate_first_range = __last1 - __first1 > __last2 - __first2; + + auto __compute_chunk = [&](size_t __chunk_size) -> __merge_range_t { + auto [__mid1, __mid2] = [&] { + if (__iterate_first_range) { + auto __m1 = __first1 + __chunk_size; + auto __m2 = std::lower_bound(__first2, __last2, __m1[-1], __comp); + return std::make_pair(__m1, __m2); + } else { + auto __m2 = __first2 + __chunk_size; + auto __m1 = std::lower_bound(__first1, __last1, __m2[-1], __comp); + return std::make_pair(__m1, __m2); + } + }(); + + __result += (__mid1 - __first1) + (__mid2 - __first2); + __first1 = __mid1; + __first2 = __mid2; + return {std::move(__mid1), std::move(__mid2), __result}; + }; + + // handle first chunk + std::__construct_at(__r++, __compute_chunk(__partitions.__first_chunk_size_)); + + // handle 2 -> N - 1 chunks + for (ptrdiff_t __i = 0; __i != __partitions.__chunk_count_ - 2; ++__i) + std::__construct_at(__r++, __compute_chunk(__partitions.__chunk_size_)); + + // handle last chunk + std::__construct_at(__r, __last1, __last2, __result); + + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __index) { + auto __first_iters = __ranges[__index]; + auto __last_iters = __ranges[__index + 1]; + __leaf_merge( + __first_iters.__mid1_, + __last_iters.__mid1_, + __first_iters.__mid2_, + __last_iters.__mid2_, + __first_iters.__result_, + __comp); + }); + }); +} + +template <class _RandomAccessIterator, class _Transform, class _Value, class _Combiner, class _Reduction> +_LIBCPP_HIDE_FROM_ABI _Value __parallel_transform_reduce( + _RandomAccessIterator __first, + _RandomAccessIterator __last, + _Transform __transform, + _Value __init, + _Combiner __combiner, + _Reduction __reduction) { + if (__first == __last) + return __init; + + auto __partitions = __libdispatch::__partition_chunks(__last - __first); + + auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) { + std::destroy_n(__ptr, __count); + std::allocator<_Value>().deallocate(__ptr, __count); + }; + + // TODO: use __uninitialized_buffer + // TODO: allocate one element per worker instead of one element per chunk + unique_ptr<_Value[], decltype(__destroy)> __values( + std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy); + + // __dispatch_apply is noexcept + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + if (__this_chunk_size != 1) { + std::__construct_at( + __values.get() + __chunk, + __reduction(__first + __index + 2, + __first + __index + __this_chunk_size, + __combiner(__transform(__first + __index), __transform(__first + __index + 1)))); + } else { + std::__construct_at(__values.get() + __chunk, __transform(__first + __index)); + } + }); + + return std::__terminate_on_exception([&] { + return std::reduce( + std::make_move_iterator(__values.get()), + std::make_move_iterator(__values.get() + __partitions.__chunk_count_), + std::move(__init), + __combiner); + }); +} + +// TODO: parallelize this +template <class _RandomAccessIterator, class _Comp, class _LeafSort> +_LIBCPP_HIDE_FROM_ABI void __parallel_stable_sort( + _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) { + __leaf_sort(__first, __last, __comp); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} + +} // namespace __libdispatch +} // namespace __par_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h new file mode 100644 index 0000000000..d5be1e302d --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_MERGE_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_MERGE_H + +#include <__algorithm/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/move.h> +#include <__utility/terminate_on_exception.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _ExecutionPolicy, + class _ForwardIterator1, + class _ForwardIterator2, + class _ForwardOutIterator, + class _Comp> +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_merge( + __cpu_backend_tag, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _ForwardOutIterator __result, + _Comp __comp) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value && + __has_random_access_iterator_category<_ForwardOutIterator>::value) { + return std::__terminate_on_exception([&] { + __par_backend::__parallel_merge( + __first1, + __last1, + __first2, + __last2, + __result, + __comp, + [](_ForwardIterator1 __g_first1, + _ForwardIterator1 __g_last1, + _ForwardIterator2 __g_first2, + _ForwardIterator2 __g_last2, + _ForwardOutIterator __g_result, + _Comp __g_comp) { + return std::__pstl_merge<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, + std::move(__g_first1), + std::move(__g_last1), + std::move(__g_first2), + std::move(__g_last2), + std::move(__g_result), + std::move(__g_comp)); + }); + return __result + (__last1 - __first1) + (__last2 - __first2); + }); + } else { + return std::merge(__first1, __last1, __first2, __last2, __result, __comp); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_MERGE_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h new file mode 100644 index 0000000000..f151c3b098 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h @@ -0,0 +1,72 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H + +#include <__config> +#include <__utility/move.h> +#include <cstddef> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend { +inline namespace __serial_cpu_backend { + +template <class _RandomAccessIterator, class _Fp> +_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) { + __f(__first, __last); +} + +template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce> +_LIBCPP_HIDE_FROM_ABI _Tp +__parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) { + return __reduce(std::move(__first), std::move(__last), std::move(__init)); +} + +template <class _RandomAccessIterator, class _Compare, class _LeafSort> +_LIBCPP_HIDE_FROM_ABI void __parallel_stable_sort( + _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) { + __leaf_sort(__first, __last, __comp); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} + +template <class _RandomAccessIterator1, + class _RandomAccessIterator2, + class _RandomAccessIterator3, + class _Compare, + class _LeafMerge> +_LIBCPP_HIDE_FROM_ABI void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __outit, + _Compare __comp, + _LeafMerge __leaf_merge) { + __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp); +} + +// TODO: Complete this list + +} // namespace __serial_cpu_backend +} // namespace __par_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h new file mode 100644 index 0000000000..0a701443b3 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H + +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__algorithm/stable_sort.h> +#include <__config> +#include <__type_traits/is_execution_policy.h> +#include <__utility/terminate_on_exception.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _ExecutionPolicy, class _RandomAccessIterator, class _Comp> +_LIBCPP_HIDE_FROM_ABI void +__pstl_stable_sort(__cpu_backend_tag, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy>) { + std::__terminate_on_exception([&] { + __par_backend::__parallel_stable_sort( + __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) { + std::stable_sort(__g_first, __g_last, __g_comp); + }); + }); + } else { + std::stable_sort(__first, __last, __comp); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h new file mode 100644 index 0000000000..30eb0ae362 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h @@ -0,0 +1,78 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H + +#include <__assert> +#include <__config> +#include <__utility/move.h> +#include <cstddef> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +// This backend implementation is for testing purposes only and not meant for production use. This will be replaced +// by a proper implementation once the PSTL implementation is somewhat stable. + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend { +inline namespace __thread_cpu_backend { + +template <class _RandomAccessIterator, class _Fp> +_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) { + __f(__first, __last); +} + +template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce> +_LIBCPP_HIDE_FROM_ABI _Tp +__parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) { + return __reduce(std::move(__first), std::move(__last), std::move(__init)); +} + +template <class _RandomAccessIterator, class _Compare, class _LeafSort> +_LIBCPP_HIDE_FROM_ABI void __parallel_stable_sort( + _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) { + __leaf_sort(__first, __last, __comp); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} + +template <class _RandomAccessIterator1, + class _RandomAccessIterator2, + class _RandomAccessIterator3, + class _Compare, + class _LeafMerge> +_LIBCPP_HIDE_FROM_ABI void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __outit, + _Compare __comp, + _LeafMerge __leaf_merge) { + __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp); +} + +} // namespace __thread_cpu_backend +} // namespace __par_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h new file mode 100644 index 0000000000..ef25ff0238 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h @@ -0,0 +1,132 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H + +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__algorithm/transform.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/is_execution_policy.h> +#include <__type_traits/remove_cvref.h> +#include <__utility/terminate_on_exception.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function> +_LIBCPP_HIDE_FROM_ABI _Iterator2 +__simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept { + _PSTL_PRAGMA_SIMD + for (_DifferenceType __i = 0; __i < __n; ++__i) + __f(__first1[__i], __first2[__i]); + return __first2 + __n; +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _ForwardOutIterator, class _UnaryOperation> +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( + __cpu_backend_tag, + _ForwardIterator __first, + _ForwardIterator __last, + _ForwardOutIterator __result, + _UnaryOperation __op) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value && + __has_random_access_iterator_category<_ForwardOutIterator>::value) { + std::__terminate_on_exception([&] { + std::__par_backend::__parallel_for( + __first, __last, [__op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + return std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __result + (__brick_first - __first), __op); + }); + }); + return __result + (__last - __first); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value && + __has_random_access_iterator_category<_ForwardOutIterator>::value) { + return std::__simd_walk_2( + __first, + __last - __first, + __result, + [&](__iter_reference<_ForwardIterator> __in_value, __iter_reference<_ForwardOutIterator> __out_value) { + __out_value = __op(__in_value); + }); + } else { + return std::transform(__first, __last, __result, __op); + } +} + +template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function> +_LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_walk_3( + _Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3, _Function __f) noexcept { + _PSTL_PRAGMA_SIMD + for (_DifferenceType __i = 0; __i < __n; ++__i) + __f(__first1[__i], __first2[__i], __first3[__i]); + return __first3 + __n; +} +template <class _ExecutionPolicy, + class _ForwardIterator1, + class _ForwardIterator2, + class _ForwardOutIterator, + class _BinaryOperation, + enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0> +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator __pstl_transform( + __cpu_backend_tag, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardOutIterator __result, + _BinaryOperation __op) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value && + __has_random_access_iterator_category<_ForwardOutIterator>::value) { + std::__terminate_on_exception([&] { + std::__par_backend::__parallel_for( + __first1, + __last1, + [__op, __first1, __first2, __result](_ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) { + return std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, + __brick_first, + __brick_last, + __first2 + (__brick_first - __first1), + __result + (__brick_first - __first1), + __op); + }); + }); + return __result + (__last1 - __first1); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value && + __has_random_access_iterator_category<_ForwardOutIterator>::value) { + return std::__simd_walk_3( + __first1, + __last1 - __first1, + __first2, + __result, + [&](__iter_reference<_ForwardIterator1> __in1, + __iter_reference<_ForwardIterator2> __in2, + __iter_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); }); + } else { + return std::transform(__first1, __last1, __first2, __result, __op); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H diff --git a/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h new file mode 100644 index 0000000000..c51c312d93 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -0,0 +1,194 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H + +#include <__algorithm/pstl_backends/cpu_backends/backend.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__numeric/transform_reduce.h> +#include <__type_traits/is_arithmetic.h> +#include <__type_traits/is_execution_policy.h> +#include <__type_traits/operation_traits.h> +#include <__utility/move.h> +#include <__utility/terminate_on_exception.h> +#include <new> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +template < + typename _DifferenceType, + typename _Tp, + typename _BinaryOperation, + typename _UnaryOperation, + __enable_if_t<__is_trivial_plus_operation<_BinaryOperation, _Tp, _Tp>::value && is_arithmetic_v<_Tp>, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +__simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _UnaryOperation __f) noexcept { + _PSTL_PRAGMA_SIMD_REDUCTION(+ : __init) + for (_DifferenceType __i = 0; __i < __n; ++__i) + __init += __f(__i); + return __init; +} + +template < + typename _Size, + typename _Tp, + typename _BinaryOperation, + typename _UnaryOperation, + __enable_if_t<!(__is_trivial_plus_operation<_BinaryOperation, _Tp, _Tp>::value && is_arithmetic_v<_Tp>), int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +__simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept { + const _Size __block_size = __lane_size / sizeof(_Tp); + if (__n > 2 * __block_size && __block_size > 1) { + alignas(__lane_size) char __lane_buffer[__lane_size]; + _Tp* __lane = reinterpret_cast<_Tp*>(__lane_buffer); + + // initializer + _PSTL_PRAGMA_SIMD + for (_Size __i = 0; __i < __block_size; ++__i) { + ::new (__lane + __i) _Tp(__binary_op(__f(__i), __f(__block_size + __i))); + } + // main loop + _Size __i = 2 * __block_size; + const _Size __last_iteration = __block_size * (__n / __block_size); + for (; __i < __last_iteration; __i += __block_size) { + _PSTL_PRAGMA_SIMD + for (_Size __j = 0; __j < __block_size; ++__j) { + __lane[__j] = __binary_op(std::move(__lane[__j]), __f(__i + __j)); + } + } + // remainder + _PSTL_PRAGMA_SIMD + for (_Size __j = 0; __j < __n - __last_iteration; ++__j) { + __lane[__j] = __binary_op(std::move(__lane[__j]), __f(__last_iteration + __j)); + } + // combiner + for (_Size __j = 0; __j < __block_size; ++__j) { + __init = __binary_op(std::move(__init), std::move(__lane[__j])); + } + // destroyer + _PSTL_PRAGMA_SIMD + for (_Size __j = 0; __j < __block_size; ++__j) { + __lane[__j].~_Tp(); + } + } else { + for (_Size __i = 0; __i < __n; ++__i) { + __init = __binary_op(std::move(__init), __f(__i)); + } + } + return __init; +} + +template <class _ExecutionPolicy, + class _ForwardIterator1, + class _ForwardIterator2, + class _Tp, + class _BinaryOperation1, + class _BinaryOperation2> +_LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( + __cpu_backend_tag, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _Tp __init, + _BinaryOperation1 __reduce, + _BinaryOperation2 __transform) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value) { + return std::__terminate_on_exception([&] { + return __par_backend::__parallel_transform_reduce( + __first1, + std::move(__last1), + [__first1, __first2, __transform](_ForwardIterator1 __iter) { + return __transform(*__iter, *(__first2 + (__iter - __first1))); + }, + std::move(__init), + std::move(__reduce), + [__first1, __first2, __reduce, __transform]( + _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last, _Tp __brick_init) { + return std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, + __brick_first, + std::move(__brick_last), + __first2 + (__brick_first - __first1), + std::move(__brick_init), + std::move(__reduce), + std::move(__transform)); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value) { + return std::__simd_transform_reduce( + __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) { + return __transform(__first1[__i], __first2[__i]); + }); + } else { + return std::transform_reduce( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__init), + std::move(__reduce), + std::move(__transform)); + } +} + +template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation> +_LIBCPP_HIDE_FROM_ABI _Tp __pstl_transform_reduce( + __cpu_backend_tag, + _ForwardIterator __first, + _ForwardIterator __last, + _Tp __init, + _BinaryOperation __reduce, + _UnaryOperation __transform) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + return std::__terminate_on_exception([&] { + return __par_backend::__parallel_transform_reduce( + std::move(__first), + std::move(__last), + [__transform](_ForwardIterator __iter) { return __transform(*__iter); }, + std::move(__init), + __reduce, + [__transform, __reduce](auto __brick_first, auto __brick_last, _Tp __brick_init) { + return std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, + std::move(__brick_first), + std::move(__brick_last), + std::move(__brick_init), + std::move(__reduce), + std::move(__transform)); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __has_random_access_iterator_category<_ForwardIterator>::value) { + return std::__simd_transform_reduce( + __last - __first, + std::move(__init), + std::move(__reduce), + [=, &__transform](__iter_diff_t<_ForwardIterator> __i) { return __transform(__first[__i]); }); + } else { + return std::transform_reduce( + std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform)); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H |