Skip to content

[libc++] Optimize {std,ranges}::for_each for iterating over __trees #164405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[libc++] Optimize {std,ranges}::for_each for iterating over __trees #164405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion libcxx/include/__algorithm/for_each.h
View file
Open in desktop
Copy link
Member

@ldionne ldionne Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the commit message, let's explain the gist of what this optimization does.

Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#define _LIBCPP___ALGORITHM_FOR_EACH_H

#include <__algorithm/for_each_segment.h>
#include <__algorithm/specialized_algorithms.h>
#include <__config>
#include <__functional/identity.h>
#include <__iterator/segmented_iterator.h>
Expand All @@ -27,7 +28,12 @@ template <class _InputIterator, class _Sent, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
__for_each(_InputIterator __first, _Sent __last, _Func& __func, _Proj& __proj) {
#ifndef _LIBCPP_CXX03_LANG
if constexpr (is_same<_InputIterator, _Sent>::value && __is_segmented_iterator_v<_InputIterator>) {
if constexpr (using _SpecialAlg =
__specialized_algorithm<_Algorithm::__for_each, __iterator_pair<_InputIterator, _Sent>>;
_SpecialAlg::__has_algorithm) {
_SpecialAlg()(__first, __last, __func, __proj);
return __last;
} else if constexpr (is_same<_InputIterator, _Sent>::value && __is_segmented_iterator_v<_InputIterator>) {
using __local_iterator_t = typename __segmented_iterator_traits<_InputIterator>::__local_iterator;
std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
std::__for_each(__lfirst, __llast, __func, __proj);
Expand Down
10 changes: 9 additions & 1 deletion libcxx/include/__algorithm/ranges_for_each.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <__algorithm/for_each.h>
#include <__algorithm/for_each_n.h>
#include <__algorithm/in_fun_result.h>
#include <__algorithm/specialized_algorithms.h>
#include <__concepts/assignable.h>
#include <__config>
#include <__functional/identity.h>
Expand All @@ -20,6 +21,7 @@
#include <__ranges/access.h>
#include <__ranges/concepts.h>
#include <__ranges/dangling.h>
#include <__type_traits/remove_cvref.h>
#include <__utility/move.h>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
Expand Down Expand Up @@ -71,7 +73,13 @@ struct __for_each {
indirectly_unary_invocable<projected<iterator_t<_Range>, _Proj>> _Func>
_LIBCPP_HIDE_FROM_ABI constexpr for_each_result<borrowed_iterator_t<_Range>, _Func>
operator()(_Range&& __range, _Func __func, _Proj __proj = {}) const {
return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
using _SpecialAlg = __specialized_algorithm<_Algorithm::__for_each, __single_range<remove_cvref_t<_Range>>>;
if constexpr (_SpecialAlg::__has_algorithm) {
auto [__iter, __func2] = _SpecialAlg()(__range, std::move(__func), std::move(__proj));
return {std::move(__iter), std::move(__func)};
} else {
return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
}
}
};

Expand Down
7 changes: 7 additions & 0 deletions libcxx/include/__algorithm/specialized_algorithms.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD

namespace _Algorithm {
struct __fill_n {};
struct __for_each {};
} // namespace _Algorithm

template <class>
struct __single_iterator;

template <class, class>
struct __iterator_pair;

template <class>
struct __single_range;

// This struct allows specializing algorithms for specific arguments. This is useful when we know a more efficient
// algorithm implementation for e.g. library-defined iterators. _Alg is one of tags defined inside the _Algorithm
// namespace above. _Ranges is an essentially arbitrary subset of the arguments to the algorithm that are used for
Expand Down
106 changes: 106 additions & 0 deletions libcxx/include/__tree
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#define _LIBCPP___TREE

#include <__algorithm/min.h>
#include <__algorithm/specialized_algorithms.h>
#include <__assert>
#include <__config>
#include <__fwd/pair.h>
Expand All @@ -36,6 +37,7 @@
#include <__type_traits/is_swappable.h>
#include <__type_traits/make_transparent.h>
#include <__type_traits/remove_const.h>
#include <__type_traits/remove_cvref.h>
#include <__utility/forward.h>
#include <__utility/lazy_synth_three_way_comparator.h>
#include <__utility/move.h>
Expand Down Expand Up @@ -656,6 +658,54 @@ struct __generic_container_node_destructor<__tree_node<_Tp, _VoidPtr>, _Alloc> :
};
#endif

// Do an in-order traversal of the tree until `__break` returns true. Takes the root node of the tree.
template <class _Reference, class _Break, class _NodePtr, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI bool __tree_iterate_from_root(_Break __break, _NodePtr __root, _Func& __func, _Proj& __proj) {
if (__root->__left_) {
if (std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__left_), __func, __proj))
return true;
}
if (__break(__root))
return true;
__func(static_cast<_Reference>(__root->__get_value()));
if (__root->__right_)
return std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__right_), __func, __proj);
return false;
}

// Do an in-order traversal of the tree from __first to __last.
template <class _NodeIter, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI void
__tree_iterate_subrange(_NodeIter __first_it, _NodeIter __last_it, _Func& __func, _Proj& __proj) {
using _NodePtr = _NodeIter::__node_pointer;
using _Reference = _NodeIter::reference;

auto __first = __first_it.__ptr_;
auto __last = __last_it.__ptr_;

while (true) {
if (__first == __last)
return;
auto __nfirst = static_cast<_NodePtr>(__first);
__func(static_cast<_Reference>(__nfirst->__get_value()));
if (__nfirst->__right_) {
if (std::__tree_iterate_from_root<_Reference>(
[&](_NodePtr __node) -> bool { return __node == __last; },
static_cast<_NodePtr>(__nfirst->__right_),
__func,
__proj))
return;
}
if (std::__tree_is_left_child(__nfirst)) {
__first = __nfirst->__parent_;
} else {
do {
__first = __nfirst->__parent_;
} while (!std::__tree_is_left_child(__nfirst));
}
}
}

template <class _Tp, class _NodePtr, class _DiffType>
class __tree_iterator {
using _NodeTypes _LIBCPP_NODEBUG = __tree_node_types<_NodePtr>;
Expand Down Expand Up @@ -715,8 +765,27 @@ private:
friend class __tree;
template <class, class, class>
friend class __tree_const_iterator;

template <class _NodeIter, class _Func, class _Proj>
friend void __tree_iterate_subrange(_NodeIter, _NodeIter, _Func&, _Proj&);
};

#ifndef _LIBCPP_CXX03_LANG
template <class _Tp, class _NodePtr, class _DiffType>
Copy link
Member

@ldionne ldionne Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to add a comment explaining that this handles std::set::iterator and std::multiset::iterator in addition to __tree::iterator, since that's not obvious.

Same below for const_iterator.

struct __specialized_algorithm<
_Algorithm::__for_each,
__iterator_pair<__tree_iterator<_Tp, _NodePtr, _DiffType>, __tree_iterator<_Tp, _NodePtr, _DiffType>>> {
static const bool __has_algorithm = true;

using __iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, _NodePtr, _DiffType>;

template <class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
std::__tree_iterate_subrange(__first, __last, __func, __proj);
}
};
#endif

template <class _Tp, class _NodePtr, class _DiffType>
class __tree_const_iterator {
using _NodeTypes _LIBCPP_NODEBUG = __tree_node_types<_NodePtr>;
Expand Down Expand Up @@ -780,8 +849,27 @@ private:

template <class, class, class>
friend class __tree;

template <class _NodeIter, class _Func, class _Proj>
friend void __tree_iterate_subrange(_NodeIter, _NodeIter, _Func&, _Proj&);
};

#ifndef _LIBCPP_CXX03_LANG
template <class _Tp, class _NodePtr, class _DiffType>
struct __specialized_algorithm<
_Algorithm::__for_each,
__iterator_pair<__tree_const_iterator<_Tp, _NodePtr, _DiffType>, __tree_const_iterator<_Tp, _NodePtr, _DiffType>>> {
static const bool __has_algorithm = true;

using __iterator _LIBCPP_NODEBUG = __tree_const_iterator<_Tp, _NodePtr, _DiffType>;

template <class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
std::__tree_iterate_subrange(__first, __last, __func, __proj);
}
};
#endif

template <class _Tp, class _Compare>
#ifndef _LIBCPP_CXX03_LANG
_LIBCPP_DIAGNOSE_WARNING(!__is_invocable_v<_Compare const&, _Tp const&, _Tp const&>,
Expand Down Expand Up @@ -1484,7 +1572,25 @@ private:
[](value_type& __lhs, value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
[this](__node_pointer __nd) { return __move_construct_tree(__nd); });
}

friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<__tree> >;
};

#if _LIBCPP_STD_VER >= 14
template <class _Tp, class _Compare, class _Allocator>
struct __specialized_algorithm<_Algorithm::__for_each, __single_range<__tree<_Tp, _Compare, _Allocator> > > {
static const bool __has_algorithm = true;

using __node_pointer _LIBCPP_NODEBUG = typename __tree<_Tp, _Compare, _Allocator>::__node_pointer;

template <class _Tree, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Tree&& __range, _Func __func, _Proj __proj) {
std::__tree_iterate_from_root<__copy_cvref_t<_Tree, typename __remove_cvref_t<_Tree>::value_type>>(
[](__node_pointer) { return false; }, __range.__root(), __func, __proj);
return std::make_pair(__range.end(), std::move(__func));
}
};
#endif

// Precondition: __size_ != 0
template <class _Tp, class _Compare, class _Allocator>
Expand Down
37 changes: 37 additions & 0 deletions libcxx/include/map
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred); // C++20
# include <__algorithm/equal.h>
# include <__algorithm/lexicographical_compare.h>
# include <__algorithm/lexicographical_compare_three_way.h>
# include <__algorithm/specialized_algorithms.h>
# include <__assert>
# include <__config>
# include <__functional/binary_function.h>
Expand Down Expand Up @@ -1370,6 +1371,8 @@ private:
# ifdef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
# endif

friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<map> >;
};

# if _LIBCPP_STD_VER >= 17
Expand Down Expand Up @@ -1422,6 +1425,22 @@ map(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
# endif

# if _LIBCPP_STD_VER >= 14
template <class _Key, class _Tp, class _Compare, class _Allocator>
struct __specialized_algorithm<_Algorithm::__for_each, __single_range<map<_Key, _Tp, _Compare, _Allocator>>> {
Copy link
Member

@ldionne ldionne Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you also intend to have a specialization for the two-iterator version for map and multimap?

using __map _LIBCPP_NODEBUG = map<_Key, _Tp, _Compare, _Allocator>;

static const bool __has_algorithm = true;

template <class _Map, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func, _Proj __proj) {
auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, __single_range<typename __map::__base>>()(
__map.__tree_, std::move(__func), std::move(__proj));
return std::make_pair(__map.end(), std::move(__func2));
}
};
# endif

# ifndef _LIBCPP_CXX03_LANG
template <class _Key, class _Tp, class _Compare, class _Allocator>
_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
Expand Down Expand Up @@ -1920,6 +1939,8 @@ private:

typedef __map_node_destructor<__node_allocator> _Dp;
typedef unique_ptr<__node, _Dp> __node_holder;

friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<multimap> >;
};

# if _LIBCPP_STD_VER >= 17
Expand Down Expand Up @@ -1972,6 +1993,22 @@ multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
# endif

# if _LIBCPP_STD_VER >= 14
template <class _Key, class _Tp, class _Compare, class _Allocator>
struct __specialized_algorithm<_Algorithm::__for_each, __single_range<multimap<_Key, _Tp, _Compare, _Allocator>>> {
using __map _LIBCPP_NODEBUG = multimap<_Key, _Tp, _Compare, _Allocator>;

static const bool __has_algorithm = true;

template <class _Map, class _Func, class _Proj>
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func, _Proj __proj) {
auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, __single_range<typename __map::__base>>()(
__map.__tree_, std::move(__func), std::move(__proj));
return std::make_pair(__map.end(), std::move(__func2));
}
};
# endif

template <class _Key, class _Tp, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
Expand Down
41 changes: 41 additions & 0 deletions libcxx/include/set
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred); // C++20
# include <__algorithm/equal.h>
# include <__algorithm/lexicographical_compare.h>
# include <__algorithm/lexicographical_compare_three_way.h>
# include <__algorithm/specialized_algorithms.h>
# include <__assert>
# include <__config>
# include <__functional/is_transparent.h>
Expand Down Expand Up @@ -898,6 +899,9 @@ public:
return __tree_.__equal_range_multi(__k);
}
# endif

template <class, class...>
friend struct __specialized_algorithm;
};

# if _LIBCPP_STD_VER >= 17
Expand Down Expand Up @@ -944,6 +948,23 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
# endif

# if _LIBCPP_STD_VER >= 14
template <class _Alg, class _Key, class _Compare, class _Allocator>
struct __specialized_algorithm<_Alg, __single_range<set<_Key, _Compare, _Allocator>>> {
using __set _LIBCPP_NODEBUG = set<_Key, _Compare, _Allocator>;

static const bool __has_algorithm =
__specialized_algorithm<_Alg, __single_range<typename __set::__base>>::__has_algorithm;

// set's begin() and end() are identical with and without const qualification
template <class... _Args>
_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
return __specialized_algorithm<_Alg, __single_range<typename __set::__base>>()(
__set.__tree_, std::forward<_Args>(__args)...);
}
};
# endif

template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
Expand Down Expand Up @@ -1342,6 +1363,9 @@ public:
return __tree_.__equal_range_multi(__k);
}
# endif

template <class, class...>
friend struct __specialized_algorithm;
};

# if _LIBCPP_STD_VER >= 17
Expand Down Expand Up @@ -1389,6 +1413,23 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
# endif

# if _LIBCPP_STD_VER >= 14
template <class _Alg, class _Key, class _Compare, class _Allocator>
struct __specialized_algorithm<_Alg, __single_range<multiset<_Key, _Compare, _Allocator>>> {
using __set _LIBCPP_NODEBUG = multiset<_Key, _Compare, _Allocator>;

static const bool __has_algorithm =
__specialized_algorithm<_Alg, __single_range<typename __set::__base>>::__has_algorithm;

// set's begin() and end() are identical with and without const qualification
template <class... _Args>
_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
return __specialized_algorithm<_Alg, __single_range<typename __set::__base>>()(
__set.__tree_, std::forward<_Args>(__args)...);
}
};
# endif

template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
Expand Down
Loading
Loading