Revert "[libc++] Optimize vector growing of trivially relocatable types (#76657)"

kstoimenov · kstoimenov · commit 2352fdd2026e · 2024-02-02T20:43:51.000Z
Broke sanitizer bots: https://lab.llvm.org/buildbot/#/builders/5/builds/40641 This reverts commit 67eee4a.
diff --git a/libcxx/benchmarks/ContainerBenchmarks.h b/libcxx/benchmarks/ContainerBenchmarks.h
@@ -80,7 +80,7 @@ void BM_ConstructFromRange(benchmark::State& st, Container, GenInputs gen) {
 }
 
 template <class Container>
-void BM_Pushback_no_grow(benchmark::State& state, Container c) {
+void BM_Pushback(benchmark::State& state, Container c) {
   int count = state.range(0);
   c.reserve(count);
   while (state.KeepRunningBatch(count)) {
diff --git a/libcxx/benchmarks/vector_operations.bench.cpp b/libcxx/benchmarks/vector_operations.bench.cpp
@@ -1,7 +1,6 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
-#include <deque>
 #include <functional>
 #include <vector>
 
@@ -40,21 +39,6 @@ BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_size_t, std::vector<size_t>{}, g
 BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_string, std::vector<std::string>{}, getRandomStringInputs)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_Pushback_no_grow, vector_int, std::vector<int>{})->Arg(TestNumInputs);
-
-template <class T>
-void bm_grow(benchmark::State& state) {
-  for (auto _ : state) {
-    std::vector<T> vec;
-    benchmark::DoNotOptimize(vec);
-    for (size_t i = 0; i != 2048; ++i)
-      vec.emplace_back();
-    benchmark::DoNotOptimize(vec);
-  }
-}
-BENCHMARK(bm_grow<int>);
-BENCHMARK(bm_grow<std::string>);
-BENCHMARK(bm_grow<std::unique_ptr<int>>);
-BENCHMARK(bm_grow<std::deque<int>>);
+BENCHMARK_CAPTURE(BM_Pushback, vector_int, std::vector<int>{})->Arg(TestNumInputs);
 
 BENCHMARK_MAIN();
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
@@ -44,8 +44,8 @@ Implemented Papers
 
 Improvements and New Features
 -----------------------------
+TODO
 
-- The performance of growing ``std::vector`` has been improved for trivially relocatable types.
 
 Deprecations and Removals
 -------------------------
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
@@ -808,7 +808,6 @@ set(files
   __type_traits/is_trivially_lexicographically_comparable.h
   __type_traits/is_trivially_move_assignable.h
   __type_traits/is_trivially_move_constructible.h
-  __type_traits/is_trivially_relocatable.h
   __type_traits/is_unbounded_array.h
   __type_traits/is_union.h
   __type_traits/is_unsigned.h
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -29,7 +29,6 @@
 #include <__type_traits/is_trivially_copy_constructible.h>
 #include <__type_traits/is_trivially_move_assignable.h>
 #include <__type_traits/is_trivially_move_constructible.h>
-#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/is_unbounded_array.h>
 #include <__type_traits/negation.h>
 #include <__type_traits/remove_const.h>
@@ -595,56 +594,60 @@ __uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1,
   return std::__rewrap_iter(__first2, __result);
 }
 
-template <class _Alloc, class _Type>
-struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
-
-template <class _Type>
-struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
-
-template <class _Alloc, class _Tp>
-struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
-
-template <class _Tp, class _Up>
-struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
-
-// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result.
-// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
-// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
-//
-// Preconditions:  __result doesn't contain any objects and [__first, __last) contains objects
-// Postconditions: __result contains the objects from [__first, __last) and
-//                 [__first, __last) doesn't contain any objects
+// Move-construct the elements [__first1, __last1) into [__first2, __first2 + N)
+// if the move constructor is noexcept, where N is distance(__first1, __last1).
 //
-// The strong exception guarantee is provided if any of the following are true:
-// - is_nothrow_move_constructible<_Tp>
-// - is_copy_constructible<_Tp>
-// - __libcpp_is_trivially_relocatable<_Tp>
-template <class _Alloc, class _Tp>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
-__uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _Tp* __result) {
+// Otherwise try to copy all elements. If an exception is thrown the already copied
+// elements are destroyed in reverse order of their construction.
+template <class _Alloc, class _Iter1, class _Sent1, class _Iter2>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter2
+__uninitialized_allocator_move_if_noexcept(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) {
   static_assert(__is_cpp17_move_insertable<_Alloc>::value,
                 "The specified type does not meet the requirements of Cpp17MoveInsertable");
-  if (__libcpp_is_constant_evaluated() || !__libcpp_is_trivially_relocatable<_Tp>::value ||
-      !__allocator_has_trivial_move_construct<_Alloc, _Tp>::value ||
-      !__allocator_has_trivial_destroy<_Alloc, _Tp>::value) {
-    auto __destruct_first = __result;
-    auto __guard =
-        std::__make_exception_guard(_AllocatorDestroyRangeReverse<_Alloc, _Tp*>(__alloc, __destruct_first, __result));
-    while (__first != __last) {
+  auto __destruct_first = __first2;
+  auto __guard =
+      std::__make_exception_guard(_AllocatorDestroyRangeReverse<_Alloc, _Iter2>(__alloc, __destruct_first, __first2));
+  while (__first1 != __last1) {
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-      allocator_traits<_Alloc>::construct(__alloc, __result, std::move_if_noexcept(*__first));
+    allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), std::move_if_noexcept(*__first1));
 #else
-      allocator_traits<_Alloc>::construct(__alloc, __result, std::move(*__first));
+    allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), std::move(*__first1));
 #endif
-      ++__first;
-      ++__result;
+    ++__first1;
+    ++__first2;
+  }
+  __guard.__complete();
+  return __first2;
+}
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
+
+#ifndef _LIBCPP_COMPILER_GCC
+template <
+    class _Alloc,
+    class _Iter1,
+    class _Iter2,
+    class _Type = typename iterator_traits<_Iter1>::value_type,
+    class = __enable_if_t<is_trivially_move_constructible<_Type>::value && is_trivially_move_assignable<_Type>::value &&
+                          __allocator_has_trivial_move_construct<_Alloc, _Type>::value> >
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter2
+__uninitialized_allocator_move_if_noexcept(_Alloc&, _Iter1 __first1, _Iter1 __last1, _Iter2 __first2) {
+  if (__libcpp_is_constant_evaluated()) {
+    while (__first1 != __last1) {
+      std::__construct_at(std::__to_address(__first2), std::move(*__first1));
+      ++__first1;
+      ++__first2;
     }
-    __guard.__complete();
-    std::__allocator_destroy(__alloc, __first, __last);
+    return __first2;
   } else {
-    __builtin_memcpy(__result, __first, sizeof(_Tp) * (__last - __first));
+    return std::move(__first1, __last1, __first2);
   }
 }
+#endif // _LIBCPP_COMPILER_GCC
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
@@ -21,7 +21,6 @@
 #include <__memory/compressed_pair.h>
 #include <__type_traits/add_lvalue_reference.h>
 #include <__type_traits/common_type.h>
-#include <__type_traits/conditional.h>
 #include <__type_traits/dependent_type.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_array.h>
@@ -34,7 +33,6 @@
 #include <__type_traits/is_reference.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/remove_extent.h>
 #include <__type_traits/type_identity.h>
@@ -131,17 +129,6 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
 
   static_assert(!is_rvalue_reference<deleter_type>::value, "the specified deleter type cannot be an rvalue reference");
 
-  // A unique_ptr contains the following members which may be trivially relocatable:
-  // - pointer : this may be trivially relocatable, so it's checked
-  // - deleter_type: this may be trivially relocatable, so it's checked
-  //
-  // This unique_ptr implementation only contains a pointer to the unique object and a deleter, so there are no
-  // references to itself. This means that the entire structure is trivially relocatable if its members are.
-  using __trivially_relocatable = __conditional_t<
-      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
-      unique_ptr,
-      void>;
-
 private:
   __compressed_pair<pointer, deleter_type> __ptr_;
 
@@ -289,17 +276,6 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   typedef _Dp deleter_type;
   typedef typename __pointer<_Tp, deleter_type>::type pointer;
 
-  // A unique_ptr contains the following members which may be trivially relocatable:
-  // - pointer : this may be trivially relocatable, so it's checked
-  // - deleter_type: this may be trivially relocatable, so it's checked
-  //
-  // This unique_ptr implementation only contains a pointer to the unique object and a deleter, so there are no
-  // references to itself. This means that the entire structure is trivially relocatable if its members are.
-  using __trivially_relocatable = __conditional_t<
-      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
-      unique_ptr,
-      void>;
-
 private:
   __compressed_pair<pointer, deleter_type> __ptr_;
 
diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
@@ -796,7 +796,6 @@
   { include: [ "<__type_traits/is_trivially_lexicographically_comparable.h>", "private", "<type_traits>", "public" ] },
   { include: [ "<__type_traits/is_trivially_move_assignable.h>", "private", "<type_traits>", "public" ] },
   { include: [ "<__type_traits/is_trivially_move_constructible.h>", "private", "<type_traits>", "public" ] },
-  { include: [ "<__type_traits/is_trivially_relocatable.h>", "private", "<type_traits>", "public" ] },
   { include: [ "<__type_traits/is_unbounded_array.h>", "private", "<type_traits>", "public" ] },
   { include: [ "<__type_traits/is_union.h>", "private", "<type_traits>", "public" ] },
   { include: [ "<__type_traits/is_unsigned.h>", "private", "<type_traits>", "public" ] },
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
@@ -1970,7 +1970,6 @@ module std_private_type_traits_is_trivially_destructible                 [system
 module std_private_type_traits_is_trivially_lexicographically_comparable [system] { header "__type_traits/is_trivially_lexicographically_comparable.h" }
 module std_private_type_traits_is_trivially_move_assignable              [system] { header "__type_traits/is_trivially_move_assignable.h" }
 module std_private_type_traits_is_trivially_move_constructible           [system] { header "__type_traits/is_trivially_move_constructible.h" }
-module std_private_type_traits_is_trivially_relocatable                  [system] { header "__type_traits/is_trivially_relocatable.h" }
 module std_private_type_traits_is_unbounded_array                        [system] { header "__type_traits/is_unbounded_array.h" }
 module std_private_type_traits_is_union                                  [system] { header "__type_traits/is_union.h" }
 module std_private_type_traits_is_unsigned                               [system] { header "__type_traits/is_unsigned.h" }
diff --git a/libcxx/include/string b/libcxx/include/string
@@ -599,7 +599,6 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 #include <__ranges/size.h>
 #include <__string/char_traits.h>
 #include <__string/extern_template_lists.h>
-#include <__type_traits/conditional.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_array.h>
 #include <__type_traits/is_convertible.h>
@@ -608,7 +607,6 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_standard_layout.h>
 #include <__type_traits/is_trivial.h>
-#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/noexcept_move_assign_container.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
@@ -726,20 +724,6 @@ public:
   typedef typename __alloc_traits::pointer pointer;
   typedef typename __alloc_traits::const_pointer const_pointer;
 
-  // A basic_string contains the following members which may be trivially relocatable:
-  // - pointer: is currently assumed to be trivially relocatable, but is still checked in case that changes
-  // - size_type: is always trivially relocatable, since it has to be an integral type
-  // - value_type: is always trivially relocatable, since it has to be trivial
-  // - unsigned char: is a fundamental type, so it's trivially relocatable
-  // - allocator_type: may or may not be trivially relocatable, so it's checked
-  //
-  // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
-  // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
-  using __trivially_relocatable = __conditional_t<
-      __libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
-      basic_string,
-      void>;
-
   static_assert((!is_array<value_type>::value), "Character type of basic_string must not be an array");
   static_assert((is_standard_layout<value_type>::value), "Character type of basic_string must be standard-layout");
   static_assert((is_trivial<value_type>::value), "Character type of basic_string must be trivial");
diff --git a/libcxx/include/vector b/libcxx/include/vector
@@ -982,54 +982,37 @@ template <ranges::input_range _Range,
 vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
 #endif
 
-// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of
-// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This
-// function has a strong exception guarantee.
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
   __annotate_delete();
-  auto __new_begin = __v.__begin_ - (__end_ - __begin_);
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
-  __v.__begin_ = __new_begin;
-  __end_       = __begin_; // All the objects have been destroyed by relocating them.
+  using _RevIter = std::reverse_iterator<pointer>;
+  __v.__begin_   = std::__uninitialized_allocator_move_if_noexcept(
+                     __alloc(), _RevIter(__end_), _RevIter(__begin_), _RevIter(__v.__begin_))
+                     .base();
   std::swap(this->__begin_, __v.__begin_);
   std::swap(this->__end_, __v.__end_);
   std::swap(this->__end_cap(), __v.__end_cap());
   __v.__first_ = __v.__begin_;
   __annotate_new(size());
 }
 
-// __swap_out_circular_buffer relocates the objects in [__begin_, __p) into the front of __v, the objects in
-// [__p, __end_) into the back of __v and swaps the buffers of *this and __v. It is assumed that __v provides space for
-// exactly (__p - __begin_) objects in the front and space for at least (__end_ - __p) objects in the back. This
-// function has a strong exception guarantee if __begin_ == __p || __end_ == __p.
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
 vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p) {
   __annotate_delete();
-  pointer __ret = __v.__begin_;
-
-  // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
-  // in case something in [__begin_, __p) throws.
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
-  __v.__end_ += (__end_ - __p);
-  __end_           = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
-  auto __new_begin = __v.__begin_ - (__p - __begin_);
-
-  std::__uninitialized_allocator_relocate(
-      __alloc(), std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
-  __v.__begin_ = __new_begin;
-  __end_       = __begin_; // All the objects have been destroyed by relocating them.
-
+  pointer __r    = __v.__begin_;
+  using _RevIter = std::reverse_iterator<pointer>;
+  __v.__begin_   = std::__uninitialized_allocator_move_if_noexcept(
+                     __alloc(), _RevIter(__p), _RevIter(__begin_), _RevIter(__v.__begin_))
+                     .base();
+  __v.__end_ = std::__uninitialized_allocator_move_if_noexcept(__alloc(), __p, __end_, __v.__end_);
   std::swap(this->__begin_, __v.__begin_);
   std::swap(this->__end_, __v.__end_);
   std::swap(this->__end_cap(), __v.__end_cap());
   __v.__first_ = __v.__begin_;
   __annotate_new(size());
-  return __ret;
+  return __r;
 }
 
 template <class _Tp, class _Allocator>
diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
diff --git a/libcxx/test/support/count_new.h b/libcxx/test/support/count_new.h

Original file line number	Diff line number	Diff line change
`@@ -80,7 +80,7 @@ void BM_ConstructFromRange(benchmark::State& st, Container, GenInputs gen) {`
`80`	`80`	`}`
`81`	`81`
`82`	`82`	`template <class Container>`
`83`		`-void BM_Pushback_no_grow(benchmark::State& state, Container c) {`
	`83`	`+void BM_Pushback(benchmark::State& state, Container c) {`
`84`	`84`	`int count = state.range(0);`
`85`	`85`	`c.reserve(count);`
`86`	`86`	`while (state.KeepRunningBatch(count)) {`