update boost

linx64/include/boost/unordered/detail/foa/element_type.hpp

@@ -0,0 +1,60 @@
+/* Copyright 2023 Christian Mazakas.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template<class T>
+struct element_type
+{
+  using value_type=T;
+  value_type* p;
+
+  /*
+   * we use a deleted copy constructor here so the type is no longer
+   * trivially copy-constructible which inhibits our memcpy
+   * optimizations when copying the tables
+   */
+  element_type() = default;
+  element_type(value_type* p_):p(p_){}
+  element_type(element_type const&) = delete;
+  element_type(element_type&& rhs) noexcept
+  {
+    p = rhs.p;
+    rhs.p = nullptr;
+  }
+
+  element_type& operator=(element_type const&)=delete;
+  element_type& operator=(element_type&& rhs)noexcept
+  {
+    if (this!=&rhs){
+      p=rhs.p;
+      rhs.p=nullptr;
+    }
+    return *this;
+  }
+
+  void swap(element_type& rhs)noexcept
+  {
+    auto tmp=p;
+    p=rhs.p;
+    rhs.p=tmp;
+  }
+};
+
+}
+}
+}
+}
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP

linx64/include/boost/unordered/detail/foa/flat_map_types.hpp

@@ -0,0 +1,73 @@
+// Copyright (C) 2023 Christian Mazakas
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_FLAT_MAP_TYPES_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_FLAT_MAP_TYPES_HPP
+
+#include <boost/core/allocator_access.hpp>
+
+namespace boost {
+  namespace unordered {
+    namespace detail {
+      namespace foa {
+        template <class Key, class T> struct flat_map_types
+        {
+          using key_type = Key;
+          using raw_key_type = typename std::remove_const<Key>::type;
+          using raw_mapped_type = typename std::remove_const<T>::type;
+
+          using init_type = std::pair<raw_key_type, raw_mapped_type>;
+          using moved_type = std::pair<raw_key_type&&, raw_mapped_type&&>;
+          using value_type = std::pair<Key const, T>;
+
+          using element_type = value_type;
+
+          static value_type& value_from(element_type& x) { return x; }
+
+          template <class K, class V>
+          static raw_key_type const& extract(std::pair<K, V> const& kv)
+          {
+            return kv.first;
+          }
+
+          static moved_type move(init_type& x)
+          {
+            return {std::move(x.first), std::move(x.second)};
+          }
+
+          static moved_type move(element_type& x)
+          {
+            // TODO: we probably need to launder here
+            return {std::move(const_cast<raw_key_type&>(x.first)),
+              std::move(const_cast<raw_mapped_type&>(x.second))};
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, init_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, value_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A> static void destroy(A& al, init_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+
+          template <class A> static void destroy(A& al, value_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+        };
+      } // namespace foa
+    }   // namespace detail
+  }     // namespace unordered
+} // namespace boost
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_FLAT_MAP_TYPES_HPP

linx64/include/boost/unordered/detail/foa/flat_set_types.hpp

@@ -0,0 +1,44 @@
+// Copyright (C) 2023 Christian Mazakas
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_FLAT_SET_TYPES_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_FLAT_SET_TYPES_HPP
+
+#include <boost/core/allocator_access.hpp>
+
+namespace boost {
+  namespace unordered {
+    namespace detail {
+      namespace foa {
+        template <class Key> struct flat_set_types
+        {
+          using key_type = Key;
+          using init_type = Key;
+          using value_type = Key;
+
+          static Key const& extract(value_type const& key) { return key; }
+
+          using element_type = value_type;
+
+          static Key& value_from(element_type& x) { return x; }
+
+          static element_type&& move(element_type& x) { return std::move(x); }
+
+          template <class A, class... Args>
+          static void construct(A& al, value_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A> static void destroy(A& al, value_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+        };
+      } // namespace foa
+    }   // namespace detail
+  }     // namespace unordered
+} // namespace boost
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_FLAT_SET_TYPES_HPP

linx64/include/boost/unordered/detail/foa/ignore_wshadow.hpp

@@ -0,0 +1,35 @@
+/* Copyright 2023 Joaquin M Lopez Munoz.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#include <boost/config.hpp>
+
+#if defined(BOOST_GCC)
+#if !defined(BOOST_UNORDERED_DETAIL_RESTORE_WSHADOW)
+ /* GCC's -Wshadow triggers at scenarios like this: 
+ *
+ *   struct foo{};
+ *   template<typename Base>
+ *   struct derived:Base
+ *   {
+ *     void f(){int foo;}
+ *   };
+ * 
+ *   derived<foo>x;
+ *   x.f(); // declaration of "foo" in derived::f shadows base type "foo"
+ *
+ * This makes shadowing warnings unavoidable in general when a class template
+ * derives from user-provided classes, as is the case with foa::table_core
+ * deriving from empty_value.
+ */
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wshadow"
+#else
+#pragma GCC diagnostic pop
+#endif
+#endif

linx64/include/boost/unordered/detail/foa/node_handle.hpp

@@ -0,0 +1,210 @@
+/* Copyright 2023 Christian Mazakas.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_NODE_HANDLE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_NODE_HANDLE_HPP
+
+#include <boost/config.hpp>
+#include <boost/core/allocator_access.hpp>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template <class Iterator,class NodeType>
+struct insert_return_type
+{
+  Iterator position;
+  bool     inserted;
+  NodeType node;
+};
+
+template <class T>
+union opt_storage {
+  BOOST_ATTRIBUTE_NO_UNIQUE_ADDRESS T t_;
+
+  opt_storage(){}
+  ~opt_storage(){}
+};
+
+template <class TypePolicy,class Allocator>
+struct node_handle_base
+{
+  protected:
+    using type_policy=TypePolicy;
+    using element_type=typename type_policy::element_type;
+
+  public:
+    using allocator_type = Allocator;
+
+  private:
+    using node_value_type=typename type_policy::value_type;
+    element_type p_;
+    BOOST_ATTRIBUTE_NO_UNIQUE_ADDRESS opt_storage<Allocator> a_;
+
+  protected:
+    node_value_type& data()noexcept
+    {
+      return *(p_.p);
+    }
+
+    node_value_type const& data()const noexcept
+    {
+      return *(p_.p);
+    }
+
+    element_type& element()noexcept
+    {
+      BOOST_ASSERT(!empty());
+      return p_;
+    }
+
+    element_type const& element()const noexcept
+    {
+      BOOST_ASSERT(!empty());
+      return p_;
+    }
+
+    Allocator& al()noexcept
+    {
+      BOOST_ASSERT(!empty());
+      return a_.t_;
+    }
+
+    Allocator const& al()const noexcept
+    {
+      BOOST_ASSERT(!empty());
+      return a_.t_;
+    }
+
+    void emplace(element_type&& x,Allocator a)
+    {
+      BOOST_ASSERT(empty());
+      auto* p=x.p;
+      p_.p=p;
+      new(&a_.t_)Allocator(a);
+      x.p=nullptr;
+    }
+
+    void reset()
+    {
+      a_.t_.~Allocator();
+      p_.p=nullptr;
+    }
+
+  public:
+    constexpr node_handle_base()noexcept:p_{nullptr}{}
+
+    node_handle_base(node_handle_base&& nh) noexcept
+    {
+      p_.p = nullptr;
+      if (!nh.empty()){
+        emplace(std::move(nh.p_),nh.al());
+        nh.reset();
+      }
+    }
+
+    node_handle_base& operator=(node_handle_base&& nh)noexcept
+    {
+      if(this!=&nh){
+        if(empty()){
+          if(nh.empty()){                      /* empty(),  nh.empty() */
+            /* nothing to do */
+          }else{                               /* empty(), !nh.empty() */
+            emplace(std::move(nh.p_),std::move(nh.al()));
+            nh.reset();
+          }
+        }else{
+          if(nh.empty()){                      /* !empty(),  nh.empty() */
+            type_policy::destroy(al(),&p_);
+            reset();
+          }else{                               /* !empty(), !nh.empty() */
+            bool const pocma=
+              boost::allocator_propagate_on_container_move_assignment<
+                Allocator>::type::value;
+
+            BOOST_ASSERT(pocma||al()==nh.al());
+
+            type_policy::destroy(al(),&p_);
+            if(pocma){
+              al()=std::move(nh.al());
+            }
+
+            p_=std::move(nh.p_);
+            nh.reset();
+          }
+        }
+      }else{
+        if(empty()){                           /* empty(),  nh.empty() */
+          /* nothing to do */
+        }else{                                 /* !empty(), !nh.empty() */
+          type_policy::destroy(al(),&p_);
+          reset();
+        }
+      }
+      return *this;
+    }
+
+    ~node_handle_base()
+    {
+      if(!empty()){
+        type_policy::destroy(al(),&p_);
+        reset();
+      }
+    }
+
+    allocator_type get_allocator()const noexcept{return al();}
+    explicit operator bool()const noexcept{ return !empty();}
+    BOOST_ATTRIBUTE_NODISCARD bool empty()const noexcept{return p_.p==nullptr;}
+
+    void swap(node_handle_base& nh) noexcept(
+      boost::allocator_is_always_equal<Allocator>::type::value||
+      boost::allocator_propagate_on_container_swap<Allocator>::type::value)
+    {
+      if(this!=&nh){
+        if(empty()){
+          if(nh.empty()) {
+            /* nothing to do here */
+          } else {
+            emplace(std::move(nh.p_), nh.al());
+            nh.reset();
+          }
+        }else{
+          if(nh.empty()){
+            nh.emplace(std::move(p_),al());
+            reset();
+          }else{
+            bool const pocs=
+              boost::allocator_propagate_on_container_swap<
+                Allocator>::type::value;
+
+            BOOST_ASSERT(pocs || al()==nh.al());
+
+            using std::swap;
+            p_.swap(nh.p_);
+            if(pocs)swap(al(),nh.al());
+          }
+        }
+      }
+    }
+
+    friend
+    void swap(node_handle_base& lhs,node_handle_base& rhs)
+      noexcept(noexcept(lhs.swap(rhs)))
+    {
+      return lhs.swap(rhs);
+    }
+};
+
+}
+}
+}
+}
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_NODE_HANDLE_HPP

linx64/include/boost/unordered/detail/foa/node_map_types.hpp

@@ -0,0 +1,132 @@
+// Copyright (C) 2023 Christian Mazakas
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_TYPES_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_TYPES_HPP
+
+#include <boost/core/allocator_access.hpp>
+#include <boost/core/no_exceptions_support.hpp>
+#include <boost/core/pointer_traits.hpp>
+
+namespace boost {
+  namespace unordered {
+    namespace detail {
+      namespace foa {
+        template <class Key, class T> struct node_map_types
+        {
+          using key_type = Key;
+          using mapped_type = T;
+          using raw_key_type = typename std::remove_const<Key>::type;
+          using raw_mapped_type = typename std::remove_const<T>::type;
+
+          using init_type = std::pair<raw_key_type, raw_mapped_type>;
+          using value_type = std::pair<Key const, T>;
+          using moved_type = std::pair<raw_key_type&&, raw_mapped_type&&>;
+
+          using element_type = foa::element_type<value_type>;
+
+          static value_type& value_from(element_type const& x)
+          {
+            return *(x.p);
+          }
+
+          template <class K, class V>
+          static raw_key_type const& extract(std::pair<K, V> const& kv)
+          {
+            return kv.first;
+          }
+
+          static raw_key_type const& extract(element_type const& kv)
+          {
+            return kv.p->first;
+          }
+
+          static element_type&& move(element_type& x) { return std::move(x); }
+          static moved_type move(init_type& x)
+          {
+            return {std::move(x.first), std::move(x.second)};
+          }
+
+          static moved_type move(value_type& x)
+          {
+            return {std::move(const_cast<raw_key_type&>(x.first)),
+              std::move(const_cast<raw_mapped_type&>(x.second))};
+          }
+
+          template <class A>
+          static void construct(A&, element_type* p, element_type&& x) noexcept
+          {
+            p->p = x.p;
+            x.p = nullptr;
+          }
+
+          template <class A>
+          static void construct(
+            A& al, element_type* p, element_type const& copy)
+          {
+            construct(al, p, *copy.p);
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, init_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, value_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, element_type* p, Args&&... args)
+          {
+            p->p = boost::to_address(boost::allocator_allocate(al, 1));
+            BOOST_TRY
+            {
+              boost::allocator_construct(al, p->p, std::forward<Args>(args)...);
+            }
+            BOOST_CATCH(...)
+            {
+              using pointer_type = typename boost::allocator_pointer<A>::type;
+              using pointer_traits = boost::pointer_traits<pointer_type>;
+
+              boost::allocator_deallocate(
+                al, pointer_traits::pointer_to(*(p->p)), 1);
+              BOOST_RETHROW
+            }
+            BOOST_CATCH_END
+          }
+
+          template <class A> static void destroy(A& al, value_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+
+          template <class A> static void destroy(A& al, init_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+
+          template <class A>
+          static void destroy(A& al, element_type* p) noexcept
+          {
+            if (p->p) {
+              using pointer_type = typename boost::allocator_pointer<A>::type;
+              using pointer_traits = boost::pointer_traits<pointer_type>;
+
+              destroy(al, p->p);
+              boost::allocator_deallocate(
+                al, pointer_traits::pointer_to(*(p->p)), 1);
+            }
+          }
+        };
+
+      } // namespace foa
+    }   // namespace detail
+  }     // namespace unordered
+} // namespace boost
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_TYPES_HPP

linx64/include/boost/unordered/detail/foa/node_set_types.hpp

@@ -0,0 +1,95 @@
+// Copyright (C) 2023 Christian Mazakas
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_NODE_SET_TYPES_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_NODE_SET_TYPES_HPP
+
+#include <boost/core/allocator_access.hpp>
+#include <boost/core/no_exceptions_support.hpp>
+#include <boost/core/pointer_traits.hpp>
+
+namespace boost {
+  namespace unordered {
+    namespace detail {
+      namespace foa {
+
+        template <class Key> struct node_set_types
+        {
+          using key_type = Key;
+          using init_type = Key;
+          using value_type = Key;
+
+          static Key const& extract(value_type const& key) { return key; }
+
+          using element_type = foa::element_type<value_type>;
+
+          static value_type& value_from(element_type const& x) { return *x.p; }
+          static Key const& extract(element_type const& k) { return *k.p; }
+          static element_type&& move(element_type& x) { return std::move(x); }
+          static value_type&& move(value_type& x) { return std::move(x); }
+
+          template <class A>
+          static void construct(
+            A& al, element_type* p, element_type const& copy)
+          {
+            construct(al, p, *copy.p);
+          }
+
+          template <typename Allocator>
+          static void construct(
+            Allocator&, element_type* p, element_type&& x) noexcept
+          {
+            p->p = x.p;
+            x.p = nullptr;
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, value_type* p, Args&&... args)
+          {
+            boost::allocator_construct(al, p, std::forward<Args>(args)...);
+          }
+
+          template <class A, class... Args>
+          static void construct(A& al, element_type* p, Args&&... args)
+          {
+            p->p = boost::to_address(boost::allocator_allocate(al, 1));
+            BOOST_TRY
+            {
+              boost::allocator_construct(al, p->p, std::forward<Args>(args)...);
+            }
+            BOOST_CATCH(...)
+            {
+              boost::allocator_deallocate(al,
+                boost::pointer_traits<typename boost::allocator_pointer<
+                  A>::type>::pointer_to(*p->p),
+                1);
+              BOOST_RETHROW
+            }
+            BOOST_CATCH_END
+          }
+
+          template <class A> static void destroy(A& al, value_type* p) noexcept
+          {
+            boost::allocator_destroy(al, p);
+          }
+
+          template <class A>
+          static void destroy(A& al, element_type* p) noexcept
+          {
+            if (p->p) {
+              destroy(al, p->p);
+              boost::allocator_deallocate(al,
+                boost::pointer_traits<typename boost::allocator_pointer<
+                  A>::type>::pointer_to(*(p->p)),
+                1);
+            }
+          }
+        };
+
+      } // namespace foa
+    }   // namespace detail
+  }     // namespace unordered
+} // namespace boost
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_NODE_SET_TYPES_HPP

linx64/include/boost/unordered/detail/foa/restore_wshadow.hpp

@@ -0,0 +1,11 @@
+/* Copyright 2023 Joaquin M Lopez Munoz.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#define BOOST_UNORDERED_DETAIL_RESTORE_WSHADOW
+#include <boost/unordered/detail/foa/ignore_wshadow.hpp>
+#undef BOOST_UNORDERED_DETAIL_RESTORE_WSHADOW

linx64/include/boost/unordered/detail/foa/rw_spinlock.hpp

@@ -0,0 +1,179 @@
+#ifndef BOOST_UNORDERED_DETAIL_FOA_RW_SPINLOCK_HPP_INCLUDED
+#define BOOST_UNORDERED_DETAIL_FOA_RW_SPINLOCK_HPP_INCLUDED
+
+// Copyright 2023 Peter Dimov
+// Distributed under the Boost Software License, Version 1.0.
+// https://www.boost.org/LICENSE_1_0.txt
+
+#include <boost/core/yield_primitives.hpp>
+#include <atomic>
+#include <cstdint>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+class rw_spinlock
+{
+private:
+
+    // bit 31: locked exclusive
+    // bit 30: writer pending
+    // bit 29..0: reader lock count
+
+    static constexpr std::uint32_t locked_exclusive_mask = 1u << 31; // 0x8000'0000
+    static constexpr std::uint32_t writer_pending_mask = 1u << 30; // 0x4000'0000
+    static constexpr std::uint32_t reader_lock_count_mask = writer_pending_mask - 1; // 0x3FFF'FFFF
+
+    std::atomic<std::uint32_t> state_ = {};
+
+private:
+
+    // Effects: Provides a hint to the implementation that the current thread
+    //          has been unable to make progress for k+1 iterations.
+
+    static void yield( unsigned k ) noexcept
+    {
+        unsigned const sleep_every = 1024; // see below
+
+        k %= sleep_every;
+
+        if( k < 5 )
+        {
+            // Intel recommendation from the Optimization Reference Manual
+            // Exponentially increase number of PAUSE instructions each
+            // iteration until reaching a maximum which is approximately
+            // one timeslice long (2^4 == 16 in our case)
+
+            unsigned const pause_count = 1u << k;
+
+            for( unsigned i = 0; i < pause_count; ++i )
+            {
+                boost::core::sp_thread_pause();
+            }
+        }
+        else if( k < sleep_every - 1 )
+        {
+            // Once the maximum number of PAUSE instructions is reached,
+            // we switch to yielding the timeslice immediately
+
+            boost::core::sp_thread_yield();
+        }
+        else
+        {
+            // After `sleep_every` iterations of no progress, we sleep,
+            // to avoid a deadlock if a lower priority thread has the lock
+
+            boost::core::sp_thread_sleep();
+        }
+    }
+
+public:
+
+    bool try_lock_shared() noexcept
+    {
+        std::uint32_t st = state_.load( std::memory_order_relaxed );
+
+        if( st >= reader_lock_count_mask )
+        {
+            // either bit 31 set, bit 30 set, or reader count is max
+            return false;
+        }
+
+        std::uint32_t newst = st + 1;
+        return state_.compare_exchange_strong( st, newst, std::memory_order_acquire, std::memory_order_relaxed );
+    }
+
+    void lock_shared() noexcept
+    {
+        for( unsigned k = 0; ; ++k )
+        {
+            std::uint32_t st = state_.load( std::memory_order_relaxed );
+
+            if( st < reader_lock_count_mask )
+            {
+                std::uint32_t newst = st + 1;
+                if( state_.compare_exchange_weak( st, newst, std::memory_order_acquire, std::memory_order_relaxed ) ) return;
+            }
+
+            yield( k );
+        }
+    }
+
+    void unlock_shared() noexcept
+    {
+        // pre: locked shared, not locked exclusive
+
+        state_.fetch_sub( 1, std::memory_order_release );
+
+        // if the writer pending bit is set, there's a writer waiting
+        // let it acquire the lock; it will clear the bit on unlock
+    }
+
+    bool try_lock() noexcept
+    {
+        std::uint32_t st = state_.load( std::memory_order_relaxed );
+
+        if( st & locked_exclusive_mask )
+        {
+            // locked exclusive
+            return false;
+        }
+
+        if( st & reader_lock_count_mask )
+        {
+            // locked shared
+            return false;
+        }
+
+        std::uint32_t newst = locked_exclusive_mask;
+        return state_.compare_exchange_strong( st, newst, std::memory_order_acquire, std::memory_order_relaxed );
+    }
+
+    void lock() noexcept
+    {
+        for( unsigned k = 0; ; ++k )
+        {
+            std::uint32_t st = state_.load( std::memory_order_relaxed );
+
+            if( st & locked_exclusive_mask )
+            {
+                // locked exclusive, spin
+            }
+            else if( ( st & reader_lock_count_mask ) == 0 )
+            {
+                // not locked exclusive, not locked shared, try to lock
+
+                std::uint32_t newst = locked_exclusive_mask;
+                if( state_.compare_exchange_weak( st, newst, std::memory_order_acquire, std::memory_order_relaxed ) ) return;
+            }
+            else if( st & writer_pending_mask )
+            {
+                // writer pending bit already set, nothing to do
+            }
+            else
+            {
+                // locked shared, set writer pending bit
+
+                std::uint32_t newst = st | writer_pending_mask;
+                state_.compare_exchange_weak( st, newst, std::memory_order_relaxed, std::memory_order_relaxed );
+            }
+
+            yield( k );
+        }
+    }
+
+    void unlock() noexcept
+    {
+        // pre: locked exclusive, not locked shared
+        state_.store( 0, std::memory_order_release );
+    }
+};
+
+} /* namespace foa */
+} /* namespace detail */
+} /* namespace unordered */
+} /* namespace boost */
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_RW_SPINLOCK_HPP_INCLUDED

linx64/include/boost/unordered/detail/foa/table.hpp

@@ -0,0 +1,556 @@
+/* Fast open-addressing hash table.
+ *
+ * Copyright 2022-2023 Joaquin M Lopez Munoz.
+ * Copyright 2023 Christian Mazakas.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_TABLE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_TABLE_HPP
+
+#include <boost/assert.hpp>
+#include <boost/config.hpp>
+#include <boost/config/workaround.hpp>
+#include <boost/unordered/detail/foa/core.hpp>
+#include <cstddef>
+#include <iterator>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+/* use plain integrals for group metadata storage */
+
+template<typename Integral>
+struct plain_integral
+{
+  operator Integral()const{return n;}
+  void operator=(Integral m){n=m;}
+
+#if BOOST_WORKAROUND(BOOST_GCC,>=50000 && BOOST_GCC<60000)
+  void operator|=(Integral m){n=static_cast<Integral>(n|m);}
+  void operator&=(Integral m){n=static_cast<Integral>(n&m);}
+#else
+  void operator|=(Integral m){n|=m;}
+  void operator&=(Integral m){n&=m;}
+#endif
+
+  Integral n;
+};
+
+struct plain_size_control
+{
+  std::size_t ml;
+  std::size_t size;
+};
+
+template<typename,typename,typename,typename>
+class table;
+
+/* table_iterator keeps two pointers:
+ * 
+ *   - A pointer p to the element slot.
+ *   - A pointer pc to the n-th byte of the associated group metadata, where n
+ *     is the position of the element in the group.
+ *
+ * A simpler solution would have been to keep a pointer p to the element, a
+ * pointer pg to the group, and the position n, but that would increase
+ * sizeof(table_iterator) by 4/8 bytes. In order to make this compact
+ * representation feasible, it is required that group objects are aligned
+ * to their size, so that we can recover pg and n as
+ * 
+ *   - n = pc%sizeof(group)
+ *   - pg = pc-n
+ * 
+ * (for explanatory purposes pg and pc are treated above as if they were memory
+ * addresses rather than pointers).
+ * 
+ * p = nullptr is conventionally used to mark end() iterators.
+ */
+
+/* internal conversion from const_iterator to iterator */
+struct const_iterator_cast_tag{}; 
+
+template<typename TypePolicy,typename Group,bool Const>
+class table_iterator
+{
+  using type_policy=TypePolicy;
+  using table_element_type=typename type_policy::element_type;
+  using group_type=Group;
+  static constexpr auto N=group_type::N;
+  static constexpr auto regular_layout=group_type::regular_layout;
+
+public:
+  using difference_type=std::ptrdiff_t;
+  using value_type=typename type_policy::value_type;
+  using pointer=
+    typename std::conditional<Const,value_type const*,value_type*>::type;
+  using reference=
+    typename std::conditional<Const,value_type const&,value_type&>::type;
+  using iterator_category=std::forward_iterator_tag;
+  using element_type=
+    typename std::conditional<Const,value_type const,value_type>::type;
+
+  table_iterator()=default;
+  template<bool Const2,typename std::enable_if<!Const2>::type* =nullptr>
+  table_iterator(const table_iterator<TypePolicy,Group,Const2>& x):
+    pc{x.pc},p{x.p}{}
+  table_iterator(
+    const_iterator_cast_tag, const table_iterator<TypePolicy,Group,true>& x):
+    pc{x.pc},p{x.p}{}
+
+  inline reference operator*()const noexcept
+    {return type_policy::value_from(*p);}
+  inline pointer operator->()const noexcept
+    {return std::addressof(type_policy::value_from(*p));}
+  inline table_iterator& operator++()noexcept{increment();return *this;}
+  inline table_iterator operator++(int)noexcept
+    {auto x=*this;increment();return x;}
+  friend inline bool operator==(
+    const table_iterator& x,const table_iterator& y)
+    {return x.p==y.p;}
+  friend inline bool operator!=(
+    const table_iterator& x,const table_iterator& y)
+    {return !(x==y);}
+
+private:
+  template<typename,typename,bool> friend class table_iterator;
+  template<typename> friend class table_erase_return_type;
+  template<typename,typename,typename,typename> friend class table;
+
+  table_iterator(Group* pg,std::size_t n,const table_element_type* p_):
+    pc{reinterpret_cast<unsigned char*>(const_cast<group_type*>(pg))+n},
+    p{const_cast<table_element_type*>(p_)}
+    {}
+
+  inline void increment()noexcept
+  {
+    BOOST_ASSERT(p!=nullptr);
+    increment(std::integral_constant<bool,regular_layout>{});
+  }
+
+  inline void increment(std::true_type /* regular layout */)noexcept
+  {
+    for(;;){
+      ++p;
+      if(reinterpret_cast<uintptr_t>(pc)%sizeof(group_type)==N-1){
+        pc+=sizeof(group_type)-(N-1);
+        break;
+      }
+      ++pc;
+      if(!group_type::is_occupied(pc))continue;
+      if(BOOST_UNLIKELY(group_type::is_sentinel(pc)))p=nullptr;
+      return;
+    }
+
+    for(;;){
+      int mask=reinterpret_cast<group_type*>(pc)->match_occupied();
+      if(mask!=0){
+        auto n=unchecked_countr_zero(mask);
+        if(BOOST_UNLIKELY(reinterpret_cast<group_type*>(pc)->is_sentinel(n))){
+          p=nullptr;
+        }
+        else{
+          pc+=n;
+          p+=n;
+        }
+        return;
+      }
+      pc+=sizeof(group_type);
+      p+=N;
+    }
+  }
+
+  inline void increment(std::false_type /* interleaved */)noexcept
+  {
+    std::size_t n0=reinterpret_cast<uintptr_t>(pc)%sizeof(group_type);
+    pc-=n0;
+
+    int mask=(
+      reinterpret_cast<group_type*>(pc)->match_occupied()>>(n0+1))<<(n0+1);
+    if(!mask){
+      do{
+        pc+=sizeof(group_type);
+        p+=N;
+      }
+      while((mask=reinterpret_cast<group_type*>(pc)->match_occupied())==0);
+    }
+
+    auto n=unchecked_countr_zero(mask);
+    if(BOOST_UNLIKELY(reinterpret_cast<group_type*>(pc)->is_sentinel(n))){
+      p=nullptr;
+    }
+    else{
+      pc+=n;
+      p-=n0;
+      p+=n;
+    }
+  }
+
+  unsigned char      *pc=nullptr;
+  table_element_type *p=nullptr;
+};
+
+/* Returned by table::erase([const_]iterator) to avoid iterator increment
+ * if discarded.
+ */
+
+template<typename Iterator>
+class table_erase_return_type; 
+
+template<typename TypePolicy,typename Group,bool Const>
+class table_erase_return_type<table_iterator<TypePolicy,Group,Const>>
+{
+  using iterator=table_iterator<TypePolicy,Group,Const>;
+  using const_iterator=table_iterator<TypePolicy,Group,true>;
+
+public:
+  /* can't delete it because VS in pre-C++17 mode needs to see it for RVO */
+  table_erase_return_type(const table_erase_return_type&);
+
+  operator iterator()const noexcept
+  {
+    auto it=pos;
+    it.increment(); /* valid even if *it was erased */
+    return iterator(const_iterator_cast_tag{},it);
+  }
+
+  template<
+    bool dependent_value=false,
+    typename std::enable_if<!Const||dependent_value>::type* =nullptr
+  >
+  operator const_iterator()const noexcept{return this->operator iterator();}
+
+private:
+  template<typename,typename,typename,typename> friend class table;
+
+  table_erase_return_type(const_iterator pos_):pos{pos_}{}
+  table_erase_return_type& operator=(const table_erase_return_type&)=delete;
+
+  const_iterator pos;
+};
+
+/* foa::table interface departs in a number of ways from that of C++ unordered
+ * associative containers because it's not for end-user consumption
+ * (boost::unordered_(flat|node)_(map|set) wrappers complete it as
+ * appropriate).
+ *
+ * The table supports two main modes of operation: flat and node-based. In the
+ * flat case, buckets directly store elements. For node-based, buckets store
+ * pointers to individually heap-allocated elements.
+ *
+ * For both flat and node-based:
+ *
+ *   - begin() is not O(1).
+ *   - No bucket API.
+ *   - Load factor is fixed and can't be set by the user.
+ * 
+ * For flat only:
+ *
+ *   - value_type must be moveable.
+ *   - Pointer stability is not kept under rehashing.
+ *   - No extract API.
+ *
+ * try_emplace, erase and find support heterogeneous lookup by default,
+ * that is, without checking for any ::is_transparent typedefs --the
+ * checking is done by boost::unordered_(flat|node)_(map|set).
+ */
+
+template <typename TypePolicy,typename Hash,typename Pred,typename Allocator>
+using table_core_impl=
+  table_core<TypePolicy,group15<plain_integral>,table_arrays,
+  plain_size_control,Hash,Pred,Allocator>;
+
+#include <boost/unordered/detail/foa/ignore_wshadow.hpp>
+
+#if defined(BOOST_MSVC)
+#pragma warning(push)
+#pragma warning(disable:4714) /* marked as __forceinline not inlined */
+#endif
+
+template<typename TypePolicy,typename Hash,typename Pred,typename Allocator>
+class table:table_core_impl<TypePolicy,Hash,Pred,Allocator>
+{
+  using super=table_core_impl<TypePolicy,Hash,Pred,Allocator>;
+  using type_policy=typename super::type_policy;
+  using group_type=typename super::group_type;
+  using super::N;
+  using prober=typename super::prober;
+  using locator=typename super::locator;
+
+public:
+  using key_type=typename super::key_type;
+  using init_type=typename super::init_type;
+  using value_type=typename super::value_type;
+  using element_type=typename super::element_type;
+
+private:
+  static constexpr bool has_mutable_iterator=
+    !std::is_same<key_type,value_type>::value;
+
+public:
+  using hasher=typename super::hasher;
+  using key_equal=typename super::key_equal;
+  using allocator_type=typename super::allocator_type;
+  using pointer=typename super::pointer;
+  using const_pointer=typename super::const_pointer;
+  using reference=typename super::reference;
+  using const_reference=typename super::const_reference;
+  using size_type=typename super::size_type;
+  using difference_type=typename super::difference_type;
+  using const_iterator=table_iterator<type_policy,group_type,true>;
+  using iterator=typename std::conditional<
+    has_mutable_iterator,
+    table_iterator<type_policy,group_type,false>,
+    const_iterator>::type;
+  using erase_return_type=table_erase_return_type<iterator>;
+
+  table(
+    std::size_t n=default_bucket_count,const Hash& h_=Hash(),
+    const Pred& pred_=Pred(),const Allocator& al_=Allocator()):
+    super{n,h_,pred_,al_}
+    {}
+
+  table(const table& x)=default;
+  table(table&& x)=default;
+  table(const table& x,const Allocator& al_):super{x,al_}{}
+  table(table&& x,const Allocator& al_):super{std::move(x),al_}{}
+  ~table()=default;
+
+  table& operator=(const table& x)=default;
+  table& operator=(table&& x)=default;
+
+  using super::get_allocator;
+
+  iterator begin()noexcept
+  {
+    iterator it{this->arrays.groups,0,this->arrays.elements};
+    if(this->arrays.elements&&
+       !(this->arrays.groups[0].match_occupied()&0x1))++it;
+    return it;
+  }
+
+  const_iterator begin()const noexcept
+                   {return const_cast<table*>(this)->begin();}
+  iterator       end()noexcept{return {};}
+  const_iterator end()const noexcept{return const_cast<table*>(this)->end();}
+  const_iterator cbegin()const noexcept{return begin();}
+  const_iterator cend()const noexcept{return end();}
+
+  using super::empty;
+  using super::size;
+  using super::max_size;
+
+  template<typename... Args>
+  BOOST_FORCEINLINE std::pair<iterator,bool> emplace(Args&&... args)
+  {
+    auto x=alloc_make_insert_type<type_policy>(
+      this->al(),std::forward<Args>(args)...);
+    return emplace_impl(type_policy::move(x.value()));
+  }
+
+  template<typename Key,typename... Args>
+  BOOST_FORCEINLINE std::pair<iterator,bool> try_emplace(
+    Key&& x,Args&&... args)
+  {
+    return emplace_impl(
+      try_emplace_args_t{},std::forward<Key>(x),std::forward<Args>(args)...);
+  }
+
+  BOOST_FORCEINLINE std::pair<iterator,bool>
+  insert(const init_type& x){return emplace_impl(x);}
+
+  BOOST_FORCEINLINE std::pair<iterator,bool>
+  insert(init_type&& x){return emplace_impl(std::move(x));}
+
+  /* template<typename=void> tilts call ambiguities in favor of init_type */
+
+  template<typename=void>
+  BOOST_FORCEINLINE std::pair<iterator,bool>
+  insert(const value_type& x){return emplace_impl(x);}
+
+  template<typename=void>
+  BOOST_FORCEINLINE std::pair<iterator,bool>
+  insert(value_type&& x){return emplace_impl(std::move(x));}
+
+  template<typename T=element_type>
+  BOOST_FORCEINLINE
+  typename std::enable_if<
+    !std::is_same<T,value_type>::value,
+    std::pair<iterator,bool>
+  >::type
+  insert(element_type&& x){return emplace_impl(std::move(x));}
+
+  template<
+    bool dependent_value=false,
+    typename std::enable_if<
+      has_mutable_iterator||dependent_value>::type* =nullptr
+  >
+  erase_return_type erase(iterator pos)noexcept
+  {return erase(const_iterator(pos));}
+
+  BOOST_FORCEINLINE
+  erase_return_type erase(const_iterator pos)noexcept
+  {
+    super::erase(pos.pc,pos.p);
+    return {pos};
+  }
+
+  template<typename Key>
+  BOOST_FORCEINLINE
+  auto erase(Key&& x) -> typename std::enable_if<
+    !std::is_convertible<Key,iterator>::value&&
+    !std::is_convertible<Key,const_iterator>::value, std::size_t>::type
+  {
+    auto it=find(x);
+    if(it!=end()){
+      erase(it);
+      return 1;
+    }
+    else return 0;
+  }
+
+  void swap(table& x)
+    noexcept(noexcept(std::declval<super&>().swap(std::declval<super&>())))
+  {
+    super::swap(x);
+  }
+
+  using super::clear;
+
+  element_type extract(const_iterator pos)
+  {
+    BOOST_ASSERT(pos!=end());
+    erase_on_exit e{*this,pos};
+    (void)e;
+    return std::move(*pos.p);
+  }
+
+  // TODO: should we accept different allocator too?
+  template<typename Hash2,typename Pred2>
+  void merge(table<TypePolicy,Hash2,Pred2,Allocator>& x)
+  {
+    x.for_all_elements([&,this](group_type* pg,unsigned int n,element_type* p){
+      erase_on_exit e{x,{pg,n,p}};
+      if(!emplace_impl(type_policy::move(*p)).second)e.rollback();
+    });
+  }
+
+  template<typename Hash2,typename Pred2>
+  void merge(table<TypePolicy,Hash2,Pred2,Allocator>&& x){merge(x);}
+
+  using super::hash_function;
+  using super::key_eq;
+
+  template<typename Key>
+  BOOST_FORCEINLINE iterator find(const Key& x)
+  {
+    return make_iterator(super::find(x));
+  }
+
+  template<typename Key>
+  BOOST_FORCEINLINE const_iterator find(const Key& x)const
+  {
+    return const_cast<table*>(this)->find(x);
+  }
+
+  using super::capacity;
+  using super::load_factor;
+  using super::max_load_factor;
+  using super::max_load;
+  using super::rehash;
+  using super::reserve;
+
+  template<typename Predicate>
+  friend std::size_t erase_if(table& x,Predicate& pr)
+  {
+    using value_reference=typename std::conditional<
+      std::is_same<key_type,value_type>::value,
+      const_reference,
+      reference
+    >::type;
+
+    std::size_t s=x.size();
+    x.for_all_elements(
+      [&](group_type* pg,unsigned int n,element_type* p){
+        if(pr(const_cast<value_reference>(type_policy::value_from(*p)))){
+          x.super::erase(pg,n,p);
+        }
+      });
+    return std::size_t(s-x.size());
+  }
+
+  friend bool operator==(const table& x,const table& y)
+  {
+    return static_cast<const super&>(x)==static_cast<const super&>(y);
+  }
+
+  friend bool operator!=(const table& x,const table& y){return !(x==y);}
+
+private:
+  struct erase_on_exit
+  {
+    erase_on_exit(table& x_,const_iterator it_):x{x_},it{it_}{}
+    ~erase_on_exit(){if(!rollback_)x.erase(it);}
+
+    void rollback(){rollback_=true;}
+
+    table&         x;
+    const_iterator it;
+    bool           rollback_=false;
+  };
+
+  static inline iterator make_iterator(const locator& l)noexcept
+  {
+    return {l.pg,l.n,l.p};
+  }
+
+  template<typename... Args>
+  BOOST_FORCEINLINE std::pair<iterator,bool> emplace_impl(Args&&... args)
+  {
+    const auto &k=this->key_from(std::forward<Args>(args)...);
+    auto        hash=this->hash_for(k);
+    auto        pos0=this->position_for(hash);
+    auto        loc=super::find(k,pos0,hash);
+
+    if(loc){
+      return {make_iterator(loc),false};
+    }
+    if(BOOST_LIKELY(this->size_ctrl.size<this->size_ctrl.ml)){
+      return {
+        make_iterator(
+          this->unchecked_emplace_at(pos0,hash,std::forward<Args>(args)...)),
+        true
+      };  
+    }
+    else{
+      return {
+        make_iterator(
+          this->unchecked_emplace_with_rehash(
+            hash,std::forward<Args>(args)...)),
+        true
+      };  
+    }
+  }
+};
+
+#if defined(BOOST_MSVC)
+#pragma warning(pop) /* C4714 */
+#endif
+
+#include <boost/unordered/detail/foa/restore_wshadow.hpp>
+
+} /* namespace foa */
+} /* namespace detail */
+} /* namespace unordered */
+} /* namespace boost */
+
+#endif

linx64/include/boost/unordered/detail/foa/tuple_rotate_right.hpp

@@ -0,0 +1,52 @@
+/* Copyright 2023 Joaquin M Lopez Munoz.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_TUPLE_ROTATE_RIGHT_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_TUPLE_ROTATE_RIGHT_HPP
+
+#include <boost/mp11/algorithm.hpp>
+#include <boost/mp11/integer_sequence.hpp>
+#include <tuple>
+#include <utility>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template<typename Tuple>
+using tuple_rotate_right_return_type=mp11::mp_rotate_right_c<
+  typename std::remove_cv<typename std::remove_reference<Tuple>::type>::type,
+  1
+>;
+
+template<std::size_t... Is,typename Tuple>
+tuple_rotate_right_return_type<Tuple>
+tuple_rotate_right_aux(mp11::index_sequence<Is...>,Tuple&& x)
+{
+  return tuple_rotate_right_return_type<Tuple>{
+    std::get<(Is+sizeof...(Is)-1)%sizeof...(Is)>(std::forward<Tuple>(x))...};
+}
+
+template<typename Tuple>
+tuple_rotate_right_return_type<Tuple> tuple_rotate_right(Tuple&& x)
+{
+  using RawTuple=typename std::remove_cv<
+    typename std::remove_reference<Tuple>::type>::type;
+
+  return tuple_rotate_right_aux(
+    mp11::make_index_sequence<std::tuple_size<RawTuple>::value>{},
+    std::forward<Tuple>(x));
+}
+
+} /* namespace foa */
+} /* namespace detail */
+} /* namespace unordered */
+} /* namespace boost */
+
+#endif