pool: use voidptr storage to avoid definition requirements for users

pool.hpp

@@ -8,8 +8,10 @@
 
 #pragma once
 
-#include "debug/assert.hpp"
 #include "cast.hpp"
+#include "debug/assert.hpp"
+#include "parallel/stack.hpp"
+#include "view.hpp"
 
 #include <atomic>
 #include <new>
@@ -18,116 +20,101 @@
 #include <cstdint>
 
 namespace cruft {
-    /// a simple pre-allocated pool for storage of PODs.
+    /// A simple, thread safe, pre-allocated pool allocator.
-    ///
-    /// non-POD types can be stored, but there are no guarantees for calling
-    /// item destructors at pool destruction time.
     template <typename T>
     class pool {
-    protected:
+    private:
-        union node;
+        /// A collection of all unallocated slots.
+        parallel::stack<void*> m_available;
 
-        union node {
+        /// A pointer to the start of the allocated region.
-            alignas(node*) std::atomic<node*> next;
+        void *m_store;
-            alignas(node*) node* raw;
-            alignas(T)     char data[sizeof(T)];
-        };
 
-        static_assert (std::atomic<node*>::is_always_lock_free);
+        /// The total number of items for which storage has been allocated.
-
-        // root address of allocation. used in deletion at destruction time.
-        node* m_head;
-
-        // the next available entry in the linked list
-        std::atomic<node *> m_next;
-
-        // the total number of items that could be stored
         std::size_t m_capacity;
 
-        // the number of items currently stored.
+        // We used to use std::aligned_storage_t and arrays/vectors proper for
-        std::atomic<size_t> m_size;
+        // data storage. But this requires that the user has already defined
+        // ValueT ahead of time (given we need to call sizeof outside a deduced
+        // context).
+        //
+        // We tried a strategy where nodes were a union of ValueT and a
+        // linked-list. However this proved heinously expensive to traverse to
+        // find allocated objects that need to be destroyed when our destructor
+        // is called.
 
     public:
+        ///////////////////////////////////////////////////////////////////////
         pool (const pool&) = delete;
         pool& operator= (const pool&) = delete;
 
+
+        //---------------------------------------------------------------------
         pool (pool &&rhs) noexcept
-            : m_head (nullptr)
+            : m_available (0)
-            , m_next (nullptr)
             , m_capacity (0)
-            , m_size (0)
         {
-            std::swap (m_head, rhs.m_head);
+            std::swap (m_available, rhs.m_available);
-
+            std::swap (m_store,     rhs.m_store);
-            m_next = rhs.m_next.load ();
+            std::swap (m_capacity,  rhs.m_capacity);
-            rhs.m_next = nullptr;
-
-            std::swap (m_capacity, rhs.m_capacity);
-
-            m_size = rhs.m_size.load ();
-            rhs.m_size = 0;
         }
 
+
+        //---------------------------------------------------------------------
         pool& operator= (pool&&);
 
-        explicit
-        pool (std::size_t _capacity):
-            m_capacity (_capacity),
-            m_size (0u)
-        {
-            // allocate the memory and note the base address for deletion in destructor
-            m_next = m_head = new node[m_capacity];
 
-            relink ();
+        //---------------------------------------------------------------------
+        explicit
+        pool (std::size_t _capacity)
+            : m_available (_capacity)
+            , m_store     (::operator new[] (_capacity * sizeof (T), std::align_val_t {alignof (T)}))
+            , m_capacity  (_capacity)
+        {
+            if (!m_store)
+                throw std::bad_alloc ();
+
+            T* elements = reinterpret_cast<T*> (m_store);
+            for (size_t i = 0; i < m_capacity; ++i)
+                m_available.push (elements + i);
         }
 
+
+        //---------------------------------------------------------------------
         ~pool ()
         {
             clear ();
 
-            // don't check if everything's been returned as pools are often used
+            ::operator delete[] (
-            // for PODs which don't need to be destructed via calling release.
+                m_store,
-            delete [] m_head;
+                m_capacity * sizeof (T),
+                std::align_val_t {alignof (T)}
+            );
         }
 
 
-        // Data management
+        ///////////////////////////////////////////////////////////////////////
         [[nodiscard]] T*
         allocate [[gnu::malloc]] [[gnu::returns_nonnull]] (void)
         {
-            // double check we have enough capacity left
+            void *raw;
-            if (!m_next)
+            if (!m_available.pop (&raw))
                 throw std::bad_alloc ();
-            CHECK_LT (m_size, m_capacity);
+            return reinterpret_cast<T*> (raw);
-
-            // unlink the current cursor
-            do {
-                node* curr = m_next;
-                node* soon = curr->next;
-
-                if (m_next.compare_exchange_weak (curr, soon)) {
-                    ++m_size;
-                    return std::launder (cruft::cast::alignment<T*> (curr));
-                }
-            } while (1);
         }
 
+
+        //---------------------------------------------------------------------
         void
-        deallocate (T *base)
+        deallocate (T *cooked)
         {
-            auto soon = cruft::cast::alignment<node*> (base);
+            void *raw = reinterpret_cast<void*> (cooked);
-
+            if (unlikely (!m_available.push (raw)))
-            do {
+                panic (__FUNCTION__);
-                node *curr = m_next;
-                soon->next = curr;
-                if (m_next.compare_exchange_weak (curr, soon)) {
-                    --m_size;
-                    return;
-                }
-            } while (1);
         }
 
 
+        ///////////////////////////////////////////////////////////////////////
         template <typename ...Args>
         T*
         construct (Args &&...args)
@@ -142,6 +129,7 @@ namespace cruft {
         }
 
 
+        //---------------------------------------------------------------------
         void
         destroy (T *ptr)
         {
@@ -150,14 +138,16 @@ namespace cruft {
         }
 
 
+        //---------------------------------------------------------------------
         void destroy (size_t idx)
         {
             return destroy (&(*this)[idx]);
         }
 
 
+        ///////////////////////////////////////////////////////////////////////
         auto capacity (void) const { return m_capacity; }
-        auto size     (void) const { return m_size.load (); }
+        auto size     (void) const { return capacity () - m_available.size (); }
         bool empty    (void) const { return size () == 0; }
         bool full     (void) const { return size () == capacity (); }
 
@@ -173,80 +163,44 @@ namespace cruft {
         /// object for the duration of this call.
         void clear (void)
         {
-            // Create a fake root so that we can always point to the parent
+            auto const valid_queue = m_available.store (
-            // of every node in the system. Hopefully this isn't too large for
+                decltype(m_available)::contract::I_HAVE_LOCKED_THIS_STRUCTURE
-            // the stack.
+            );
-            node container;
+            std::sort (valid_queue.begin (), valid_queue.end ());
-            container.next.store (m_next.load ());
-
-            // Sort the node list. We walk the list, and at each step reparent
-            // the child at the lowest memory address to the cursor.
-            for (node* start = container.raw; start; start = start->raw) {
-                node* parent = start;
-
-                // Find the node whose child is the lowest pointer
-                int count = 0;
-                for (auto cursor = parent; cursor->raw; cursor = cursor->raw) {
-                    ++count;
-                    CHECK_NEQ (cursor->raw, start);
-                    if (cursor->raw < parent)
-                        parent = cursor;
-                }
-
-                // Parent the lowest child to the start of the sorted list
-                auto tmp = start->raw;
-                start->raw = parent->raw;
-
-                // Remove the lowest child from their old parent
-                auto parent_next = parent->raw;
-                parent->raw = parent_next ? parent_next->raw : nullptr;
-
-                // Parent the old successor of the start to the lowest child
-                start->raw = tmp;
-            }
 
             // Now that we've ordered the nodes we can walk the list from
             // start to finish and find nodes that aren't in the free list.
             // Call the destructors on the data contained in these.
-            auto node_cursor = m_next.load (std::memory_order_relaxed);
+            auto node_cursor = valid_queue.begin ();
-            auto data_cursor = m_head;
+            auto data_cursor = reinterpret_cast<T*> (m_store);
+            auto const data_end = data_cursor + m_capacity;
 
-            while (node_cursor) {
+            while (node_cursor != valid_queue.end ()) {
-                while (data_cursor < node_cursor) {
+                while (&*data_cursor < *node_cursor) {
-                    cruft::cast::alignment<T*> (data_cursor->data)->~T ();
+                    reinterpret_cast<T*> (&*data_cursor)->~T ();
                     ++data_cursor;
                 }
 
-                node_cursor = node_cursor->raw;
+                ++node_cursor;
                 ++data_cursor;
             }
 
-            while (data_cursor < m_head + m_capacity) {
+            while (data_cursor != data_end) {
-                cruft::cast::alignment<T*> (data_cursor->data)->~T ();
+                reinterpret_cast<T*> (&*data_cursor)->~T ();
                 ++data_cursor;
             }
 
-            relink ();
+            m_available.clear ();
+            T* elements = reinterpret_cast<T*> (m_store);
+            for (size_t i = 0; i < m_capacity; ++i)
+                m_available.push (elements + i);
         }
 
-    private:
-        void relink (void)
-        {
-            // Reset the allocation cursor to point to the start of the region
-            m_next = m_head;
 
-            // build out the linked list from all the nodes.
+        ///////////////////////////////////////////////////////////////////////
-            for (size_t i = 0; i < m_capacity - 1; ++i)
-                m_next[i].next = m_next + i + 1;
-            m_next[m_capacity - 1].next = nullptr;
-        }
-
-    public:
-        // Indexing
         size_t index (T const *ptr) const
         {
-            CHECK_LIMIT (cruft::cast::alignment<node const*> (ptr), m_head, m_head + m_capacity);
+            return ptr - reinterpret_cast<T*> (m_store);
-            return cruft::cast::alignment<node const*> (ptr) - m_head;
         }
 
 
@@ -255,22 +209,27 @@ namespace cruft {
         /// guaranteed to point to the first _possible_ allocated value;
         /// however it may not be _live_ at any given moment.
         ///
-        /// DO NOT use this pointer for indexing as you will be unable to
+        /// DO NOT use this pointer for indexing as you _may_ be unable to
-        /// account for internal node sizes, alignment, or padding.
+        /// account for internal node sizes, alignment, or padding. This is
-        void      * base (void)      & { return m_head; }
+        /// why the return type is void.
-        void const* base (void) const& { return m_head; }
+        ///
+        /// We may be using one particular representation at the moment but
+        /// stability is not guaranteed at this point.
+        void      * base (void)      & { return m_store; }
+        void const* base (void) const& { return m_store; }
 
+
+        ///////////////////////////////////////////////////////////////////////
         T& operator[] (size_t idx) &
         {
-            CHECK_LIMIT (idx, 0u, capacity ());
+            return reinterpret_cast<T*> (m_store) [idx];
-            return *cruft::cast::alignment<T*> (&m_head[idx].data[0]);
         }
 
 
+        //---------------------------------------------------------------------
         T const& operator[] (size_t idx) const&
         {
-            CHECK_LIMIT (idx, 0u, capacity ());
+            return reinterpret_cast<T*> (m_store) [idx];
-            return *cruft::cast::alignment<T const*> (&m_head[idx].data[0]);
         }
     };
 }