| //===----------------------------------------------------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include <memory> |
| #include <memory_resource> |
| |
| #ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER |
| # include <atomic> |
| #elif !defined(_LIBCPP_HAS_NO_THREADS) |
| # include <mutex> |
| # if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB) |
| # pragma comment(lib, "pthread") |
| # endif |
| #endif |
| |
| _LIBCPP_BEGIN_NAMESPACE_STD |
| |
| namespace pmr { |
| |
| // memory_resource |
| |
| memory_resource::~memory_resource() = default; |
| |
| // new_delete_resource() |
| |
| #ifdef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION |
| static bool is_aligned_to(void* ptr, size_t align) { |
| void* p2 = ptr; |
| size_t space = 1; |
| void* result = std::align(align, 1, p2, space); |
| return (result == ptr); |
| } |
| #endif |
| |
| class _LIBCPP_EXPORTED_FROM_ABI __new_delete_memory_resource_imp : public memory_resource { |
| void* do_allocate(size_t bytes, size_t align) override { |
| #ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION |
| return std::__libcpp_allocate(bytes, align); |
| #else |
| if (bytes == 0) |
| bytes = 1; |
| void* result = std::__libcpp_allocate(bytes, align); |
| if (!is_aligned_to(result, align)) { |
| std::__libcpp_deallocate(result, bytes, align); |
| __throw_bad_alloc(); |
| } |
| return result; |
| #endif |
| } |
| |
| void do_deallocate(void* p, size_t bytes, size_t align) override { std::__libcpp_deallocate(p, bytes, align); } |
| |
| bool do_is_equal(const memory_resource& other) const noexcept override { return &other == this; } |
| }; |
| |
| // null_memory_resource() |
| |
| class _LIBCPP_EXPORTED_FROM_ABI __null_memory_resource_imp : public memory_resource { |
| void* do_allocate(size_t, size_t) override { __throw_bad_alloc(); } |
| void do_deallocate(void*, size_t, size_t) override {} |
| bool do_is_equal(const memory_resource& other) const noexcept override { return &other == this; } |
| }; |
| |
| namespace { |
| |
| union ResourceInitHelper { |
| struct { |
| __new_delete_memory_resource_imp new_delete_res; |
| __null_memory_resource_imp null_res; |
| } resources; |
| char dummy; |
| constexpr ResourceInitHelper() : resources() {} |
| ~ResourceInitHelper() {} |
| }; |
| |
| // Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority |
| // attribute with a value that's reserved for the implementation (we're the implementation). |
| #include "memory_resource_init_helper.h" |
| |
| } // namespace |
| |
| memory_resource* new_delete_resource() noexcept { return &res_init.resources.new_delete_res; } |
| |
| memory_resource* null_memory_resource() noexcept { return &res_init.resources.null_res; } |
| |
| // default_memory_resource() |
| |
| static memory_resource* __default_memory_resource(bool set = false, memory_resource* new_res = nullptr) noexcept { |
| #ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER |
| static constinit atomic<memory_resource*> __res{&res_init.resources.new_delete_res}; |
| if (set) { |
| new_res = new_res ? new_res : new_delete_resource(); |
| // TODO: Can a weaker ordering be used? |
| return std::atomic_exchange_explicit(&__res, new_res, memory_order_acq_rel); |
| } else { |
| return std::atomic_load_explicit(&__res, memory_order_acquire); |
| } |
| #elif !defined(_LIBCPP_HAS_NO_THREADS) |
| static constinit memory_resource* res = &res_init.resources.new_delete_res; |
| static mutex res_lock; |
| if (set) { |
| new_res = new_res ? new_res : new_delete_resource(); |
| lock_guard<mutex> guard(res_lock); |
| memory_resource* old_res = res; |
| res = new_res; |
| return old_res; |
| } else { |
| lock_guard<mutex> guard(res_lock); |
| return res; |
| } |
| #else |
| static constinit memory_resource* res = &res_init.resources.new_delete_res; |
| if (set) { |
| new_res = new_res ? new_res : new_delete_resource(); |
| memory_resource* old_res = res; |
| res = new_res; |
| return old_res; |
| } else { |
| return res; |
| } |
| #endif |
| } |
| |
| memory_resource* get_default_resource() noexcept { return __default_memory_resource(); } |
| |
| memory_resource* set_default_resource(memory_resource* __new_res) noexcept { |
| return __default_memory_resource(true, __new_res); |
| } |
| |
| // 23.12.5, mem.res.pool |
| |
| static size_t roundup(size_t count, size_t alignment) { |
| size_t mask = alignment - 1; |
| return (count + mask) & ~mask; |
| } |
| |
| struct unsynchronized_pool_resource::__adhoc_pool::__chunk_footer { |
| __chunk_footer* __next_; |
| char* __start_; |
| size_t __align_; |
| size_t __allocation_size() { return (reinterpret_cast<char*>(this) - __start_) + sizeof(*this); } |
| }; |
| |
| void unsynchronized_pool_resource::__adhoc_pool::__release_ptr(memory_resource* upstream) { |
| while (__first_ != nullptr) { |
| __chunk_footer* next = __first_->__next_; |
| upstream->deallocate(__first_->__start_, __first_->__allocation_size(), __first_->__align_); |
| __first_ = next; |
| } |
| } |
| |
| void* unsynchronized_pool_resource::__adhoc_pool::__do_allocate(memory_resource* upstream, size_t bytes, size_t align) { |
| const size_t footer_size = sizeof(__chunk_footer); |
| const size_t footer_align = alignof(__chunk_footer); |
| |
| if (align < footer_align) |
| align = footer_align; |
| |
| size_t aligned_capacity = roundup(bytes, footer_align) + footer_size; |
| |
| void* result = upstream->allocate(aligned_capacity, align); |
| |
| __chunk_footer* h = (__chunk_footer*)((char*)result + aligned_capacity - footer_size); |
| h->__next_ = __first_; |
| h->__start_ = (char*)result; |
| h->__align_ = align; |
| __first_ = h; |
| return result; |
| } |
| |
| void unsynchronized_pool_resource::__adhoc_pool::__do_deallocate( |
| memory_resource* upstream, void* p, size_t bytes, size_t align) { |
| _LIBCPP_ASSERT_NON_NULL(__first_ != nullptr, "deallocating a block that was not allocated with this allocator"); |
| if (__first_->__start_ == p) { |
| __chunk_footer* next = __first_->__next_; |
| upstream->deallocate(p, __first_->__allocation_size(), __first_->__align_); |
| __first_ = next; |
| } else { |
| for (__chunk_footer* h = __first_; h->__next_ != nullptr; h = h->__next_) { |
| if (h->__next_->__start_ == p) { |
| __chunk_footer* next = h->__next_->__next_; |
| upstream->deallocate(p, h->__next_->__allocation_size(), h->__next_->__align_); |
| h->__next_ = next; |
| return; |
| } |
| } |
| // The request to deallocate memory ends up being a no-op, likely resulting in a memory leak. |
| _LIBCPP_ASSERT_VALID_DEALLOCATION(false, "deallocating a block that was not allocated with this allocator"); |
| } |
| } |
| |
| class unsynchronized_pool_resource::__fixed_pool { |
| struct __chunk_footer { |
| __chunk_footer* __next_; |
| char* __start_; |
| size_t __align_; |
| size_t __allocation_size() { return (reinterpret_cast<char*>(this) - __start_) + sizeof(*this); } |
| }; |
| |
| struct __vacancy_header { |
| __vacancy_header* __next_vacancy_; |
| }; |
| |
| __chunk_footer* __first_chunk_ = nullptr; |
| __vacancy_header* __first_vacancy_ = nullptr; |
| |
| public: |
| explicit __fixed_pool() = default; |
| |
| void __release_ptr(memory_resource* upstream) { |
| __first_vacancy_ = nullptr; |
| while (__first_chunk_ != nullptr) { |
| __chunk_footer* next = __first_chunk_->__next_; |
| upstream->deallocate(__first_chunk_->__start_, __first_chunk_->__allocation_size(), __first_chunk_->__align_); |
| __first_chunk_ = next; |
| } |
| } |
| |
| void* __try_allocate_from_vacancies() { |
| if (__first_vacancy_ != nullptr) { |
| void* result = __first_vacancy_; |
| __first_vacancy_ = __first_vacancy_->__next_vacancy_; |
| return result; |
| } |
| return nullptr; |
| } |
| |
| void* __allocate_in_new_chunk(memory_resource* upstream, size_t block_size, size_t chunk_size) { |
| _LIBCPP_ASSERT_INTERNAL(chunk_size % block_size == 0, ""); |
| static_assert(__default_alignment >= alignof(std::max_align_t), ""); |
| static_assert(__default_alignment >= alignof(__chunk_footer), ""); |
| static_assert(__default_alignment >= alignof(__vacancy_header), ""); |
| |
| const size_t footer_size = sizeof(__chunk_footer); |
| const size_t footer_align = alignof(__chunk_footer); |
| |
| size_t aligned_capacity = roundup(chunk_size, footer_align) + footer_size; |
| |
| void* result = upstream->allocate(aligned_capacity, __default_alignment); |
| |
| __chunk_footer* h = (__chunk_footer*)((char*)result + aligned_capacity - footer_size); |
| h->__next_ = __first_chunk_; |
| h->__start_ = (char*)result; |
| h->__align_ = __default_alignment; |
| __first_chunk_ = h; |
| |
| if (chunk_size > block_size) { |
| __vacancy_header* last_vh = this->__first_vacancy_; |
| for (size_t i = block_size; i != chunk_size; i += block_size) { |
| __vacancy_header* vh = (__vacancy_header*)((char*)result + i); |
| vh->__next_vacancy_ = last_vh; |
| last_vh = vh; |
| } |
| this->__first_vacancy_ = last_vh; |
| } |
| return result; |
| } |
| |
| void __evacuate(void* p) { |
| __vacancy_header* vh = (__vacancy_header*)(p); |
| vh->__next_vacancy_ = __first_vacancy_; |
| __first_vacancy_ = vh; |
| } |
| |
| size_t __previous_chunk_size_in_bytes() const { return __first_chunk_ ? __first_chunk_->__allocation_size() : 0; } |
| |
| static const size_t __default_alignment = alignof(max_align_t); |
| }; |
| |
| size_t unsynchronized_pool_resource::__pool_block_size(int i) const { return size_t(1) << __log2_pool_block_size(i); } |
| |
| int unsynchronized_pool_resource::__log2_pool_block_size(int i) const { return (i + __log2_smallest_block_size); } |
| |
| int unsynchronized_pool_resource::__pool_index(size_t bytes, size_t align) const { |
| if (align > alignof(std::max_align_t) || bytes > (size_t(1) << __num_fixed_pools_)) |
| return __num_fixed_pools_; |
| else { |
| int i = 0; |
| bytes = (bytes > align) ? bytes : align; |
| bytes -= 1; |
| bytes >>= __log2_smallest_block_size; |
| while (bytes != 0) { |
| bytes >>= 1; |
| i += 1; |
| } |
| return i; |
| } |
| } |
| |
| unsynchronized_pool_resource::unsynchronized_pool_resource(const pool_options& opts, memory_resource* upstream) |
| : __res_(upstream), __fixed_pools_(nullptr) { |
| size_t largest_block_size; |
| if (opts.largest_required_pool_block == 0) |
| largest_block_size = __default_largest_block_size; |
| else if (opts.largest_required_pool_block < __smallest_block_size) |
| largest_block_size = __smallest_block_size; |
| else if (opts.largest_required_pool_block > __max_largest_block_size) |
| largest_block_size = __max_largest_block_size; |
| else |
| largest_block_size = opts.largest_required_pool_block; |
| |
| if (opts.max_blocks_per_chunk == 0) |
| __options_max_blocks_per_chunk_ = __max_blocks_per_chunk; |
| else if (opts.max_blocks_per_chunk < __min_blocks_per_chunk) |
| __options_max_blocks_per_chunk_ = __min_blocks_per_chunk; |
| else if (opts.max_blocks_per_chunk > __max_blocks_per_chunk) |
| __options_max_blocks_per_chunk_ = __max_blocks_per_chunk; |
| else |
| __options_max_blocks_per_chunk_ = opts.max_blocks_per_chunk; |
| |
| __num_fixed_pools_ = 1; |
| size_t capacity = __smallest_block_size; |
| while (capacity < largest_block_size) { |
| capacity <<= 1; |
| __num_fixed_pools_ += 1; |
| } |
| } |
| |
| pool_options unsynchronized_pool_resource::options() const { |
| pool_options p; |
| p.max_blocks_per_chunk = __options_max_blocks_per_chunk_; |
| p.largest_required_pool_block = __pool_block_size(__num_fixed_pools_ - 1); |
| return p; |
| } |
| |
| void unsynchronized_pool_resource::release() { |
| __adhoc_pool_.__release_ptr(__res_); |
| if (__fixed_pools_ != nullptr) { |
| const int n = __num_fixed_pools_; |
| for (int i = 0; i < n; ++i) |
| __fixed_pools_[i].__release_ptr(__res_); |
| __res_->deallocate(__fixed_pools_, __num_fixed_pools_ * sizeof(__fixed_pool), alignof(__fixed_pool)); |
| __fixed_pools_ = nullptr; |
| } |
| } |
| |
| void* unsynchronized_pool_resource::do_allocate(size_t bytes, size_t align) { |
| // A pointer to allocated storage (6.6.4.4.1) with a size of at least bytes. |
| // The size and alignment of the allocated memory shall meet the requirements for |
| // a class derived from memory_resource (23.12). |
| // If the pool selected for a block of size bytes is unable to satisfy the memory request |
| // from its own internal data structures, it will call upstream_resource()->allocate() |
| // to obtain more memory. If bytes is larger than that which the largest pool can handle, |
| // then memory will be allocated using upstream_resource()->allocate(). |
| |
| int i = __pool_index(bytes, align); |
| if (i == __num_fixed_pools_) |
| return __adhoc_pool_.__do_allocate(__res_, bytes, align); |
| else { |
| if (__fixed_pools_ == nullptr) { |
| __fixed_pools_ = |
| (__fixed_pool*)__res_->allocate(__num_fixed_pools_ * sizeof(__fixed_pool), alignof(__fixed_pool)); |
| __fixed_pool* first = __fixed_pools_; |
| __fixed_pool* last = __fixed_pools_ + __num_fixed_pools_; |
| for (__fixed_pool* pool = first; pool != last; ++pool) |
| ::new ((void*)pool) __fixed_pool; |
| } |
| void* result = __fixed_pools_[i].__try_allocate_from_vacancies(); |
| if (result == nullptr) { |
| auto min = [](size_t a, size_t b) { return a < b ? a : b; }; |
| auto max = [](size_t a, size_t b) { return a < b ? b : a; }; |
| |
| size_t prev_chunk_size_in_bytes = __fixed_pools_[i].__previous_chunk_size_in_bytes(); |
| size_t prev_chunk_size_in_blocks = prev_chunk_size_in_bytes >> __log2_pool_block_size(i); |
| |
| size_t chunk_size_in_blocks; |
| |
| if (prev_chunk_size_in_blocks == 0) { |
| size_t min_blocks_per_chunk = max(__min_bytes_per_chunk >> __log2_pool_block_size(i), __min_blocks_per_chunk); |
| chunk_size_in_blocks = min_blocks_per_chunk; |
| } else { |
| static_assert(__max_bytes_per_chunk <= SIZE_MAX - (__max_bytes_per_chunk / 4), "unsigned overflow is possible"); |
| chunk_size_in_blocks = prev_chunk_size_in_blocks + (prev_chunk_size_in_blocks / 4); |
| } |
| |
| size_t max_blocks_per_chunk = |
| min((__max_bytes_per_chunk >> __log2_pool_block_size(i)), |
| min(__max_blocks_per_chunk, __options_max_blocks_per_chunk_)); |
| if (chunk_size_in_blocks > max_blocks_per_chunk) |
| chunk_size_in_blocks = max_blocks_per_chunk; |
| |
| size_t block_size = __pool_block_size(i); |
| |
| size_t chunk_size_in_bytes = (chunk_size_in_blocks << __log2_pool_block_size(i)); |
| result = __fixed_pools_[i].__allocate_in_new_chunk(__res_, block_size, chunk_size_in_bytes); |
| } |
| return result; |
| } |
| } |
| |
| void unsynchronized_pool_resource::do_deallocate(void* p, size_t bytes, size_t align) { |
| // Returns the memory at p to the pool. It is unspecified if, |
| // or under what circumstances, this operation will result in |
| // a call to upstream_resource()->deallocate(). |
| |
| int i = __pool_index(bytes, align); |
| if (i == __num_fixed_pools_) |
| return __adhoc_pool_.__do_deallocate(__res_, p, bytes, align); |
| else { |
| _LIBCPP_ASSERT_NON_NULL( |
| __fixed_pools_ != nullptr, "deallocating a block that was not allocated with this allocator"); |
| __fixed_pools_[i].__evacuate(p); |
| } |
| } |
| |
| bool synchronized_pool_resource::do_is_equal(const memory_resource& other) const noexcept { return &other == this; } |
| |
| // 23.12.6, mem.res.monotonic.buffer |
| |
| static void* align_down(size_t align, size_t size, void*& ptr, size_t& space) { |
| if (size > space) |
| return nullptr; |
| |
| char* p1 = static_cast<char*>(ptr); |
| char* new_ptr = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 - size) & ~(align - 1)); |
| |
| if (new_ptr < (p1 - space)) |
| return nullptr; |
| |
| ptr = new_ptr; |
| space -= p1 - new_ptr; |
| |
| return ptr; |
| } |
| |
| void* monotonic_buffer_resource::__initial_descriptor::__try_allocate_from_chunk(size_t bytes, size_t align) { |
| if (!__cur_) |
| return nullptr; |
| void* new_ptr = static_cast<void*>(__cur_); |
| size_t new_capacity = (__cur_ - __start_); |
| void* aligned_ptr = align_down(align, bytes, new_ptr, new_capacity); |
| if (aligned_ptr != nullptr) |
| __cur_ = static_cast<char*>(new_ptr); |
| return aligned_ptr; |
| } |
| |
| void* monotonic_buffer_resource::__chunk_footer::__try_allocate_from_chunk(size_t bytes, size_t align) { |
| void* new_ptr = static_cast<void*>(__cur_); |
| size_t new_capacity = (__cur_ - __start_); |
| void* aligned_ptr = align_down(align, bytes, new_ptr, new_capacity); |
| if (aligned_ptr != nullptr) |
| __cur_ = static_cast<char*>(new_ptr); |
| return aligned_ptr; |
| } |
| |
| void* monotonic_buffer_resource::do_allocate(size_t bytes, size_t align) { |
| const size_t footer_size = sizeof(__chunk_footer); |
| const size_t footer_align = alignof(__chunk_footer); |
| |
| auto previous_allocation_size = [&]() { |
| if (__chunks_ != nullptr) |
| return __chunks_->__allocation_size(); |
| |
| size_t newsize = (__initial_.__start_ != nullptr) ? (__initial_.__end_ - __initial_.__start_) : __initial_.__size_; |
| |
| return roundup(newsize, footer_align) + footer_size; |
| }; |
| |
| if (void* result = __initial_.__try_allocate_from_chunk(bytes, align)) |
| return result; |
| if (__chunks_ != nullptr) { |
| if (void* result = __chunks_->__try_allocate_from_chunk(bytes, align)) |
| return result; |
| } |
| |
| // Allocate a brand-new chunk. |
| |
| if (align < footer_align) |
| align = footer_align; |
| |
| size_t aligned_capacity = roundup(bytes, footer_align) + footer_size; |
| size_t previous_capacity = previous_allocation_size(); |
| |
| if (aligned_capacity <= previous_capacity) { |
| size_t newsize = 2 * (previous_capacity - footer_size); |
| aligned_capacity = roundup(newsize, footer_align) + footer_size; |
| } |
| |
| char* start = (char*)__res_->allocate(aligned_capacity, align); |
| auto end = start + aligned_capacity - footer_size; |
| __chunk_footer* footer = (__chunk_footer*)(end); |
| footer->__next_ = __chunks_; |
| footer->__start_ = start; |
| footer->__cur_ = end; |
| footer->__align_ = align; |
| __chunks_ = footer; |
| |
| return __chunks_->__try_allocate_from_chunk(bytes, align); |
| } |
| |
| } // namespace pmr |
| |
| _LIBCPP_END_NAMESPACE_STD |