Actual source code: object_pool.cxx

  1: #include <petsc/private/cpp/object_pool.hpp>

  3: #include <new>       // std::nothrow
  4: #include <limits>    // std::numeric_limits
  5: #include <algorithm> // std::lower_bound()
  6: #include <cstdio>    // std::printf

  8: namespace Petsc
  9: {

 11: namespace memory
 12: {

 14: // ==========================================================================================
 15: // PoolAllocator -- Private API -- AllocationHeader
 16: //
 17: // The header inserted for each allocated pointer. It stores:
 18: //
 19: // - size - the size (in bytes) of the allocation. This includes ONLY the size as requested by
 20: //          the user. i.e. if a user requested 10 bytes but alignment, padding and header
 21: //          overhead results in the actual allocation being 30 bytes, then size = 10.
 22: // - align - the alignment (in bytes) of the allocated pointer.
 23: // ==========================================================================================

 25: struct PoolAllocator::AllocationHeader {
 26:   constexpr AllocationHeader(size_type, align_type) noexcept;

 28:   PETSC_NODISCARD static constexpr align_type max_alignment() noexcept;
 29:   PETSC_NODISCARD static constexpr size_type  header_size() noexcept;
 30:   PETSC_NODISCARD static constexpr size_type  buffer_zone_size() noexcept;

 32:   size_type  size;
 33:   align_type align;
 34: };

 36: // ==========================================================================================
 37: // PoolAllocator -- Private API -- AllocationHeader -- Public API
 38: // ==========================================================================================

 40: /*
 41:   PoolAllocator::AllocationHeader::AllocationHeader
 42: */
 43: constexpr PoolAllocator::AllocationHeader::AllocationHeader(size_type size, align_type align) noexcept : size{size}, align{align} { }

 45: /*
 46:   PoolAllocator::AllocationHeader::max_alignment

 48:   Returns the maximum supported alignment (in bytes) of the memory pool.
 49: */
 50: constexpr PoolAllocator::align_type PoolAllocator::AllocationHeader::max_alignment() noexcept
 51: {
 52: #if PETSC_CPP_VERSION >= 14
 53:   constexpr auto max_align = std::numeric_limits<unsigned char>::max() + 1;
 54:   static_assert(!(max_align & (max_align - 1)), "Maximum alignment must be a power of 2");
 55:   return static_cast<align_type>(max_align);
 56: #else
 57:   return static_cast<align_type>(std::numeric_limits<unsigned char>::max() + 1);
 58: #endif
 59: }

 61: /*
 62:   PoolAllocator::AllocationHeader::buffer_zone_size

 64:   Notes:
 65:   Returns the number of bytes between the allocated pointer and the location where the
 66:   alignment diff is stored. i.e. size of the buffer zone + 1.

 68:   If ASAN is enabled then this buffer zone is poisoned, so any overrun on part of the user is
 69:   potentially caught. The larger the buffer zone, the more likely that the user lands in
 70:   poisoned memory. Turned off in optimized builds.
 71: */
 72: constexpr PoolAllocator::size_type PoolAllocator::AllocationHeader::buffer_zone_size() noexcept
 73: {
 74:   return (PetscDefined(USE_DEBUG) ? 32 : 0) + 1;
 75: }

 77: /*
 78:   PoolAllocator::AllocationHeader::header_size

 80:   Notes:
 81:   Returns the minimum size of the allocation header in bytes. Essentially (literally) the size
 82:   of the header object + size of the buffer zone. Does not include padding due to alignment
 83:   offset itself though.
 84: */
 85: constexpr PoolAllocator::size_type PoolAllocator::AllocationHeader::header_size() noexcept
 86: {
 87:   return sizeof(AllocationHeader) + buffer_zone_size();
 88: }

 90: /*
 91:   PoolAllocator::total_size_ - Compute the maximum total size for an allocation

 93:   Input Parameters:
 94: + size  - the size (in bytes) requested by the user
 95: - align - the alignment (in bytes) requested by the user

 97:   Notes:
 98:   Returns a size so that std::malloc(total_size_(size, align)) allocates enough memory to store
 99:   the allocation header, buffer zone, alignment offset and requested size including any
100:   potential changes due to alignment.
101: */
102: constexpr PoolAllocator::size_type PoolAllocator::total_size_(size_type size, align_type align) noexcept
103: {
104:   // align - 1 because aligning the pointer up by align bytes just returns it to its original
105:   // alignment, so we can save the byte
106:   return AllocationHeader::header_size() + size + util::to_underlying(align) - 1;
107: }

109: /*
110:   PoolAllocator::delete_ptr_ - deletes a pointer and the corresponding header

112:   Input Parameter:
113: + in_ptr - the pointer to user-memory which to delete

115:   Notes:
116:   in_ptr may point to nullptr (in which case this does nothing), otherwise it must have been
117:   allocated by the pool.

119:   in_ptr may point to poisoned memory, this routine will remove any poisoning before
120:   deallocating.

122:   in_ptr is set to nullptr on return.
123: */
124: PetscErrorCode PoolAllocator::delete_ptr_(void **in_ptr) noexcept
125: {
126:   PetscFunctionBegin;
127:   PetscAssertPointer(in_ptr, 1);
128:   if (const auto ptr = util::exchange(*in_ptr, nullptr)) {
129:     AllocationHeader *header = nullptr;

131:     PetscCall(extract_header_(ptr, &header, false));
132:     // must unpoison the header itself before we can access the members
133:     PetscCall(PetscUnpoisonMemoryRegion(header, sizeof(*header)));
134:     PetscCall(PetscUnpoisonMemoryRegion(header, total_size_(header->size, header->align)));
135:     PetscCallCXX(::delete[] reinterpret_cast<unsigned char *>(header));
136:   }
137:   PetscFunctionReturn(PETSC_SUCCESS);
138: }

140: /*
141:   PoolAllocator::find_align_ - Return an iterator to the memory pool for a particular alignment

143:   Input Parameter:
144: . align - The alignment (in bytes) to search for

146:   Notes:
147:   returns pool().end() if alignment not found.
148: */
149: PoolAllocator::pool_type::iterator PoolAllocator::find_align_(align_type align) noexcept
150: {
151:   return std::lower_bound(this->pool().begin(), this->pool().end(), align, [](const pool_type::value_type &pair, const align_type &align) { return pair.first < align; });
152: }

154: PoolAllocator::pool_type::const_iterator PoolAllocator::find_align_(align_type align) const noexcept
155: {
156:   return std::lower_bound(this->pool().begin(), this->pool().end(), align, [](const pool_type::value_type &pair, const align_type &align) { return pair.first < align; });
157: }

159: /*
160:   PoolAllocator::clear_ - Clear the memory pool

162:   Output Parameter:
163: . remaining - The number of remaining allocations in the pool, nullptr if not needed

165:   Notes:
166:   This will clean up the pool, deallocating any memory checked back into the pool. This does
167:   not delete allocations that were allocated by the pool but not yet returned to it.

169:   remaining is useful in determining if any allocations were "leaked". Suppose an object
170:   internally manages a resource and uses the pool to allocate said resource. On destruction the
171:   object expects to have the pool be empty, i.e. have remaining = 0. This implies all resources
172:   were returned to the pool.
173: */
174: PetscErrorCode PoolAllocator::clear_(size_type *remaining) noexcept
175: {
176:   size_type remain = 0;

178:   PetscFunctionBegin;
179:   if (remaining) PetscAssertPointer(remaining, 1);
180:   // clang-format off
181:   PetscCall(
182:     this->for_each([&](void *&ptr)
183:     {
184:       PetscFunctionBegin;
185:       ++remain;
186:       PetscCall(delete_ptr_(&ptr));
187:       PetscFunctionReturn(PETSC_SUCCESS);
188:     })
189:   );
190:   // clang-format on
191:   PetscCallCXX(this->pool().clear());
192:   if (remaining) *remaining = remain;
193:   PetscFunctionReturn(PETSC_SUCCESS);
194: }

196: /*
197:   PoolAllocator::finalize_ - Routine automatically called during PetscFinalize()

199:   Notes:
200:   This will go through and reap any regions that it owns in the underlying container. If it
201:   owns all regions, it resets the container.

203:   There currently is no way to ensure that objects remaining in the pool aren't leaked, since
204:   this routine cannot actually re-register the pool for finalizations without causing an
205:   infinite loop...

207:   Thus it is up to the owned object to ensure that the pool is properly finalized.
208: */
209: PetscErrorCode PoolAllocator::finalize_() noexcept
210: {
211:   PetscFunctionBegin;
212:   PetscCall(clear_());
213:   PetscFunctionReturn(PETSC_SUCCESS);
214: }

216: // a quick sanity check that the alignment is valid, does nothing in optimized builds
217: PetscErrorCode PoolAllocator::valid_alignment_(align_type in_align) noexcept
218: {
219:   constexpr auto max_align = util::to_underlying(AllocationHeader::max_alignment());
220:   const auto     align     = util::to_underlying(in_align);

222:   PetscFunctionBegin;
223:   PetscAssert((align > 0) && (align <= max_align), PETSC_COMM_SELF, PETSC_ERR_MEMC, "Alignment %zu must be (0, %zu]", align, max_align);
224:   PetscAssert(!(align & (align - 1)), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Alignment %zu must be a power of 2", align);
225:   PetscFunctionReturn(PETSC_SUCCESS);
226: }

228: /*
229:   PoolAllocator::extract_header_ - Extract the header pointer from the aligned pointer

231:   Input Parameters:
232: + user_ptr     - the pointer to the aligned user memory
233: - check_in_ptr - whether to test the validity of aligned_ptr

235:   Output Parameter:
236: . header - the pointer to the header

238:   Notes:
239:   Setting check_in_ptr to false disabled the PetscAssertPointer() check in the function
240:   preamble. This allows the method to be used even if the aligned_ptr is poisoned (for example
241:   when extracting the header from a pointer that is checked into the pool).

243:   aligned_ptr must have been allocated by the pool.

245:   The returned header is still poisoned, the user is responsible for unpoisoning it.
246: */
247: PetscErrorCode PoolAllocator::extract_header_(void *user_ptr, AllocationHeader **header, bool check_in_ptr) noexcept
248: {
249:   PetscFunctionBegin;
250:   if (check_in_ptr) PetscAssertPointer(user_ptr, 1);
251:   PetscAssertPointer(header, 2);
252:   {
253:     //       AllocationHeader::alignment_offset() (at least 1)
254:     //                        |
255:     // header                 |  user_ptr/aligned_ptr
256:     // |                      |          |
257:     // v                  v~~~~~~~~~~~~~~v
258:     // A==============B===C==============D--------------------- ...
259:     // ^~~~~~~~~~~~~~~^^~~^              ^~~~~~~~~~~~~~~~~~~~~~ ...
260:     //       |             \______             user memory
261:     //       |                    buffer_zone_end
262:     // sizeof(AllocationHeader)
263:     //
264:     // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
265:     //                poisoned
266:     //
267:     const auto aligned_ptr     = reinterpret_cast<unsigned char *>(user_ptr);
268:     const auto buffer_zone_end = aligned_ptr - AllocationHeader::buffer_zone_size();

270:     PetscCall(PetscUnpoisonMemoryRegion(buffer_zone_end, sizeof(*buffer_zone_end)));
271:     {
272:       // offset added to original pointer due to alignment, B -> C above (may be zero)
273:       const auto alignment_offset = *buffer_zone_end;

275:       *header = reinterpret_cast<AllocationHeader *>(buffer_zone_end - alignment_offset - sizeof(AllocationHeader));
276:     }
277:     PetscCall(PetscPoisonMemoryRegion(buffer_zone_end, sizeof(*buffer_zone_end)));
278:   }
279:   PetscFunctionReturn(PETSC_SUCCESS);
280: }

282: /*
283:   PoolAllocator::allocate_ptr_ - Allocate a pointer and header given requested size and
284:   alignment

286:   Input Parameters:
287: + size  - the size (in bytes) to allocate
288: - align - the size (in bytes) to align the pointer to

290:   Output Parameter:
291: . ret_ptr - the resulting pointer to user-memory

293:   Notes:
294:   Both size and align must be > 0. align must be a power of 2.
295:   This both allocates the user memory and the corresponding metadata region.
296: */
297: PetscErrorCode PoolAllocator::allocate_ptr_(size_type size, align_type align, void **ret_ptr) noexcept
298: {
299:   constexpr auto header_size = AllocationHeader::header_size();
300:   const auto     total_size  = total_size_(size, align);
301:   const auto     size_before = total_size - header_size;
302:   auto           usable_size = size_before;
303:   void          *aligned_ptr = nullptr;
304:   unsigned char *base_ptr    = nullptr;

306:   PetscFunctionBegin;
307:   PetscAssertPointer(ret_ptr, 1);
308:   PetscCall(valid_alignment_(align));
309:   // memory is laid out as follows:
310:   //
311:   //                            aligned_ptr     ret_ptr (and aligned_ptr after std::align())
312:   // base_ptr     buffer_zone       |     _____/
313:   // |                      |       |    /     user memory
314:   // v                  v~~~~~~~~~~~x~~~v~~~~~~~~~~~~~~~~~~~~~ ...
315:   // A==============B===C===========D===E--------------------- ...
316:   // ^~~~~~~~~~~~~~~^^~~^
317:   //        |            \_________
318:   // sizeof(AllocationHeader)      |
319:   //                               alignment_offset (may be 0)
320:   //
321:   // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
322:   //                poisoned
323:   //                                ^~~~~~~~~~~~~~~~~~~~~~~~~~ ...
324:   //                                          usable_size
325:   // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ...
326:   //                       total_size_()
327:   //
328:   base_ptr = ::new (std::nothrow) unsigned char[total_size];
329:   PetscAssert(base_ptr, PETSC_COMM_SELF, PETSC_ERR_MEM, "operator new() failed to allocate %zu bytes", total_size);
330:   PetscCallCXX(base_ptr = reinterpret_cast<unsigned char *>(util::construct_at(reinterpret_cast<AllocationHeader *>(base_ptr), size, align)));
331:   aligned_ptr = base_ptr + header_size;
332:   // storing to ret_ptr and not aligned_ptr is deliberate! std::align() returns nullptr if it
333:   // fails, so we do not want to clobber aligned_ptr
334:   *ret_ptr = std::align(util::to_underlying(align), size, aligned_ptr, usable_size);
335:   // note usable_size is has now shrunk by alignment_offset
336:   PetscAssert(*ret_ptr, PETSC_COMM_SELF, PETSC_ERR_LIB, "std::align() failed to align pointer %p (size %zu, alignment %zu)", aligned_ptr, size, util::to_underlying(align));
337:   {
338:     constexpr auto max_align        = util::to_underlying(AllocationHeader::max_alignment());
339:     const auto     alignment_offset = size_before - usable_size;

341:     PetscAssert(alignment_offset <= max_align, PETSC_COMM_SELF, PETSC_ERR_MEMC, "Computed alignment offset %zu > maximum allowed alignment %zu", alignment_offset, max_align);
342:     *(reinterpret_cast<unsigned char *>(aligned_ptr) - AllocationHeader::buffer_zone_size()) = static_cast<unsigned char>(alignment_offset);
343:     if (PetscDefined(USE_DEBUG)) {
344:       const auto computed_aligned_ptr = base_ptr + header_size + alignment_offset;

346:       PetscCheck(computed_aligned_ptr == aligned_ptr, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Base pointer %p + header size %zu + alignment offset %zu = %p != aligned pointer %p", static_cast<void *>(base_ptr), header_size, alignment_offset, static_cast<void *>(computed_aligned_ptr), aligned_ptr);
347:     }
348:   }
349:   // Poison the entire region first, then unpoison only the user region. This ensures that
350:   // any extra space on *either* ends of the array are poisoned
351:   PetscCall(PetscPoisonMemoryRegion(base_ptr, total_size));
352:   PetscCall(PetscUnpoisonMemoryRegion(aligned_ptr, size));
353:   PetscFunctionReturn(PETSC_SUCCESS);
354: }

356: // ==========================================================================================
357: // PoolAllocator -- Public API
358: // ==========================================================================================

360: PoolAllocator::~PoolAllocator() noexcept
361: {
362:   size_type leaked{};
363:   PetscBool init;

365:   PetscFunctionBegin;
366:   PetscCallAbort(PETSC_COMM_SELF, clear_(&leaked));
367:   PetscCallAbort(PETSC_COMM_SELF, PetscInitialized(&init));
368:   if (init) PetscCheckAbort(leaked == 0, PETSC_COMM_SELF, PETSC_ERR_MEM_LEAK, "%zu objects remaining in the pool are leaked", leaked);
369:   PetscFunctionReturnVoid();
370: }

372: /*
373:   PoolAllocator::get_attributes - Get the size and alignment of an allocated pointer

375:   Input Parameter:
376: . ptr - the pointer to query

378:   Output Parameters:
379: + size - the size (in bytes) of the allocated area, nullptr if not needed
380: - align - the alignment (in bytes) of the allocated, nullptr if not needed

382:   Note:
383:   ptr must have been allocated by the pool, and is exactly the pointer returned by either
384:   allocate() or try_allocate() (if successful).
385: */
386: PetscErrorCode PoolAllocator::get_attributes(const void *ptr, size_type *size, align_type *align) noexcept
387: {
388:   PetscFunctionBegin;
389:   // ptr may be poisoned, so cannot check it here
390:   // PetscAssertPointer(out_ptr, 1);
391:   if (size) PetscAssertPointer(size, 2);
392:   if (align) PetscAssertPointer(align, 3);
393:   if (PetscLikely(size || align)) {
394:     AllocationHeader *header = nullptr;

396:     PetscCall(extract_header_(const_cast<void *>(ptr), &header, /* check ptr = */ false));
397:     PetscCall(PetscUnpoisonMemoryRegion(header, sizeof(*header)));
398:     if (size) *size = header->size;
399:     if (align) *align = header->align;
400:     PetscCall(PetscPoisonMemoryRegion(header, sizeof(*header)));
401:   }
402:   PetscFunctionReturn(PETSC_SUCCESS);
403: }

405: /*
406:   PoolAllocator::try_allocate - Attempt to allocate memory from the pool

408:   Input Parameter:
409: + size  - the size (in bytes) to attempt to allocate
410: - align - the alignment (in bytes) of the requested allocation

412:   Output Parameters:
413: + out_ptr - the pointer to return the allocated memory in
414: - success - set to true if out_ptr was successfully is allocated

416:   Notes:
417:   Differs from allocate() insofar that this routine does not allocate new memory if it does not
418:   find a suitable memory chunk in the pool.

420:   align must be a power of 2, and > 0.

422:   If size is 0, out_ptr is set to nullptr and success set to false
423: */
424: PetscErrorCode PoolAllocator::try_allocate(void **out_ptr, size_type size, align_type align, bool *success) noexcept
425: {
426:   void *ptr{};
427:   bool  found{};

429:   PetscFunctionBegin;
430:   PetscAssertPointer(out_ptr, 1);
431:   PetscAssertPointer(success, 3);
432:   PetscCall(valid_alignment_(align));
433:   PetscCall(this->register_finalize());
434:   if (PetscLikely(size)) {
435:     const auto align_it = find_align_(align);

437:     if (align_it != this->pool().end() && align_it->first == align) {
438:       auto     &&size_map = align_it->second;
439:       const auto size_it  = size_map.find(size);

441:       if (size_it != size_map.end()) {
442:         auto &&ptr_list = size_it->second;

444:         if (!ptr_list.empty()) {
445:           found = true;
446:           ptr   = ptr_list.back();
447:           PetscCallCXX(ptr_list.pop_back());
448:           PetscCall(PetscUnpoisonMemoryRegion(ptr, size));
449:         }
450:       }
451:     }
452:   }
453:   *out_ptr = ptr;
454:   *success = found;
455:   PetscFunctionReturn(PETSC_SUCCESS);
456: }

458: /*
459:   PoolAllocator::allocate - Allocate a chunk of memory from the pool

461:   Input Parameters:
462: + size  - The size (in bytes) to allocate
463: - align - The alignment (in bytes) to align the allocation to

465:   Output Parameters:
466: + out_ptr             - A pointer containing the beginning of the allocated region
467: - allocated_from_pool - True if the region was allocated from the pool, false otherwise

469:   Notes:
470:   If size is 0, out_ptr is set to nullptr and was_allocated is set to false.
471: */
472: PetscErrorCode PoolAllocator::allocate(void **out_ptr, size_type size, align_type align, bool *allocated_from_pool) noexcept
473: {
474:   bool success{};

476:   PetscFunctionBegin;
477:   PetscAssertPointer(out_ptr, 1);
478:   if (allocated_from_pool) PetscAssertPointer(allocated_from_pool, 3);
479:   PetscCall(try_allocate(out_ptr, size, align, &success));
480:   if (!success) PetscCall(allocate_ptr_(size, align, out_ptr));
481:   if (allocated_from_pool) *allocated_from_pool = success;
482:   PetscFunctionReturn(PETSC_SUCCESS);
483: }

485: /*
486:   PoolAllocate::deallocate - Return a pointer to the pool

488:   Input Parameter:
489: . in_ptr - A pointer to the beginning of the allocated region

491:   Notes:
492:   On success the region pointed to by in_ptr is poisoned. Any further attempts to access
493:   the memory pointed to by in_ptr will result in an error.

495:   in_ptr must have been allocated by the pool, and must point to the beginning of the allocated
496:   region.

498:   The value in_ptr points to may be nullptr, in which case this routine does nothing.
499: */
500: PetscErrorCode PoolAllocator::deallocate(void **in_ptr, size_type size, align_type align) noexcept
501: {
502:   PetscFunctionBegin;
503:   PetscAssertPointer(in_ptr, 1);
504:   if (auto ptr = util::exchange(*in_ptr, nullptr)) {
505:     if (locked_) {
506:       // This is necessary if an object is "reclaimed" within another PetscFinalize()
507:       // registered cleanup after this pool has returned from its finalizer. In this case,
508:       // instead of pushing onto the stack we just delete the pointer directly.
509:       //
510:       // However this path is *only* valid if we have already finalized!
511:       PetscCall(delete_ptr_(&ptr));
512:     } else {
513:       auto it = find_align_(align);

515:       if (it == this->pool().end() || it->first != align) PetscCallCXX(it = this->pool().insert(it, {align, {}}));
516:       PetscCallCXX(it->second[size].emplace_back(ptr));
517:       PetscCall(PetscPoisonMemoryRegion(ptr, size));
518:     }
519:   }
520:   PetscFunctionReturn(PETSC_SUCCESS);
521: }

523: /*
524:   PoolAllocator::unpoison - Unpoison a pool-allocated pointer

526:   Input Parameter:
527: . ptr - the pointer to poison

529:   Output Parameter:
530: . size - the size (in bytes) of the region pointed to by ptr

532:   Notes:
533:   ptr must not be nullptr.

535:   size should be passed to the corresponding repoison() to undo the effects of this
536:   routine.

538:   Using this routine in conjunction with unpoison() allows a user to temporarily push and pop
539:   the poisoning state of a given pointer. The pool does not repoison the pointer for you, so
540:   use at your own risk!
541: */
542: PetscErrorCode PoolAllocator::unpoison(const void *ptr, size_type *size) noexcept
543: {
544:   PetscFunctionBegin;
545:   // ptr may be poisoned, so cannot check it here
546:   // PetscAssertPointer(ptr, 1);
547:   PetscAssertPointer(size, 2);
548:   PetscCall(get_attributes(ptr, size, nullptr));
549:   PetscCall(PetscUnpoisonMemoryRegion(ptr, *size));
550:   PetscFunctionReturn(PETSC_SUCCESS);
551: }

553: /*
554:   PoolAllocator::repoison - Poison a pointer previously unpoisoned via unpoison()

556:   Input Parameters:
557: + ptr  - the pointer to the unpoisoned region
558: - size - the size of the region

560:   Notes:
561:   size must be exactly the value returned by unpoison().

563:   ptr cannot be nullptr
564: */
565: PetscErrorCode PoolAllocator::repoison(const void *ptr, size_type size) noexcept
566: {
567:   PetscFunctionBegin;
568:   PetscAssertPointer(ptr, 1);
569:   PetscCall(PetscPoisonMemoryRegion(ptr, size));
570:   PetscFunctionReturn(PETSC_SUCCESS);
571: }

573: PoolAllocator::LockGuard PoolAllocator::lock_guard() noexcept
574: {
575:   return LockGuard{this};
576: }

578: // ==========================================================================================
579: // PoolAllocated -- Public API
580: // ==========================================================================================

582: void *PoolAllocated::operator new(size_type size) noexcept
583: {
584:   void *ptr{};

586:   PetscFunctionBegin;
587:   PetscCallAbort(PETSC_COMM_SELF, pool().allocate(&ptr, size, static_cast<align_type>(alignof(std::max_align_t))));
588:   PetscFunctionReturn(ptr);
589: }

591: void PoolAllocated::operator delete(void *ptr) noexcept
592: {
593:   PetscFunctionBegin;
594:   if (PetscLikely(ptr)) {
595:     size_type  size{};
596:     align_type align{};

598:     PetscCallAbort(PETSC_COMM_SELF, pool().get_attributes(ptr, &size, &align));
599:     PetscCallAbort(PETSC_COMM_SELF, pool().deallocate(&ptr, size, align));
600:   }
601:   PetscFunctionReturnVoid();
602: }

604: #if PETSC_CPP_VERSION >= 17
605: void *PoolAllocated::operator new(size_type size, std::align_val_t align) noexcept
606: {
607:   void *ptr{};

609:   PetscFunctionBegin;
610:   PetscCallAbort(PETSC_COMM_SELF, pool().allocate(&ptr, size, static_cast<align_type>(align)));
611:   PetscFunctionReturn(ptr);
612: }

614: void PoolAllocated::operator delete(void *ptr, std::align_val_t align) noexcept
615: {
616:   PetscFunctionBegin;
617:   if (PetscLikely(ptr)) {
618:     size_type size{};

620:     PetscCallAbort(PETSC_COMM_SELF, pool().get_attributes(ptr, &size, nullptr));
621:     PetscCallAbort(PETSC_COMM_SELF, pool().deallocate(&ptr, size, static_cast<align_type>(align)));
622:   }
623:   PetscFunctionReturnVoid();
624: }
625: #endif

627: // ==========================================================================================
628: // PoolAllocated -- Protected API
629: // ==========================================================================================

631: PoolAllocated::allocator_type &PoolAllocated::pool() noexcept
632: {
633:   return pool_;
634: }

636: } // namespace memory

638: } // namespace Petsc