diff --git a/src/util/ralloc.c b/src/util/ralloc.c index 0d57a6045bd..2a0c26db19f 100644 --- a/src/util/ralloc.c +++ b/src/util/ralloc.c @@ -28,6 +28,7 @@ #include #include +#include "util/list.h" #include "util/macros.h" #include "util/u_math.h" #include "util/u_printf.h" @@ -528,6 +529,382 @@ ralloc_vasprintf_rewrite_tail(char **str, size_t *start, const char *fmt, return true; } +/*************************************************************************** + * GC context. + *************************************************************************** + */ + +/* The maximum size of an object that will be allocated specially. + */ +#define MAX_FREELIST_SIZE 512 + +/* Allocations small enough to be allocated from a freelist will be aligned up + * to this size. + */ +#define FREELIST_ALIGNMENT 32 + +#define NUM_FREELIST_BUCKETS (MAX_FREELIST_SIZE / FREELIST_ALIGNMENT) + +/* The size of a slab. */ +#define SLAB_SIZE (32 * 1024) + +#define GC_CANARY 0xAF6B5B72 + +enum gc_flags { + IS_USED = (1 << 0), + CURRENT_GENERATION = (1 << 1), +}; + +typedef struct +{ +#ifndef NDEBUG + /* A canary value used to determine whether a pointer is allocated using gc_alloc. */ + unsigned canary; +#endif + + uint16_t slab_offset; + uint8_t bucket; + uint8_t flags; +} gc_block_header; + +/* This structure is at the start of the slab. Objects inside a slab are + * allocated using a freelist backed by a simple linear allocator. + */ +typedef struct gc_slab { + HEADER_ALIGN + + gc_ctx *ctx; + + /* Objects are allocated using either linear or freelist allocation. "next_available" is the + * pointer used for linear allocation, while "freelist" is the next free object for freelist + * allocation. + */ + char *next_available; + gc_block_header *freelist; + + /* Slabs that handle the same-sized objects. */ + struct list_head link; + + /* Free slabs that handle the same-sized objects. */ + struct list_head free_link; + + /* Number of allocated and free objects, recorded so that we can free the slab if it + * becomes empty or add one to the freelist if it's no longer full. + */ + unsigned num_allocated; + unsigned num_free; +} gc_slab; + +struct gc_ctx { + /* Array of slabs for fixed-size allocations. Each slab tracks allocations + * of specific sized blocks. User allocations are rounded up to the nearest + * fixed size. slabs[N] contains allocations of size + * FREELIST_ALIGNMENT * (N + 1). + */ + struct { + /* List of slabs in this bucket. */ + struct list_head slabs; + + /* List of slabs with free space in this bucket, so we can quickly choose one when + * allocating. + */ + struct list_head free_slabs; + } slabs[NUM_FREELIST_BUCKETS]; + + uint8_t current_gen; + void *rubbish; +}; + +static gc_block_header * +get_gc_header(const void *ptr) +{ + gc_block_header *info = (gc_block_header *) (((char *) ptr) - + sizeof(gc_block_header)); + assert(info->canary == GC_CANARY); + return info; +} + +static gc_block_header * +get_gc_freelist_next(gc_block_header *ptr) +{ + gc_block_header *next; + /* work around possible strict aliasing bug using memcpy */ + memcpy(&next, (void*)(ptr + 1), sizeof(next)); + return next; +} + +static void +set_gc_freelist_next(gc_block_header *ptr, gc_block_header *next) +{ + memcpy((void*)(ptr + 1), &next, sizeof(next)); +} + +static gc_slab * +get_gc_slab(gc_block_header *header) +{ + return (gc_slab *)((char *)header - header->slab_offset); +} + +gc_ctx * +gc_context(const void *parent) +{ + gc_ctx *ctx = rzalloc(parent, gc_ctx); + for (unsigned i = 0; i < NUM_FREELIST_BUCKETS; i++) { + list_inithead(&ctx->slabs[i].slabs); + list_inithead(&ctx->slabs[i].free_slabs); + } + return ctx; +} + +static size_t +gc_bucket_obj_size(unsigned bucket) +{ + return (bucket + 1) * FREELIST_ALIGNMENT; +} + +static unsigned +gc_bucket_for_size(size_t size) +{ + return (size - 1) / FREELIST_ALIGNMENT; +} + +static unsigned +gc_bucket_num_objs(unsigned bucket) +{ + return (SLAB_SIZE - sizeof(gc_slab)) / gc_bucket_obj_size(bucket); +} + +static gc_block_header * +alloc_from_slab(gc_slab *slab, unsigned bucket) +{ + size_t size = gc_bucket_obj_size(bucket); + gc_block_header *header; + if (slab->freelist) { + /* Prioritize already-allocated chunks, since they probably have a page + * backing them. + */ + header = slab->freelist; + slab->freelist = get_gc_freelist_next(slab->freelist); + } else if (slab->next_available + size <= ((char *) slab) + SLAB_SIZE) { + header = (gc_block_header *) slab->next_available; + header->slab_offset = (char *) header - (char *) slab; + header->bucket = bucket; + slab->next_available += size; + } else { + return NULL; + } + + slab->num_allocated++; + slab->num_free--; + if (!slab->num_free) + list_del(&slab->free_link); + return header; +} + +static void +free_slab(gc_slab *slab) +{ + if (list_is_linked(&slab->free_link)) + list_del(&slab->free_link); + list_del(&slab->link); + ralloc_free(slab); +} + +static void +free_from_slab(gc_block_header *header, bool keep_empty_slabs) +{ + gc_slab *slab = get_gc_slab(header); + + if (slab->num_allocated == 1 && !(keep_empty_slabs && list_is_singular(&slab->free_link))) { + /* Free the slab if this is the last object. */ + free_slab(slab); + return; + } else if (slab->num_free == 0) { + list_add(&slab->free_link, &slab->ctx->slabs[header->bucket].free_slabs); + } else { + /* Keep the free list sorted by the number of free objects in ascending order. By prefering to + * allocate from the slab with the fewest free objects, we help free the slabs with many free + * objects. + */ + while (slab->free_link.next != &slab->ctx->slabs[header->bucket].free_slabs && + slab->num_free > list_entry(slab->free_link.next, gc_slab, free_link)->num_free) { + gc_slab *next = list_entry(slab->free_link.next, gc_slab, free_link); + + /* Move "slab" to after "next". */ + list_move_to(&slab->free_link, &next->free_link); + } + } + + set_gc_freelist_next(header, slab->freelist); + slab->freelist = header; + + slab->num_allocated--; + slab->num_free++; +} + +static unsigned +get_slab_size(unsigned bucket) +{ + /* SLAB_SIZE rounded down to a multiple of the object size so that it's not larger than what can + * be used. + */ + unsigned obj_size = gc_bucket_obj_size(bucket); + unsigned num_objs = gc_bucket_num_objs(bucket); + return align64(sizeof(gc_slab) + num_objs * obj_size, alignof(gc_slab)); +} + +static gc_slab * +create_slab(gc_ctx *ctx, unsigned bucket) +{ + gc_slab *slab = ralloc_size(ctx, get_slab_size(bucket)); + if (unlikely(!slab)) + return NULL; + + slab->ctx = ctx; + slab->freelist = NULL; + slab->next_available = (char*)(slab + 1); + slab->num_allocated = 0; + slab->num_free = gc_bucket_num_objs(bucket); + + list_addtail(&slab->link, &ctx->slabs[bucket].slabs); + list_addtail(&slab->free_link, &ctx->slabs[bucket].free_slabs); + + return slab; +} + +void * +gc_alloc_size(gc_ctx *ctx, size_t size, size_t align) +{ + assert(ctx); + assert(util_is_power_of_two_nonzero(align)); + + align = MAX2(align, alignof(gc_block_header)); + + size = align64(size, align); + size += align64(sizeof(gc_block_header), align); + + gc_block_header *header = NULL; + if (size <= MAX_FREELIST_SIZE) { + unsigned bucket = gc_bucket_for_size(size); + if (list_is_empty(&ctx->slabs[bucket].free_slabs) && !create_slab(ctx, bucket)) + return NULL; + gc_slab *slab = list_first_entry(&ctx->slabs[bucket].free_slabs, gc_slab, free_link); + header = alloc_from_slab(slab, bucket); + } else { + header = ralloc_size(ctx, size); + if (unlikely(!header)) + return NULL; + /* Mark the header as allocated directly, so we know to actually free it. */ + header->bucket = NUM_FREELIST_BUCKETS; + } + + header->flags = ctx->current_gen | IS_USED; +#ifndef NDEBUG + header->canary = GC_CANARY; +#endif + + void *ptr = (char *)header + sizeof(gc_block_header); + assert(((uintptr_t)ptr & (align - 1)) == 0); + return ptr; +} + +void * +gc_zalloc_size(gc_ctx *ctx, size_t size, size_t align) +{ + void *ptr = gc_alloc_size(ctx, size, align); + + if (likely(ptr)) + memset(ptr, 0, size); + + return ptr; +} + +void +gc_free(void *ptr) +{ + if (!ptr) + return; + + gc_block_header *header = get_gc_header(ptr); + header->flags &= ~IS_USED; + + if (header->bucket < NUM_FREELIST_BUCKETS) + free_from_slab(header, true); + else + ralloc_free(header); +} + +gc_ctx *gc_get_context(void *ptr) +{ + gc_block_header *header = get_gc_header(ptr); + + if (header->bucket < NUM_FREELIST_BUCKETS) + return get_gc_slab(header)->ctx; + else + return ralloc_parent(header); +} + +void +gc_sweep_start(gc_ctx *ctx) +{ + ctx->current_gen ^= CURRENT_GENERATION; + + ctx->rubbish = ralloc_context(NULL); + ralloc_adopt(ctx->rubbish, ctx); +} + +void +gc_mark_live(gc_ctx *ctx, const void *mem) +{ + gc_block_header *header = get_gc_header(mem); + if (header->bucket < NUM_FREELIST_BUCKETS) + header->flags ^= CURRENT_GENERATION; + else + ralloc_steal(ctx, header); +} + +void +gc_sweep_end(gc_ctx *ctx) +{ + assert(ctx->rubbish); + + for (unsigned i = 0; i < NUM_FREELIST_BUCKETS; i++) { + unsigned obj_size = gc_bucket_obj_size(i); + list_for_each_entry_safe(gc_slab, slab, &ctx->slabs[i].slabs, link) { + if (!slab->num_allocated) { + free_slab(slab); + continue; + } + + for (char *ptr = (char*)(slab + 1); ptr != slab->next_available; ptr += obj_size) { + gc_block_header *header = (gc_block_header *)ptr; + if (!(header->flags & IS_USED)) + continue; + if ((header->flags & CURRENT_GENERATION) == ctx->current_gen) + continue; + + bool last = slab->num_allocated == 1; + + header->flags &= ~IS_USED; + free_from_slab(header, false); + + if (last) + break; + } + } + } + + for (unsigned i = 0; i < NUM_FREELIST_BUCKETS; i++) { + list_for_each_entry(gc_slab, slab, &ctx->slabs[i].slabs, link) { + assert(slab->num_allocated > 0); /* free_from_slab() should free it otherwise */ + ralloc_steal(ctx, slab); + } + } + + ralloc_free(ctx->rubbish); + ctx->rubbish = NULL; +} + /*************************************************************************** * Linear allocator for short-lived allocations. *************************************************************************** diff --git a/src/util/ralloc.h b/src/util/ralloc.h index 857ca5f797f..0fdcb3c309c 100644 --- a/src/util/ralloc.h +++ b/src/util/ralloc.h @@ -478,6 +478,35 @@ bool ralloc_asprintf_append (char **str, const char *fmt, ...) bool ralloc_vasprintf_append(char **str, const char *fmt, va_list args); /// @} +typedef struct gc_ctx gc_ctx; + +/** + * Allocate a new garbage collection context. The children of the + * context are not necessarily ralloc'd pointers and cannot be stolen to a ralloc context. Instead, + * The user should use the mark-and-sweep interface below to free any unused children. Under the + * hood, this restriction lets us manage allocations ourselves, using a freelist. This means that + * GC contexts should be used for scenarios where there are many allocations and frees, most of + * which use only a few different sizes. + */ +gc_ctx *gc_context(const void *parent); + +#define gc_alloc(ctx, type, count) gc_alloc_size(ctx, sizeof(type) * (count), alignof(type)) +#define gc_zalloc(ctx, type, count) gc_zalloc_size(ctx, sizeof(type) * (count), alignof(type)) + +#define gc_alloc_zla(ctx, type, type2, count) \ + gc_alloc_size(ctx, sizeof(type) + sizeof(type2) * (count), MAX2(alignof(type), alignof(type2))) +#define gc_zalloc_zla(ctx, type, type2, count) \ + gc_zalloc_size(ctx, sizeof(type) + sizeof(type2) * (count), MAX2(alignof(type), alignof(type2))) + +void *gc_alloc_size(gc_ctx *ctx, size_t size, size_t align) MALLOCLIKE; +void *gc_zalloc_size(gc_ctx *ctx, size_t size, size_t align) MALLOCLIKE; +void gc_free(void *ptr); +gc_ctx *gc_get_context(void *ptr); + +void gc_sweep_start(gc_ctx *ctx); +void gc_mark_live(gc_ctx *ctx, const void *mem); +void gc_sweep_end(gc_ctx *ctx); + /** * Declare C++ new and delete operators which use ralloc. *