From 2e51ccac82a4cb7f3d0c6f522b3c594c3980c1b2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 11 Jan 2023 20:57:27 +0900 Subject: [PATCH] asahi: Split off common BO code into its own file In preparation for splitting off the macOS backend implementation into its own file, pull out the shared BO code from agx_device.c into agx_bo.c. Signed-off-by: Asahi Lina Part-of: --- src/asahi/lib/agx_bo.c | 268 +++++++++++++++++++++++++++++++++++++ src/asahi/lib/agx_bo.h | 7 + src/asahi/lib/agx_device.c | 240 +-------------------------------- src/asahi/lib/agx_device.h | 1 + src/asahi/lib/meson.build | 1 + 5 files changed, 279 insertions(+), 238 deletions(-) create mode 100644 src/asahi/lib/agx_bo.c diff --git a/src/asahi/lib/agx_bo.c b/src/asahi/lib/agx_bo.c new file mode 100644 index 00000000000..af34cd23a17 --- /dev/null +++ b/src/asahi/lib/agx_bo.c @@ -0,0 +1,268 @@ +/* + * Copyright (C) 2021 Alyssa Rosenzweig + * Copyright 2019 Collabora, Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "agx_bo.h" +#include +#include "agx_device.h" +#include "decode.h" + +/* Helper to calculate the bucket index of a BO */ +static unsigned +agx_bucket_index(unsigned size) +{ + /* Round down to POT to compute a bucket index */ + unsigned bucket_index = util_logbase2(size); + + /* Clamp to supported buckets. Huge allocations use the largest bucket */ + bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET); + + /* Reindex from 0 */ + return (bucket_index - MIN_BO_CACHE_BUCKET); +} + +static struct list_head * +agx_bucket(struct agx_device *dev, unsigned size) +{ + return &dev->bo_cache.buckets[agx_bucket_index(size)]; +} + +static bool +agx_bo_wait(struct agx_bo *bo, int64_t timeout_ns) +{ + /* TODO: When we allow parallelism we'll need to implement this for real */ + return true; +} + +static void +agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo) +{ + simple_mtx_assert_locked(&dev->bo_cache.lock); + list_del(&bo->bucket_link); + list_del(&bo->lru_link); + dev->bo_cache.size -= bo->size; +} + +/* Tries to fetch a BO of sufficient size with the appropriate flags from the + * BO cache. If it succeeds, it returns that BO and removes the BO from the + * cache. If it fails, it returns NULL signaling the caller to allocate a new + * BO. */ + +struct agx_bo * +agx_bo_cache_fetch(struct agx_device *dev, size_t size, uint32_t flags, + const bool dontwait) +{ + simple_mtx_lock(&dev->bo_cache.lock); + struct list_head *bucket = agx_bucket(dev, size); + struct agx_bo *bo = NULL; + + /* Iterate the bucket looking for something suitable */ + list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) { + if (entry->size < size || entry->flags != flags) + continue; + + /* If the oldest BO in the cache is busy, likely so is + * everything newer, so bail. */ + if (!agx_bo_wait(entry, dontwait ? 0 : INT64_MAX)) + break; + + /* This one works, use it */ + agx_bo_cache_remove_locked(dev, entry); + bo = entry; + break; + } + simple_mtx_unlock(&dev->bo_cache.lock); + + return bo; +} + +static void +agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec) +{ + struct timespec time; + + clock_gettime(CLOCK_MONOTONIC, &time); + list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru, + lru_link) { + /* We want all entries that have been used more than 1 sec ago to be + * dropped, others can be kept. Note the <= 2 check and not <= 1. It's + * here to account for the fact that we're only testing ->tv_sec, not + * ->tv_nsec. That means we might keep entries that are between 1 and 2 + * seconds old, but we don't really care, as long as unused BOs are + * dropped at some point. + */ + if (time.tv_sec - entry->last_used <= 2) + break; + + agx_bo_cache_remove_locked(dev, entry); + agx_bo_free(dev, entry); + } +} + +static void +agx_bo_cache_put_locked(struct agx_bo *bo) +{ + struct agx_device *dev = bo->dev; + struct list_head *bucket = agx_bucket(dev, bo->size); + struct timespec time; + + /* Add us to the bucket */ + list_addtail(&bo->bucket_link, bucket); + + /* Add us to the LRU list and update the last_used field. */ + list_addtail(&bo->lru_link, &dev->bo_cache.lru); + clock_gettime(CLOCK_MONOTONIC, &time); + bo->last_used = time.tv_sec; + + /* Update statistics */ + dev->bo_cache.size += bo->size; + + if (0) { + printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64 + "/%" PRIu64 ")\n", + DIV_ROUND_UP(dev->bo_cache.size, 1024), + DIV_ROUND_UP(bo->size, 1024), bo->label, dev->bo_cache.hits, + dev->bo_cache.misses); + } + + /* Update label for debug */ + bo->label = "Unused (BO cache)"; + + /* Let's do some cleanup in the BO cache while we hold the lock. */ + agx_bo_cache_evict_stale_bos(dev, time.tv_sec); +} + +/* Tries to add a BO to the cache. Returns if it was successful */ +static bool +agx_bo_cache_put(struct agx_bo *bo) +{ + struct agx_device *dev = bo->dev; + + if (bo->flags & AGX_BO_SHARED) { + return false; + } else { + simple_mtx_lock(&dev->bo_cache.lock); + agx_bo_cache_put_locked(bo); + simple_mtx_unlock(&dev->bo_cache.lock); + + return true; + } +} + +void +agx_bo_cache_evict_all(struct agx_device *dev) +{ + simple_mtx_lock(&dev->bo_cache.lock); + for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) { + struct list_head *bucket = &dev->bo_cache.buckets[i]; + + list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) { + agx_bo_cache_remove_locked(dev, entry); + agx_bo_free(dev, entry); + } + } + simple_mtx_unlock(&dev->bo_cache.lock); +} + +void +agx_bo_reference(struct agx_bo *bo) +{ + if (bo) { + ASSERTED int count = p_atomic_inc_return(&bo->refcnt); + assert(count != 1); + } +} + +void +agx_bo_unreference(struct agx_bo *bo) +{ + if (!bo) + return; + + /* Don't return to cache if there are still references */ + if (p_atomic_dec_return(&bo->refcnt)) + return; + + struct agx_device *dev = bo->dev; + + pthread_mutex_lock(&dev->bo_map_lock); + + /* Someone might have imported this BO while we were waiting for the + * lock, let's make sure it's still not referenced before freeing it. + */ + if (p_atomic_read(&bo->refcnt) == 0) { + if (dev->debug & AGX_DBG_TRACE) + agxdecode_track_free(bo); + + if (!agx_bo_cache_put(bo)) + agx_bo_free(dev, bo); + } + + pthread_mutex_unlock(&dev->bo_map_lock); +} + +struct agx_bo * +agx_bo_create(struct agx_device *dev, unsigned size, enum agx_bo_flags flags, + const char *label) +{ + struct agx_bo *bo; + assert(size > 0); + + /* To maximize BO cache usage, don't allocate tiny BOs */ + size = ALIGN_POT(size, 16384); + + /* See if we have a BO already in the cache */ + bo = agx_bo_cache_fetch(dev, size, flags, true); + + /* Update stats based on the first attempt to fetch */ + if (bo != NULL) + dev->bo_cache.hits++; + else + dev->bo_cache.misses++; + + /* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting + * for something in the cache. But if there's no nothing suitable, we should + * flush the cache to make space for the new allocation. + */ + if (!bo) + bo = agx_bo_alloc(dev, size, flags); + if (!bo) + bo = agx_bo_cache_fetch(dev, size, flags, false); + if (!bo) { + agx_bo_cache_evict_all(dev); + bo = agx_bo_alloc(dev, size, flags); + } + + if (!bo) { + fprintf(stderr, "BO creation failed\n"); + return NULL; + } + + bo->label = label; + p_atomic_set(&bo->refcnt, 1); + + if (dev->debug & AGX_DBG_TRACE) + agxdecode_track_alloc(bo); + + return bo; +} diff --git a/src/asahi/lib/agx_bo.h b/src/asahi/lib/agx_bo.h index c0c8b4173af..4a735319777 100644 --- a/src/asahi/lib/agx_bo.h +++ b/src/asahi/lib/agx_bo.h @@ -119,4 +119,11 @@ void agx_bo_unreference(struct agx_bo *bo); struct agx_bo *agx_bo_import(struct agx_device *dev, int fd); int agx_bo_export(struct agx_bo *bo); +void agx_bo_free(struct agx_device *dev, struct agx_bo *bo); +struct agx_bo *agx_bo_alloc(struct agx_device *dev, size_t size, + enum agx_bo_flags flags); +struct agx_bo *agx_bo_cache_fetch(struct agx_device *dev, size_t size, + uint32_t flags, const bool dontwait); +void agx_bo_cache_evict_all(struct agx_device *dev); + #endif diff --git a/src/asahi/lib/agx_device.c b/src/asahi/lib/agx_device.c index ba66a19794b..93d252a5c33 100644 --- a/src/asahi/lib/agx_device.c +++ b/src/asahi/lib/agx_device.c @@ -31,7 +31,7 @@ unsigned AGX_FAKE_HANDLE = 0; uint64_t AGX_FAKE_LO = 0; uint64_t AGX_FAKE_HI = (1ull << 32); -static void +void agx_bo_free(struct agx_device *dev, struct agx_bo *bo) { #if __APPLE__ @@ -110,7 +110,7 @@ agx_shmem_alloc(struct agx_device *dev, size_t size, bool cmdbuf) return bo; } -static struct agx_bo * +struct agx_bo * agx_bo_alloc(struct agx_device *dev, size_t size, enum agx_bo_flags flags) { struct agx_bo *bo; @@ -182,197 +182,6 @@ agx_bo_alloc(struct agx_device *dev, size_t size, enum agx_bo_flags flags) return bo; } -/* Helper to calculate the bucket index of a BO */ -static unsigned -agx_bucket_index(unsigned size) -{ - /* Round down to POT to compute a bucket index */ - unsigned bucket_index = util_logbase2(size); - - /* Clamp to supported buckets. Huge allocations use the largest bucket */ - bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET); - - /* Reindex from 0 */ - return (bucket_index - MIN_BO_CACHE_BUCKET); -} - -static struct list_head * -agx_bucket(struct agx_device *dev, unsigned size) -{ - return &dev->bo_cache.buckets[agx_bucket_index(size)]; -} - -static bool -agx_bo_wait(struct agx_bo *bo, int64_t timeout_ns) -{ - /* TODO: When we allow parallelism we'll need to implement this for real */ - return true; -} - -static void -agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo) -{ - simple_mtx_assert_locked(&dev->bo_cache.lock); - list_del(&bo->bucket_link); - list_del(&bo->lru_link); - dev->bo_cache.size -= bo->size; -} - -/* Tries to fetch a BO of sufficient size with the appropriate flags from the - * BO cache. If it succeeds, it returns that BO and removes the BO from the - * cache. If it fails, it returns NULL signaling the caller to allocate a new - * BO. */ - -static struct agx_bo * -agx_bo_cache_fetch(struct agx_device *dev, size_t size, uint32_t flags, - const bool dontwait) -{ - simple_mtx_lock(&dev->bo_cache.lock); - struct list_head *bucket = agx_bucket(dev, size); - struct agx_bo *bo = NULL; - - /* Iterate the bucket looking for something suitable */ - list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) { - if (entry->size < size || entry->flags != flags) - continue; - - /* If the oldest BO in the cache is busy, likely so is - * everything newer, so bail. */ - if (!agx_bo_wait(entry, dontwait ? 0 : INT64_MAX)) - break; - - /* This one works, use it */ - agx_bo_cache_remove_locked(dev, entry); - bo = entry; - break; - } - simple_mtx_unlock(&dev->bo_cache.lock); - - return bo; -} - -static void -agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec) -{ - list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru, - lru_link) { - /* We want all entries that have been used more than 1 sec ago to be - * dropped, others can be kept. Note the <= 2 check and not <= 1. It's - * here to account for the fact that we're only testing ->tv_sec, not - * ->tv_nsec. That means we might keep entries that are between 1 and 2 - * seconds old, but we don't really care, as long as unused BOs are - * dropped at some point. - */ - if (tv_sec - entry->last_used <= 2) - break; - - agx_bo_cache_remove_locked(dev, entry); - agx_bo_free(dev, entry); - } -} - -static void -agx_bo_cache_put_locked(struct agx_bo *bo) -{ - struct agx_device *dev = bo->dev; - struct list_head *bucket = agx_bucket(dev, bo->size); - struct timespec time; - - /* Add us to the bucket */ - list_addtail(&bo->bucket_link, bucket); - - /* Add us to the LRU list and update the last_used field. */ - list_addtail(&bo->lru_link, &dev->bo_cache.lru); - clock_gettime(CLOCK_MONOTONIC, &time); - bo->last_used = time.tv_sec; - - /* Update statistics */ - dev->bo_cache.size += bo->size; - - if (0) { - printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64 - "/%" PRIu64 ")\n", - DIV_ROUND_UP(dev->bo_cache.size, 1024), - DIV_ROUND_UP(bo->size, 1024), bo->label, dev->bo_cache.hits, - dev->bo_cache.misses); - } - - /* Update label for debug */ - bo->label = "Unused (BO cache)"; - - /* Let's do some cleanup in the BO cache while we hold the lock. */ - agx_bo_cache_evict_stale_bos(dev, time.tv_sec); -} - -/* Tries to add a BO to the cache. Returns if it was successful */ -static bool -agx_bo_cache_put(struct agx_bo *bo) -{ - struct agx_device *dev = bo->dev; - - if (bo->flags & AGX_BO_SHARED) { - return false; - } else { - simple_mtx_lock(&dev->bo_cache.lock); - agx_bo_cache_put_locked(bo); - simple_mtx_unlock(&dev->bo_cache.lock); - - return true; - } -} - -static void -agx_bo_cache_evict_all(struct agx_device *dev) -{ - simple_mtx_lock(&dev->bo_cache.lock); - for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) { - struct list_head *bucket = &dev->bo_cache.buckets[i]; - - list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) { - agx_bo_cache_remove_locked(dev, entry); - agx_bo_free(dev, entry); - } - } - simple_mtx_unlock(&dev->bo_cache.lock); -} - -void -agx_bo_reference(struct agx_bo *bo) -{ - if (bo) { - ASSERTED int count = p_atomic_inc_return(&bo->refcnt); - assert(count != 1); - } -} - -void -agx_bo_unreference(struct agx_bo *bo) -{ - if (!bo) - return; - - /* Don't return to cache if there are still references */ - if (p_atomic_dec_return(&bo->refcnt)) - return; - - struct agx_device *dev = bo->dev; - - pthread_mutex_lock(&dev->bo_map_lock); - - /* Someone might have imported this BO while we were waiting for the - * lock, let's make sure it's still not referenced before freeing it. - */ - if (p_atomic_read(&bo->refcnt) == 0) { - if (dev->debug & AGX_DBG_TRACE) - agxdecode_track_free(bo); - - if (!agx_bo_cache_put(bo)) - agx_bo_free(dev, bo); - } - - pthread_mutex_unlock(&dev->bo_map_lock); -} - struct agx_bo * agx_bo_import(struct agx_device *dev, int fd) { @@ -387,51 +196,6 @@ agx_bo_export(struct agx_bo *bo) unreachable("Linux UAPI not yet upstream"); } -struct agx_bo * -agx_bo_create(struct agx_device *dev, unsigned size, enum agx_bo_flags flags, - const char *label) -{ - struct agx_bo *bo; - assert(size > 0); - - /* To maximize BO cache usage, don't allocate tiny BOs */ - size = ALIGN_POT(size, 16384); - - /* See if we have a BO already in the cache */ - bo = agx_bo_cache_fetch(dev, size, flags, true); - - /* Update stats based on the first attempt to fetch */ - if (bo != NULL) - dev->bo_cache.hits++; - else - dev->bo_cache.misses++; - - /* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting - * for something in the cache. But if there's no nothing suitable, we should - * flush the cache to make space for the new allocation. - */ - if (!bo) - bo = agx_bo_alloc(dev, size, flags); - if (!bo) - bo = agx_bo_cache_fetch(dev, size, flags, false); - if (!bo) { - agx_bo_cache_evict_all(dev); - bo = agx_bo_alloc(dev, size, flags); - } - - if (!bo) { - fprintf(stderr, "BO creation failed\n"); - return NULL; - } - - bo->label = label; - p_atomic_set(&bo->refcnt, 1); - - if (dev->debug & AGX_DBG_TRACE) - agxdecode_track_alloc(bo); - - return bo; -} static void agx_get_global_ids(struct agx_device *dev) diff --git a/src/asahi/lib/agx_device.h b/src/asahi/lib/agx_device.h index e40fd16d4ce..c1c5c400639 100644 --- a/src/asahi/lib/agx_device.h +++ b/src/asahi/lib/agx_device.h @@ -26,6 +26,7 @@ #include "util/simple_mtx.h" #include "util/sparse_array.h" +#include "agx_bo.h" #include "agx_formats.h" #include "io.h" diff --git a/src/asahi/lib/meson.build b/src/asahi/lib/meson.build index 5bff002b442..c695f429fee 100644 --- a/src/asahi/lib/meson.build +++ b/src/asahi/lib/meson.build @@ -22,6 +22,7 @@ dep_iokit = dependency('IOKit', required : false) libasahi_lib_files = files( + 'agx_bo.c', 'agx_device.c', 'agx_formats.c', 'agx_meta.c',