asahi: Split off common BO code into its own file

In preparation for splitting off the macOS backend implementation into
its own file, pull out the shared BO code from agx_device.c into
agx_bo.c.

Signed-off-by: Asahi Lina <lina@asahilina.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/21058>
This commit is contained in:
Asahi Lina 2023-01-11 20:57:27 +09:00 committed by Marge Bot
parent ea285aea8d
commit 2e51ccac82
5 changed files with 279 additions and 238 deletions

268
src/asahi/lib/agx_bo.c Normal file
View file

@ -0,0 +1,268 @@
/*
* Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io>
* Copyright 2019 Collabora, Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "agx_bo.h"
#include <inttypes.h>
#include "agx_device.h"
#include "decode.h"
/* Helper to calculate the bucket index of a BO */
static unsigned
agx_bucket_index(unsigned size)
{
/* Round down to POT to compute a bucket index */
unsigned bucket_index = util_logbase2(size);
/* Clamp to supported buckets. Huge allocations use the largest bucket */
bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET);
/* Reindex from 0 */
return (bucket_index - MIN_BO_CACHE_BUCKET);
}
static struct list_head *
agx_bucket(struct agx_device *dev, unsigned size)
{
return &dev->bo_cache.buckets[agx_bucket_index(size)];
}
static bool
agx_bo_wait(struct agx_bo *bo, int64_t timeout_ns)
{
/* TODO: When we allow parallelism we'll need to implement this for real */
return true;
}
static void
agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo)
{
simple_mtx_assert_locked(&dev->bo_cache.lock);
list_del(&bo->bucket_link);
list_del(&bo->lru_link);
dev->bo_cache.size -= bo->size;
}
/* Tries to fetch a BO of sufficient size with the appropriate flags from the
* BO cache. If it succeeds, it returns that BO and removes the BO from the
* cache. If it fails, it returns NULL signaling the caller to allocate a new
* BO. */
struct agx_bo *
agx_bo_cache_fetch(struct agx_device *dev, size_t size, uint32_t flags,
const bool dontwait)
{
simple_mtx_lock(&dev->bo_cache.lock);
struct list_head *bucket = agx_bucket(dev, size);
struct agx_bo *bo = NULL;
/* Iterate the bucket looking for something suitable */
list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
if (entry->size < size || entry->flags != flags)
continue;
/* If the oldest BO in the cache is busy, likely so is
* everything newer, so bail. */
if (!agx_bo_wait(entry, dontwait ? 0 : INT64_MAX))
break;
/* This one works, use it */
agx_bo_cache_remove_locked(dev, entry);
bo = entry;
break;
}
simple_mtx_unlock(&dev->bo_cache.lock);
return bo;
}
static void
agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec)
{
struct timespec time;
clock_gettime(CLOCK_MONOTONIC, &time);
list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru,
lru_link) {
/* We want all entries that have been used more than 1 sec ago to be
* dropped, others can be kept. Note the <= 2 check and not <= 1. It's
* here to account for the fact that we're only testing ->tv_sec, not
* ->tv_nsec. That means we might keep entries that are between 1 and 2
* seconds old, but we don't really care, as long as unused BOs are
* dropped at some point.
*/
if (time.tv_sec - entry->last_used <= 2)
break;
agx_bo_cache_remove_locked(dev, entry);
agx_bo_free(dev, entry);
}
}
static void
agx_bo_cache_put_locked(struct agx_bo *bo)
{
struct agx_device *dev = bo->dev;
struct list_head *bucket = agx_bucket(dev, bo->size);
struct timespec time;
/* Add us to the bucket */
list_addtail(&bo->bucket_link, bucket);
/* Add us to the LRU list and update the last_used field. */
list_addtail(&bo->lru_link, &dev->bo_cache.lru);
clock_gettime(CLOCK_MONOTONIC, &time);
bo->last_used = time.tv_sec;
/* Update statistics */
dev->bo_cache.size += bo->size;
if (0) {
printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64
"/%" PRIu64 ")\n",
DIV_ROUND_UP(dev->bo_cache.size, 1024),
DIV_ROUND_UP(bo->size, 1024), bo->label, dev->bo_cache.hits,
dev->bo_cache.misses);
}
/* Update label for debug */
bo->label = "Unused (BO cache)";
/* Let's do some cleanup in the BO cache while we hold the lock. */
agx_bo_cache_evict_stale_bos(dev, time.tv_sec);
}
/* Tries to add a BO to the cache. Returns if it was successful */
static bool
agx_bo_cache_put(struct agx_bo *bo)
{
struct agx_device *dev = bo->dev;
if (bo->flags & AGX_BO_SHARED) {
return false;
} else {
simple_mtx_lock(&dev->bo_cache.lock);
agx_bo_cache_put_locked(bo);
simple_mtx_unlock(&dev->bo_cache.lock);
return true;
}
}
void
agx_bo_cache_evict_all(struct agx_device *dev)
{
simple_mtx_lock(&dev->bo_cache.lock);
for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) {
struct list_head *bucket = &dev->bo_cache.buckets[i];
list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
agx_bo_cache_remove_locked(dev, entry);
agx_bo_free(dev, entry);
}
}
simple_mtx_unlock(&dev->bo_cache.lock);
}
void
agx_bo_reference(struct agx_bo *bo)
{
if (bo) {
ASSERTED int count = p_atomic_inc_return(&bo->refcnt);
assert(count != 1);
}
}
void
agx_bo_unreference(struct agx_bo *bo)
{
if (!bo)
return;
/* Don't return to cache if there are still references */
if (p_atomic_dec_return(&bo->refcnt))
return;
struct agx_device *dev = bo->dev;
pthread_mutex_lock(&dev->bo_map_lock);
/* Someone might have imported this BO while we were waiting for the
* lock, let's make sure it's still not referenced before freeing it.
*/
if (p_atomic_read(&bo->refcnt) == 0) {
if (dev->debug & AGX_DBG_TRACE)
agxdecode_track_free(bo);
if (!agx_bo_cache_put(bo))
agx_bo_free(dev, bo);
}
pthread_mutex_unlock(&dev->bo_map_lock);
}
struct agx_bo *
agx_bo_create(struct agx_device *dev, unsigned size, enum agx_bo_flags flags,
const char *label)
{
struct agx_bo *bo;
assert(size > 0);
/* To maximize BO cache usage, don't allocate tiny BOs */
size = ALIGN_POT(size, 16384);
/* See if we have a BO already in the cache */
bo = agx_bo_cache_fetch(dev, size, flags, true);
/* Update stats based on the first attempt to fetch */
if (bo != NULL)
dev->bo_cache.hits++;
else
dev->bo_cache.misses++;
/* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting
* for something in the cache. But if there's no nothing suitable, we should
* flush the cache to make space for the new allocation.
*/
if (!bo)
bo = agx_bo_alloc(dev, size, flags);
if (!bo)
bo = agx_bo_cache_fetch(dev, size, flags, false);
if (!bo) {
agx_bo_cache_evict_all(dev);
bo = agx_bo_alloc(dev, size, flags);
}
if (!bo) {
fprintf(stderr, "BO creation failed\n");
return NULL;
}
bo->label = label;
p_atomic_set(&bo->refcnt, 1);
if (dev->debug & AGX_DBG_TRACE)
agxdecode_track_alloc(bo);
return bo;
}

View file

@ -119,4 +119,11 @@ void agx_bo_unreference(struct agx_bo *bo);
struct agx_bo *agx_bo_import(struct agx_device *dev, int fd);
int agx_bo_export(struct agx_bo *bo);
void agx_bo_free(struct agx_device *dev, struct agx_bo *bo);
struct agx_bo *agx_bo_alloc(struct agx_device *dev, size_t size,
enum agx_bo_flags flags);
struct agx_bo *agx_bo_cache_fetch(struct agx_device *dev, size_t size,
uint32_t flags, const bool dontwait);
void agx_bo_cache_evict_all(struct agx_device *dev);
#endif

View file

@ -31,7 +31,7 @@ unsigned AGX_FAKE_HANDLE = 0;
uint64_t AGX_FAKE_LO = 0;
uint64_t AGX_FAKE_HI = (1ull << 32);
static void
void
agx_bo_free(struct agx_device *dev, struct agx_bo *bo)
{
#if __APPLE__
@ -110,7 +110,7 @@ agx_shmem_alloc(struct agx_device *dev, size_t size, bool cmdbuf)
return bo;
}
static struct agx_bo *
struct agx_bo *
agx_bo_alloc(struct agx_device *dev, size_t size, enum agx_bo_flags flags)
{
struct agx_bo *bo;
@ -182,197 +182,6 @@ agx_bo_alloc(struct agx_device *dev, size_t size, enum agx_bo_flags flags)
return bo;
}
/* Helper to calculate the bucket index of a BO */
static unsigned
agx_bucket_index(unsigned size)
{
/* Round down to POT to compute a bucket index */
unsigned bucket_index = util_logbase2(size);
/* Clamp to supported buckets. Huge allocations use the largest bucket */
bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET);
/* Reindex from 0 */
return (bucket_index - MIN_BO_CACHE_BUCKET);
}
static struct list_head *
agx_bucket(struct agx_device *dev, unsigned size)
{
return &dev->bo_cache.buckets[agx_bucket_index(size)];
}
static bool
agx_bo_wait(struct agx_bo *bo, int64_t timeout_ns)
{
/* TODO: When we allow parallelism we'll need to implement this for real */
return true;
}
static void
agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo)
{
simple_mtx_assert_locked(&dev->bo_cache.lock);
list_del(&bo->bucket_link);
list_del(&bo->lru_link);
dev->bo_cache.size -= bo->size;
}
/* Tries to fetch a BO of sufficient size with the appropriate flags from the
* BO cache. If it succeeds, it returns that BO and removes the BO from the
* cache. If it fails, it returns NULL signaling the caller to allocate a new
* BO. */
static struct agx_bo *
agx_bo_cache_fetch(struct agx_device *dev, size_t size, uint32_t flags,
const bool dontwait)
{
simple_mtx_lock(&dev->bo_cache.lock);
struct list_head *bucket = agx_bucket(dev, size);
struct agx_bo *bo = NULL;
/* Iterate the bucket looking for something suitable */
list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
if (entry->size < size || entry->flags != flags)
continue;
/* If the oldest BO in the cache is busy, likely so is
* everything newer, so bail. */
if (!agx_bo_wait(entry, dontwait ? 0 : INT64_MAX))
break;
/* This one works, use it */
agx_bo_cache_remove_locked(dev, entry);
bo = entry;
break;
}
simple_mtx_unlock(&dev->bo_cache.lock);
return bo;
}
static void
agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec)
{
list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru,
lru_link) {
/* We want all entries that have been used more than 1 sec ago to be
* dropped, others can be kept. Note the <= 2 check and not <= 1. It's
* here to account for the fact that we're only testing ->tv_sec, not
* ->tv_nsec. That means we might keep entries that are between 1 and 2
* seconds old, but we don't really care, as long as unused BOs are
* dropped at some point.
*/
if (tv_sec - entry->last_used <= 2)
break;
agx_bo_cache_remove_locked(dev, entry);
agx_bo_free(dev, entry);
}
}
static void
agx_bo_cache_put_locked(struct agx_bo *bo)
{
struct agx_device *dev = bo->dev;
struct list_head *bucket = agx_bucket(dev, bo->size);
struct timespec time;
/* Add us to the bucket */
list_addtail(&bo->bucket_link, bucket);
/* Add us to the LRU list and update the last_used field. */
list_addtail(&bo->lru_link, &dev->bo_cache.lru);
clock_gettime(CLOCK_MONOTONIC, &time);
bo->last_used = time.tv_sec;
/* Update statistics */
dev->bo_cache.size += bo->size;
if (0) {
printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64
"/%" PRIu64 ")\n",
DIV_ROUND_UP(dev->bo_cache.size, 1024),
DIV_ROUND_UP(bo->size, 1024), bo->label, dev->bo_cache.hits,
dev->bo_cache.misses);
}
/* Update label for debug */
bo->label = "Unused (BO cache)";
/* Let's do some cleanup in the BO cache while we hold the lock. */
agx_bo_cache_evict_stale_bos(dev, time.tv_sec);
}
/* Tries to add a BO to the cache. Returns if it was successful */
static bool
agx_bo_cache_put(struct agx_bo *bo)
{
struct agx_device *dev = bo->dev;
if (bo->flags & AGX_BO_SHARED) {
return false;
} else {
simple_mtx_lock(&dev->bo_cache.lock);
agx_bo_cache_put_locked(bo);
simple_mtx_unlock(&dev->bo_cache.lock);
return true;
}
}
static void
agx_bo_cache_evict_all(struct agx_device *dev)
{
simple_mtx_lock(&dev->bo_cache.lock);
for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) {
struct list_head *bucket = &dev->bo_cache.buckets[i];
list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
agx_bo_cache_remove_locked(dev, entry);
agx_bo_free(dev, entry);
}
}
simple_mtx_unlock(&dev->bo_cache.lock);
}
void
agx_bo_reference(struct agx_bo *bo)
{
if (bo) {
ASSERTED int count = p_atomic_inc_return(&bo->refcnt);
assert(count != 1);
}
}
void
agx_bo_unreference(struct agx_bo *bo)
{
if (!bo)
return;
/* Don't return to cache if there are still references */
if (p_atomic_dec_return(&bo->refcnt))
return;
struct agx_device *dev = bo->dev;
pthread_mutex_lock(&dev->bo_map_lock);
/* Someone might have imported this BO while we were waiting for the
* lock, let's make sure it's still not referenced before freeing it.
*/
if (p_atomic_read(&bo->refcnt) == 0) {
if (dev->debug & AGX_DBG_TRACE)
agxdecode_track_free(bo);
if (!agx_bo_cache_put(bo))
agx_bo_free(dev, bo);
}
pthread_mutex_unlock(&dev->bo_map_lock);
}
struct agx_bo *
agx_bo_import(struct agx_device *dev, int fd)
{
@ -387,51 +196,6 @@ agx_bo_export(struct agx_bo *bo)
unreachable("Linux UAPI not yet upstream");
}
struct agx_bo *
agx_bo_create(struct agx_device *dev, unsigned size, enum agx_bo_flags flags,
const char *label)
{
struct agx_bo *bo;
assert(size > 0);
/* To maximize BO cache usage, don't allocate tiny BOs */
size = ALIGN_POT(size, 16384);
/* See if we have a BO already in the cache */
bo = agx_bo_cache_fetch(dev, size, flags, true);
/* Update stats based on the first attempt to fetch */
if (bo != NULL)
dev->bo_cache.hits++;
else
dev->bo_cache.misses++;
/* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting
* for something in the cache. But if there's no nothing suitable, we should
* flush the cache to make space for the new allocation.
*/
if (!bo)
bo = agx_bo_alloc(dev, size, flags);
if (!bo)
bo = agx_bo_cache_fetch(dev, size, flags, false);
if (!bo) {
agx_bo_cache_evict_all(dev);
bo = agx_bo_alloc(dev, size, flags);
}
if (!bo) {
fprintf(stderr, "BO creation failed\n");
return NULL;
}
bo->label = label;
p_atomic_set(&bo->refcnt, 1);
if (dev->debug & AGX_DBG_TRACE)
agxdecode_track_alloc(bo);
return bo;
}
static void
agx_get_global_ids(struct agx_device *dev)

View file

@ -26,6 +26,7 @@
#include "util/simple_mtx.h"
#include "util/sparse_array.h"
#include "agx_bo.h"
#include "agx_formats.h"
#include "io.h"

View file

@ -22,6 +22,7 @@
dep_iokit = dependency('IOKit', required : false)
libasahi_lib_files = files(
'agx_bo.c',
'agx_device.c',
'agx_formats.c',
'agx_meta.c',