diff --git a/linux-core/ttm/ttm_bo.c b/linux-core/ttm/ttm_bo.c index 280c1a4e..db27650f 100644 --- a/linux-core/ttm/ttm_bo.c +++ b/linux-core/ttm/ttm_bo.c @@ -315,16 +315,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, goto out_err; moved: - if (bo->priv_flags & TTM_BO_PRIV_FLAG_EVICTED) { + if (bo->evicted) { ret = bdev->driver->invalidate_caches(bdev, bo->mem.flags); if (ret) printk(KERN_ERR "Can not flush read caches\n"); + bo->evicted = false; } - ttm_flag_masked(&bo->priv_flags, - (evict) ? TTM_BO_PRIV_FLAG_EVICTED : 0, - TTM_BO_PRIV_FLAG_EVICTED); - if (bo->mem.mm_node) bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; @@ -367,7 +364,7 @@ static int ttm_bo_expire_sync_obj(struct ttm_buffer_object *bo, } if (bo->sync_obj) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); } } return 0; @@ -391,7 +388,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) if (bo->sync_obj && driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); } if (bo->sync_obj && remove_all) @@ -588,10 +585,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, unsigned mem_type, evict_mem.mm_node = NULL; } spin_unlock(&bdev->lru_lock); - - ttm_flag_masked(&bo->priv_flags, TTM_BO_PRIV_FLAG_EVICTED, - TTM_BO_PRIV_FLAG_EVICTED); - + bo->evicted = true; out: return ret; } @@ -827,13 +821,13 @@ static int ttm_bo_busy(struct ttm_buffer_object *bo) if (sync_obj) { if (driver->sync_obj_signaled(sync_obj, bo->sync_obj_arg)) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); return 0; } driver->sync_obj_flush(sync_obj, bo->sync_obj_arg); if (driver->sync_obj_signaled(sync_obj, bo->sync_obj_arg)) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); return 0; } return 1; @@ -1538,7 +1532,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, while (bo->sync_obj) { if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); goto out; } if (no_wait) { @@ -1559,7 +1553,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (bo->sync_obj == sync_obj) { driver->sync_obj_unref(&bo->sync_obj); - bo->priv_flags &= ~TTM_BO_PRIV_FLAG_MOVING; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); } driver->sync_obj_unref(&sync_obj); } diff --git a/linux-core/ttm/ttm_bo_api.h b/linux-core/ttm/ttm_bo_api.h index 67383b12..a6014ed9 100644 --- a/linux-core/ttm/ttm_bo_api.h +++ b/linux-core/ttm/ttm_bo_api.h @@ -40,6 +40,7 @@ #include #include #include +#include struct ttm_bo_device; @@ -96,6 +97,13 @@ struct ttm_tt; * struct ttm_buffer_object * * @bdev: Pointer to the buffer object device structure. + * @buffer_start: The virtual user-space start address of ttm_bo_type_user + * buffers. + * @type: The bo type. + * @destroy: Destruction function. If NULL, kfree is used. + * @num_pages: Actual number of pages. + * @addr_space_offset: Address space offset. + * @acc_size: Accounted size for this object. * @kref: Reference count of this buffer object. When this refcount reaches * zero, the object is put on the delayed delete list. * @list_kref: List reference count of this buffer object. This member is @@ -103,43 +111,36 @@ struct ttm_tt; * Lru lists may keep one refcount, the delayed delete list, and kref != 0 * keeps one refcount. When this refcount reaches zero, * the object is destroyed. + * @event_queue: Queue for processes waiting on buffer object status change. + * @mutex: Lock protecting all members with the exception of constant members + * and list heads. We should really use a spinlock here. * @proposed_flags: Proposed placement for the buffer. Changed only by the * creator prior to validation as opposed to bo->mem.proposed_flags which is * changed by the implementation prior to a buffer move if it wants to outsmart * the buffer creator / user. This latter happens, for example, at eviction. - * @buffer_start: The virtual user-space start address of ttm_bo_type_user - * buffers. - * @type: The bo type. * @offset: The current GPU offset, which can have different meanings * depending on the memory type. For SYSTEM type memory, it should be 0. * @mem: structure describing current placement. + * @persistant_swap_storage: Usually the swap storage is deleted for buffers + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistant shmem object. + * @ttm: TTM structure holding system pages. + * @evicted: Whether the object was evicted without user-space knowing. + * @cpu_writes: For synchronization. Number of cpu writers. + * @lru: List head for the lru list. + * @ddestroy: List head for the delayed destroy list. + * @swap: List head for swap LRU list. * @val_seq: Sequence of the validation holding the @reserved lock. * Used to avoid starvation when many processes compete to validate the * buffer. This member is protected by the bo_device::lru_lock. * @seq_valid: The value of @val_seq is valid. This value is protected by * the bo_device::lru_lock. - * @lru: List head for the lru list. - * @ddestroy: List head for the delayed destroy list. - * @swap: List head for swap LRU list. - * @persistant_swap_storage: Usually the swap storage is deleted for buffers - * pinned in physical memory. If this behaviour is not desired, this member - * holds a pointer to a persistant shmem object. - * @destroy: Destruction function. If NULL, kfree is used. + * @reserved: Deadlock-free lock used for synchronization state transitions. * @sync_obj_arg: Opaque argument to synchronization object function. * @sync_obj: Pointer to a synchronization object. * @priv_flags: Flags describing buffer object internal state. - * @event_queue: Queue for processes waiting on buffer object status change. - * @mutex: Lock protecting all members with the exception of constant members - * and list heads. We should really use a spinlock here. - * @num_pages: Actual number of pages. - * @ttm: TTM structure holding system pages. - * @vm_hash: Hash item for fast address space lookup. Need to change to a - * rb-tree node. + * @vm_rb: Rb node for the vm rb tree. * @vm_node: Address space manager node. - * @addr_space_offset: Address space offset. - * @cpu_writes: For synchronization. Number of cpu writers. - * @reserved: Deadlock-free lock used for synchronization state transitions. - * @acc_size: Accounted size for this object. * * Base class for TTM buffer object, that deals with data placement and CPU * mappings. GPU mappings are really up to the driver, but for simpler GPUs @@ -154,48 +155,78 @@ struct ttm_tt; */ struct ttm_buffer_object { + /** + * Members constant at init. + */ + struct ttm_bo_device *bdev; + unsigned long buffer_start; + enum ttm_bo_type type; + void (*destroy) (struct ttm_buffer_object *); + unsigned long num_pages; + uint64_t addr_space_offset; + size_t acc_size; + + /** + * Members not needing protection. + */ + struct kref kref; struct kref list_kref; + wait_queue_head_t event_queue; + struct mutex mutex; - /* - * If there is a possibility that the usage variable is zero, - * then dev->struct_mutex should be locked before incrementing it. + /** + * Members protected by the bo::reserved lock. */ uint32_t proposed_flags; - unsigned long buffer_start; - enum ttm_bo_type type; unsigned long offset; struct ttm_mem_reg mem; - uint32_t val_seq; - bool seq_valid; + struct file *persistant_swap_storage; + struct ttm_tt *ttm; + bool evicted; + + /** + * Members protected by the bo::reserved lock only when written to. + */ + + atomic_t cpu_writers; + + /** + * Members protected by the bdev::lru_lock. + */ struct list_head lru; struct list_head ddestroy; struct list_head swap; + uint32_t val_seq; + bool seq_valid; - struct file *persistant_swap_storage; + /** + * Members protected by the bdev::lru_lock + * only when written to. + */ - void (*destroy) (struct ttm_buffer_object *); + atomic_t reserved; + + + /** + * Members protected by the bo::mutex + */ void *sync_obj_arg; void *sync_obj; + unsigned long priv_flags; - uint32_t priv_flags; - wait_queue_head_t event_queue; - struct mutex mutex; - unsigned long num_pages; + /** + * Members protected by the bdev::vm_lock + */ - struct ttm_tt *ttm; struct rb_node vm_rb; struct drm_mm_node *vm_node; - uint64_t addr_space_offset; - atomic_t cpu_writers; - atomic_t reserved; - size_t acc_size; }; /** diff --git a/linux-core/ttm/ttm_bo_driver.h b/linux-core/ttm/ttm_bo_driver.h index 85653177..213a9926 100644 --- a/linux-core/ttm/ttm_bo_driver.h +++ b/linux-core/ttm/ttm_bo_driver.h @@ -356,9 +356,9 @@ struct ttm_bo_driver { #define TTM_NUM_MEM_TYPES 8 -#define TTM_BO_PRIV_FLAG_EVICTED (1 << 0) /* Buffer object is evicted. */ -#define TTM_BO_PRIV_FLAG_MOVING (1 << 1) /* Buffer object is moving and needs - idling before CPU mapping */ +#define TTM_BO_PRIV_FLAG_MOVING 0 /* Buffer object is moving and needs + idling before CPU mapping */ +#define TTM_BO_PRIV_FLAG_MAX 1 /** * struct ttm_bo_device - Buffer object driver device-specific data. * diff --git a/linux-core/ttm/ttm_bo_util.c b/linux-core/ttm/ttm_bo_util.c index a893dbda..85611f94 100644 --- a/linux-core/ttm/ttm_bo_util.c +++ b/linux-core/ttm/ttm_bo_util.c @@ -539,7 +539,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, else bo->ttm = NULL; - bo->priv_flags |= TTM_BO_PRIV_FLAG_MOVING; + set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); ttm_bo_unreserve(ghost_obj); ttm_bo_unref(&ghost_obj); } diff --git a/linux-core/ttm/ttm_bo_vm.c b/linux-core/ttm/ttm_bo_vm.c index d2dbf421..c207b775 100644 --- a/linux-core/ttm/ttm_bo_vm.c +++ b/linux-core/ttm/ttm_bo_vm.c @@ -115,7 +115,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) * move. */ - if (bo->priv_flags & TTM_BO_PRIV_FLAG_MOVING) { + if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { ret = ttm_bo_wait(bo, false, true, false); if (unlikely(ret != 0)) { retval = (ret != -ERESTART) ? @@ -259,7 +259,7 @@ static unsigned long ttm_bo_vm_nopfn(struct vm_area_struct *vma, * move. */ - if (bo->priv_flags & TTM_BO_PRIV_FLAG_MOVING) { + if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { ret = ttm_bo_wait(bo, false, true, false); if (unlikely(ret != 0)) { retval = (ret != -ERESTART) ?