diff --git a/docs/features.txt b/docs/features.txt index e48b0afc411..d244881e161 100644 --- a/docs/features.txt +++ b/docs/features.txt @@ -858,7 +858,7 @@ Rusticl extensions: cl_khr_mipmap_image_writes not started cl_khr_pci_bus_info DONE (iris, nvc0, radeonsi, zink) cl_khr_priority_hints DONE (asahi, freedreno, iris, panfrost, radeonsi) - cl_khr_semaphore not started + cl_khr_semaphore DONE (radeonsi, zink) cl_khr_spirv_extended_debug_info not started cl_khr_spirv_linkonce_odr DONE cl_khr_spirv_no_integer_wrap_decoration DONE diff --git a/docs/relnotes/new_features.txt b/docs/relnotes/new_features.txt index e94dcd29984..8729eda733c 100644 --- a/docs/relnotes/new_features.txt +++ b/docs/relnotes/new_features.txt @@ -11,3 +11,4 @@ GL_ATI_meminfo and GL_NVX_gpu_memory_info on r300 VK_KHR_shader_untyped_pointers on anv and RADV VK_KHR_maintenance8 on NVK VK_KHR_maintenance9 on NVK +cl_khr_semaphore on radeonsi and zink diff --git a/src/gallium/frontends/rusticl/api/device.rs b/src/gallium/frontends/rusticl/api/device.rs index d323e9a1357..d360ef1b642 100644 --- a/src/gallium/frontends/rusticl/api/device.rs +++ b/src/gallium/frontends/rusticl/api/device.rs @@ -273,6 +273,9 @@ unsafe impl CLInfo for cl_device_id { (CL_QUEUE_PROFILING_ENABLE | CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE).into(), ), CL_DEVICE_REFERENCE_COUNT => v.write::(1), + CL_DEVICE_SEMAPHORE_TYPES_KHR if dev.are_semaphores_supported() => { + v.write::<&[cl_semaphore_type_khr]>(&[CL_SEMAPHORE_TYPE_BINARY_KHR]) + } CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL => { v.write::(0) } diff --git a/src/gallium/frontends/rusticl/api/icd.rs b/src/gallium/frontends/rusticl/api/icd.rs index 6d5a7c184c5..3de9cb2c6f6 100644 --- a/src/gallium/frontends/rusticl/api/icd.rs +++ b/src/gallium/frontends/rusticl/api/icd.rs @@ -229,6 +229,7 @@ pub enum RusticlTypes { Program, Queue, Sampler, + Semaphore, } impl RusticlTypes { @@ -247,6 +248,7 @@ impl RusticlTypes { 0xec4cf9af => Self::Program, 0xec4cf9b0 => Self::Queue, 0xec4cf9b1 => Self::Sampler, + 0xec4cf9b2 => Self::Semaphore, _ => return None, }; debug_assert!(result.u32() == val); diff --git a/src/gallium/frontends/rusticl/api/platform.rs b/src/gallium/frontends/rusticl/api/platform.rs index 5fefa87555e..36d94d3cb33 100644 --- a/src/gallium/frontends/rusticl/api/platform.rs +++ b/src/gallium/frontends/rusticl/api/platform.rs @@ -27,6 +27,13 @@ unsafe impl CLInfo for cl_platform_id { CL_PLATFORM_NAME => v.write::<&CStr>(c"rusticl"), CL_PLATFORM_NUMERIC_VERSION => v.write::(CLVersion::Cl3_0.into()), CL_PLATFORM_PROFILE => v.write::<&CStr>(c"FULL_PROFILE"), + CL_PLATFORM_SEMAPHORE_TYPES_KHR => { + v.write::<&[cl_semaphore_type_khr]>(if Platform::get().all_devs_have_semaphores() { + &[CL_SEMAPHORE_TYPE_BINARY_KHR] + } else { + &[] + }) + } CL_PLATFORM_VENDOR => v.write::<&CStr>(c"Mesa/X.org"), // OpenCL CL_PLATFORM_VERSION => v.write::<&CStr>(c"OpenCL 3.0 "), diff --git a/src/gallium/frontends/rusticl/api/semaphore.rs b/src/gallium/frontends/rusticl/api/semaphore.rs index 7e2b58572d4..e3db0dd4203 100644 --- a/src/gallium/frontends/rusticl/api/semaphore.rs +++ b/src/gallium/frontends/rusticl/api/semaphore.rs @@ -2,54 +2,255 @@ // SPDX-License-Identifier: MIT use { - crate::api::{ - icd::CLResult, - util::{CLInfo, CLInfoRes, CLInfoValue}, + crate::{ + api::{ + event::create_and_queue, + icd::{ArcedCLObject, BaseCLObject, CLResult, ReferenceCountedAPIPointer}, + util::{event_list_from_cl, CLInfo, CLInfoRes, CLInfoValue}, + }, + core::{context::Context, device::Device, queue::Queue, semaphore::Semaphore}, }, + mesa_rust_util::{conversion::TryIntoWithErr, properties::MultiValProperties}, rusticl_opencl_gen::*, rusticl_proc_macros::{cl_entrypoint, cl_info_entrypoint}, - std::ffi::{c_int, c_void}, + std::{ + ffi::{c_int, c_void}, + sync::Arc, + }, }; #[cl_info_entrypoint(clGetSemaphoreInfoKHR)] unsafe impl CLInfo for cl_semaphore_khr { - fn query(&self, _q: cl_semaphore_info_khr, _v: CLInfoValue) -> CLResult { - Err(CL_INVALID_OPERATION) + fn query(&self, q: cl_semaphore_info_khr, v: CLInfoValue) -> CLResult { + let sema = Semaphore::ref_from_raw(*self)?; + + match q { + CL_SEMAPHORE_CONTEXT_KHR => { + v.write::(cl_context::from_ptr(Arc::as_ptr(&sema.ctx))) + } + CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR => { + v.write::<&[cl_device_id]>(&[cl_device_id::from_ptr(sema.dev)]) + } + CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR => { + // In reality it's a list, but we are not supporting more than one anyway, so Option + // is fine here. + v.write::>(None) + } + CL_SEMAPHORE_EXPORTABLE_KHR => v.write::(CL_FALSE), + CL_SEMAPHORE_PAYLOAD_KHR => { + v.write::(sema.is_signalled().into()) + } + CL_SEMAPHORE_PROPERTIES_KHR => { + v.write::<&MultiValProperties>(&sema.props) + } + CL_SEMAPHORE_REFERENCE_COUNT_KHR => v.write::(Semaphore::refcnt(*self)?), + CL_SEMAPHORE_TYPE_KHR => v.write::(CL_SEMAPHORE_TYPE_BINARY_KHR), + _ => Err(CL_INVALID_VALUE), + } } } #[cl_entrypoint(clCreateSemaphoreWithPropertiesKHR)] fn create_semaphore( - _context: cl_context, - _sema_props: *const cl_semaphore_properties_khr, + context: cl_context, + sema_props: *const cl_semaphore_properties_khr, ) -> CLResult { - Err(CL_INVALID_OPERATION) + let context = Context::arc_from_raw(context)?; + + // CL_INVALID_VALUE if sema_props is NULL + if sema_props.is_null() { + return Err(CL_INVALID_VALUE); + } + + let mut sema_type = 0; + let mut dev = None; + let sema_props = unsafe { + MultiValProperties::new( + sema_props, + &[ + CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR.into(), + CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR.into(), + ], + ) + // CL_INVALID_PROPERTY [..] if the same property name is specified more than once. + .ok_or(CL_INVALID_PROPERTY)? + }; + + for (key, vals) in sema_props.iter() { + // CL_INVALID_PROPERTY if a property name in sema_props is not a supported property name + match u32::try_from(key).or(Err(CL_INVALID_PROPERTY))? { + CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR => { + // CL_INVALID_DEVICE if CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR is specified as part of + // sema_props, but it does not identify exactly one valid device; + let Some((&dev_in, &[])) = vals.split_first() else { + return Err(CL_INVALID_DEVICE); + }; + + let dev_in = Device::ref_from_raw(dev_in as _)? + .to_static() + .ok_or(CL_INVALID_DEVICE)?; + + // CL_INVALID_DEVICE [..] if a device identified by + // CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR is not one of the devices within context. + if !context.devs.contains(&dev_in) { + return Err(CL_INVALID_DEVICE); + } + + dev = Some(dev_in); + } + CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR => { + // CL_INVALID_VALUE if more than one semaphore handle type is specified in the + // CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR list. + if vals.len() > 1 { + return Err(CL_INVALID_VALUE); + } + } + CL_SEMAPHORE_TYPE_KHR => { + // CL_INVALID_PROPERTY [..] if the value specified for a supported property name is + // not valid + sema_type = vals[0].try_into_with_err(CL_INVALID_PROPERTY)?; + if sema_type != CL_SEMAPHORE_TYPE_BINARY_KHR { + return Err(CL_INVALID_PROPERTY); + } + } + // CL_INVALID_PROPERTY if a property name in sema_props is not a supported property name + _ => return Err(CL_INVALID_PROPERTY), + } + } + + let dev = match dev { + Some(dev) => dev, + None => { + // CL_INVALID_PROPERTY [..] Additionally, if context is a multiple device context and + // sema_props does not specify CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR. + let Some((dev_from_ctx, &[])) = context.devs.split_first() else { + return Err(CL_INVALID_PROPERTY); + }; + + // If CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR is not specified as part of sema_props, the + // semaphore object created by clCreateSemaphoreWithPropertiesKHR is by default + // associated with all devices in the context. + dev_from_ctx + } + }; + + // CL_INVALID_VALUE [..] if sema_props do not specify pairs for minimum set of + // properties (i.e. CL_SEMAPHORE_TYPE_KHR) required for successful creation of a + // cl_semaphore_khr + if sema_type == 0 { + return Err(CL_INVALID_VALUE); + } + + Ok(Semaphore::new(context, sema_props, dev)?.into_cl()) + + // CL_INVALID_DEVICE if one or more devices identified by properties CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR cannot import the requested external semaphore handle type. + // CL_INVALID_OPERATION If props_list specifies a cl_external_semaphore_handle_type_khr followed by a handle as well as CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR. Exporting a semaphore handle from a semaphore that was created by importing an external semaphore handle is not permitted. + // CL_INVALID_PROPERTY if sema_props includes more than one external semaphore handle. } #[cl_entrypoint(clEnqueueSignalSemaphoresKHR)] fn enqueue_signal_semaphores( - _command_queue: cl_command_queue, - _num_sema_objects: cl_uint, - _sema_objects: *const cl_semaphore_khr, + command_queue: cl_command_queue, + num_sema_objects: cl_uint, + sema_objects: *const cl_semaphore_khr, _sema_payload_list: *const cl_semaphore_payload_khr, - _num_events_in_wait_list: cl_uint, - _event_wait_list: *const cl_event, - _event: *mut cl_event, + num_events_in_wait_list: cl_uint, + event_wait_list: *const cl_event, + event: *mut cl_event, ) -> CLResult<()> { - Err(CL_INVALID_OPERATION) + let q = Queue::arc_from_raw(command_queue)?; + let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; + + // CL_INVALID_VALUE if num_sema_objects is 0. + if num_sema_objects == 0 { + return Err(CL_INVALID_VALUE); + } + + // CL_INVALID_SEMAPHORE_KHR if any of the semaphore objects specified by sema_objects is not + // valid. + let semas = Semaphore::arcs_from_arr(sema_objects, num_sema_objects)?; + + // CL_INVALID_CONTEXT if the context associated with command_queue and any of the semaphore + // objects in sema_objects are not the same + if semas.iter().any(|sema| sema.ctx != q.context) { + return Err(CL_INVALID_CONTEXT); + } + + // CL_INVALID_COMMAND_QUEUE [..] if the device associated with command_queue is not same as one + // of the devices specified by CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR at the time of creating one + // or more of sema_objects, or if one or more of sema_objects belong to a context that does not + // contain a device associated with command_queue. + for sema in &semas { + if q.device != sema.dev || !sema.ctx.devs.contains(&q.device) { + return Err(CL_INVALID_COMMAND_QUEUE); + } + } + + create_and_queue( + q, + CL_COMMAND_SEMAPHORE_SIGNAL_KHR, + evs, + event, + false, + Semaphore::gpu_signal(semas), + ) + + // CL_INVALID_VALUE if any of the semaphore objects specified by sema_objects requires a semaphore payload and sema_payload_list is NULL. } #[cl_entrypoint(clEnqueueWaitSemaphoresKHR)] fn enqueue_wait_semaphores( - _command_queue: cl_command_queue, - _num_sema_objects: cl_uint, - _sema_objects: *const cl_semaphore_khr, + command_queue: cl_command_queue, + num_sema_objects: cl_uint, + sema_objects: *const cl_semaphore_khr, _sema_payload_list: *const cl_semaphore_payload_khr, - _num_events_in_wait_list: cl_uint, - _event_wait_list: *const cl_event, - _event: *mut cl_event, + num_events_in_wait_list: cl_uint, + event_wait_list: *const cl_event, + event: *mut cl_event, ) -> CLResult<()> { - Err(CL_INVALID_OPERATION) + let q = Queue::arc_from_raw(command_queue)?; + let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; + + // CL_INVALID_VALUE if num_sema_objects is 0. + if num_sema_objects == 0 { + return Err(CL_INVALID_VALUE); + } + + // CL_INVALID_SEMAPHORE_KHR if any of the semaphore objects specified by sema_objects is not + // valid. + let semas = Semaphore::arcs_from_arr(sema_objects, num_sema_objects)?; + + // CL_INVALID_CONTEXT if the context associated with command_queue and any of the semaphore + // objects in sema_objects are not the same + if semas.iter().any(|sema| sema.ctx != q.context) { + return Err(CL_INVALID_CONTEXT); + } + + // CL_INVALID_COMMAND_QUEUE [..] if the device associated with command_queue is not same as one + // of the devices specified by CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR at the time of creating one + // or more of sema_objects, or if one or more of sema_objects belong to a context that does not + // contain a device associated with command_queue. + for sema in &semas { + if q.device != sema.dev || !sema.ctx.devs.contains(&q.device) { + return Err(CL_INVALID_COMMAND_QUEUE); + } + } + + create_and_queue( + q, + CL_COMMAND_SEMAPHORE_WAIT_KHR, + evs, + event, + false, + Box::new(|_, ctx| { + for sema in semas { + sema.gpu_wait(ctx)?; + } + Ok(()) + }), + ) + + // CL_INVALID_VALUE if any of the semaphore objects specified by sema_objects requires a semaphore payload and sema_payload_list is NULL. } #[cl_entrypoint(clGetSemaphoreHandleForTypeKHR)] @@ -65,13 +266,13 @@ fn get_semaphore_handle_for_type( } #[cl_entrypoint(clReleaseSemaphoreKHR)] -fn release_semaphore(_sema_object: cl_semaphore_khr) -> CLResult<()> { - Err(CL_INVALID_OPERATION) +fn release_semaphore(sema_object: cl_semaphore_khr) -> CLResult<()> { + Semaphore::release(sema_object) } #[cl_entrypoint(clRetainSemaphoreKHR)] -fn retain_semaphore(_sema_object: cl_semaphore_khr) -> CLResult<()> { - Err(CL_INVALID_OPERATION) +fn retain_semaphore(sema_object: cl_semaphore_khr) -> CLResult<()> { + Semaphore::retain(sema_object) } #[cl_entrypoint(clReImportSemaphoreSyncFdKHR)] diff --git a/src/gallium/frontends/rusticl/api/util.rs b/src/gallium/frontends/rusticl/api/util.rs index fe8c2c91e61..c901f113ef4 100644 --- a/src/gallium/frontends/rusticl/api/util.rs +++ b/src/gallium/frontends/rusticl/api/util.rs @@ -6,7 +6,7 @@ use crate::api::types::*; use crate::core::event::*; use crate::core::queue::*; -use mesa_rust_util::properties::Properties; +use mesa_rust_util::properties::{MultiValProperties, Properties}; use rusticl_opencl_gen::*; use std::cmp; @@ -412,6 +412,21 @@ where } } +impl CLProp for &MultiValProperties +where + T: CLProp + Copy, +{ + type Output = T; + + fn count(&self) -> usize { + self.as_raw_slice().count() + } + + fn write_to(&self, out: &mut [MaybeUninit]) { + self.as_raw_slice().write_to(out); + } +} + impl CLProp for Option where T: CLProp + Copy, diff --git a/src/gallium/frontends/rusticl/core.rs b/src/gallium/frontends/rusticl/core.rs index bda6da5d46a..3dd2550687d 100644 --- a/src/gallium/frontends/rusticl/core.rs +++ b/src/gallium/frontends/rusticl/core.rs @@ -11,5 +11,6 @@ pub mod memory; pub mod platform; pub mod program; pub mod queue; +pub mod semaphore; pub mod util; pub mod version; diff --git a/src/gallium/frontends/rusticl/core/device.rs b/src/gallium/frontends/rusticl/core/device.rs index 77ddb5faad7..ac8b759a739 100644 --- a/src/gallium/frontends/rusticl/core/device.rs +++ b/src/gallium/frontends/rusticl/core/device.rs @@ -770,6 +770,10 @@ impl DeviceBase { add_ext(1, 0, 2, "cl_ext_buffer_device_address"); } + if self.are_semaphores_supported() { + add_ext(1, 0, 1, "cl_khr_semaphore"); + } + self.extensions = exts; self.clc_features = feats; self.extension_string = exts_str.join(" "); @@ -1253,6 +1257,10 @@ impl DeviceBase { ..Default::default() } } + + pub fn are_semaphores_supported(&self) -> bool { + self.screen().caps().fence_signal && self.screen().has_semaphore_create() + } } impl Device { diff --git a/src/gallium/frontends/rusticl/core/platform.rs b/src/gallium/frontends/rusticl/core/platform.rs index f65a15238bf..6a8c34026ea 100644 --- a/src/gallium/frontends/rusticl/core/platform.rs +++ b/src/gallium/frontends/rusticl/core/platform.rs @@ -277,6 +277,10 @@ impl Platform { #[allow(static_mut_refs)] PLATFORM_ONCE.call_once(|| unsafe { PLATFORM.init() }); } + + pub fn all_devs_have_semaphores(&self) -> bool { + self.devs.iter().all(|dev| dev.are_semaphores_supported()) + } } impl Drop for Platform { diff --git a/src/gallium/frontends/rusticl/core/semaphore.rs b/src/gallium/frontends/rusticl/core/semaphore.rs new file mode 100644 index 00000000000..73f888de2f2 --- /dev/null +++ b/src/gallium/frontends/rusticl/core/semaphore.rs @@ -0,0 +1,145 @@ +// Copyright 2025 Red Hat. +// SPDX-License-Identifier: MIT + +use { + crate::{ + api::icd::{CLObjectBase, CLResult, RusticlTypes}, + core::{context::Context, device::Device, event::EventSig}, + impl_cl_type_trait, + }, + mesa_rust::pipe::{context::PipeContext, fence::PipeFence}, + mesa_rust_util::properties::MultiValProperties, + rusticl_opencl_gen::*, + std::sync::{Arc, Condvar, Mutex, MutexGuard}, +}; + +struct RealFence { + fence: PipeFence, + is_signalled: bool, +} + +struct SemaphoreState { + fence: RealFence, +} + +enum SemaphoreWaitAction { + Signal, + Wait, +} + +impl SemaphoreState { + fn gpu_signal(&mut self, ctx: &PipeContext) { + self.fence.fence.gpu_signal(ctx); + self.fence.is_signalled = true; + } + + fn gpu_wait(&mut self, ctx: &PipeContext) { + self.fence.fence.gpu_wait(ctx); + self.fence.is_signalled = false; + } + + fn is_signalled(&self) -> bool { + self.fence.is_signalled + } + + fn do_action( + mut this: MutexGuard, + cv: &Condvar, + action: SemaphoreWaitAction, + ctx: &PipeContext, + ) -> CLResult<()> { + // We need to wait until is_signalled gets set to the proper state. + let is_signalled_expected = !matches!(action, SemaphoreWaitAction::Signal); + + this = cv + .wait_while(this, |state| state.is_signalled() != is_signalled_expected) + .or(Err(CL_OUT_OF_HOST_MEMORY))?; + + match action { + // If this semaphore was already signalled, we need to wait until something + // successfully waited on it before we can re-signal. + SemaphoreWaitAction::Signal => this.gpu_signal(ctx), + + // We need to wait until something signals this semaphore before we can wait on it. + SemaphoreWaitAction::Wait => this.gpu_wait(ctx), + } + + drop(this); + cv.notify_all(); + Ok(()) + } +} + +/// Object representing a GPU semaphore that can be signalled and waited on. A semaphore is either +/// in a signalled or reset state and can only be waited on by a single consumer after which it +/// needs to be reset by a new call to gpu_signal. +pub struct Semaphore { + pub base: CLObjectBase, + pub ctx: Arc, + pub props: MultiValProperties, + pub dev: &'static Device, + state: Mutex, + /// Condition Variable used for waiting on a gpu_signal or gpu_wait operation to executed on the + /// queue thread. + signal_cv: Condvar, +} + +impl_cl_type_trait!(cl_semaphore_khr, Semaphore, CL_INVALID_SEMAPHORE_KHR); + +impl Semaphore { + pub fn new( + ctx: Arc, + props: MultiValProperties, + dev: &'static Device, + ) -> CLResult> { + Ok(Arc::new(Self { + base: CLObjectBase::new(RusticlTypes::Semaphore), + ctx: ctx, + props: props, + dev: dev, + state: Mutex::new(SemaphoreState { + fence: RealFence { + fence: dev + .screen() + .create_semaphore() + .ok_or(CL_OUT_OF_HOST_MEMORY)?, + is_signalled: false, + }, + }), + signal_cv: Condvar::new(), + })) + } + + /// Makes the GPU signal the semaphore. + pub fn gpu_signal(semas: Vec>) -> EventSig { + Box::new(move |_, ctx| { + for sema in semas { + SemaphoreState::do_action( + sema.state(), + &sema.signal_cv, + SemaphoreWaitAction::Signal, + ctx, + )?; + } + Ok(()) + }) + } + + /// Makes the GPU wait on the semaphore to be signalled. + pub fn gpu_wait(&self, ctx: &PipeContext) -> CLResult<()> { + SemaphoreState::do_action( + self.state(), + &self.signal_cv, + SemaphoreWaitAction::Wait, + ctx, + ) + } + + pub fn is_signalled(&self) -> bool { + self.state().is_signalled() + } + + fn state(&self) -> MutexGuard { + self.state.lock().unwrap() + } +} diff --git a/src/gallium/frontends/rusticl/meson.build b/src/gallium/frontends/rusticl/meson.build index 01cbb7c5890..c81924d4ca4 100644 --- a/src/gallium/frontends/rusticl/meson.build +++ b/src/gallium/frontends/rusticl/meson.build @@ -57,6 +57,7 @@ rusticl_files = files( 'core/platform.rs', 'core/program.rs', 'core/queue.rs', + 'core/semaphore.rs', 'core/util.rs', 'core/version.rs', 'core/gl.rs',