diff --git a/src/gallium/frontends/rusticl/api/memory.rs b/src/gallium/frontends/rusticl/api/memory.rs index 1de96f22829..eef984e6387 100644 --- a/src/gallium/frontends/rusticl/api/memory.rs +++ b/src/gallium/frontends/rusticl/api/memory.rs @@ -1591,7 +1591,6 @@ pub fn enqueue_map_buffer( evs, event, block, - // we don't really have anything to do here? Box::new(move |q, ctx| { cloned.set(b.map_buffer(q, Some(ctx), offset, size)); Ok(()) @@ -1600,17 +1599,17 @@ pub fn enqueue_map_buffer( ptr.get() } else { + let ptr = b.map_buffer(&q, None, offset, size); create_and_queue( - q.clone(), + q, CL_COMMAND_MAP_BUFFER, evs, event, block, - // we don't really have anything to do here? - Box::new(|_, _| Ok(())), + Box::new(move |q, ctx| b.sync_shadow_buffer(q, ctx, true)), )?; - b.map_buffer(&q, None, offset, size) + ptr } // TODO @@ -2072,24 +2071,25 @@ pub fn enqueue_map_image( *image_slice_pitch = res.2; res.0 } else { - create_and_queue( - q.clone(), - CL_COMMAND_MAP_IMAGE, - evs, - event, - block, - // we don't really have anything to do here? - Box::new(|_, _| Ok(())), - )?; - - i.map_image( + let ptr = i.map_image( &q, None, &origin, ®ion, unsafe { image_row_pitch.as_mut().unwrap() }, image_slice_pitch, - ) + ); + + create_and_queue( + q.clone(), + CL_COMMAND_MAP_IMAGE, + evs, + event, + block, + Box::new(move |q, ctx| i.sync_shadow_image(q, ctx, true)), + )?; + + ptr } //• CL_INVALID_VALUE if region being mapped given by (origin, origin + region) is out of bounds or if values specified in map_flags are not valid. @@ -2131,10 +2131,7 @@ pub fn enqueue_unmap_mem_object( evs, event, false, - Box::new(move |q, ctx| { - m.unmap(q, ctx, mapped_ptr); - Ok(()) - }), + Box::new(move |q, ctx| m.unmap(q, ctx, mapped_ptr)), ) } diff --git a/src/gallium/frontends/rusticl/core/context.rs b/src/gallium/frontends/rusticl/core/context.rs index 1c5686216fe..5c8ee467c8b 100644 --- a/src/gallium/frontends/rusticl/core/context.rs +++ b/src/gallium/frontends/rusticl/core/context.rs @@ -45,19 +45,30 @@ impl Context { &self, size: usize, user_ptr: *mut c_void, + copy: bool, ) -> CLResult, Arc>> { let adj_size: u32 = size.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?; let mut res = HashMap::new(); for dev in &self.devs { - let resource = dev - .screen() - .resource_create_buffer(adj_size) - .ok_or(CL_OUT_OF_RESOURCES); + let mut resource = None; + + if !user_ptr.is_null() && !copy { + resource = dev + .screen() + .resource_create_buffer_from_user(adj_size, user_ptr) + } + + if resource.is_none() { + resource = dev.screen().resource_create_buffer(adj_size) + } + + let resource = resource.ok_or(CL_OUT_OF_RESOURCES); res.insert(Arc::clone(dev), Arc::new(resource?)); } if !user_ptr.is_null() { res.iter() + .filter(|(_, r)| copy || !r.is_user) .map(|(d, r)| { d.helper_ctx() .exec(|ctx| ctx.buffer_subdata(r, 0, user_ptr, size.try_into().unwrap())) @@ -68,28 +79,12 @@ impl Context { Ok(res) } - pub fn create_buffer_from_user( - &self, - size: usize, - user_ptr: *mut c_void, - ) -> CLResult, Arc>> { - let adj_size: u32 = size.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?; - let mut res = HashMap::new(); - for dev in &self.devs { - let resource = dev - .screen() - .resource_create_buffer_from_user(adj_size, user_ptr) - .ok_or(CL_OUT_OF_RESOURCES); - res.insert(Arc::clone(dev), Arc::new(resource?)); - } - Ok(res) - } - pub fn create_texture( &self, desc: &cl_image_desc, format: &cl_image_format, user_ptr: *mut c_void, + copy: bool, ) -> CLResult, Arc>> { let width = desc .image_width @@ -112,10 +107,22 @@ impl Context { let mut res = HashMap::new(); for dev in &self.devs { - let resource = dev - .screen() - .resource_create_texture(width, height, depth, array_size, target, format) - .ok_or(CL_OUT_OF_RESOURCES); + let mut resource = None; + + // we can't specify custom pitches/slices, so this won't work for non 1D images + if !user_ptr.is_null() && !copy && desc.image_type == CL_MEM_OBJECT_IMAGE1D { + resource = dev.screen().resource_create_texture_from_user( + width, height, depth, array_size, target, format, user_ptr, + ) + } + + if resource.is_none() { + resource = dev + .screen() + .resource_create_texture(width, height, depth, array_size, target, format) + } + + let resource = resource.ok_or(CL_OUT_OF_RESOURCES); res.insert(Arc::clone(dev), Arc::new(resource?)); } @@ -125,6 +132,7 @@ impl Context { let layer_stride = desc.slice_pitch()?; res.iter() + .filter(|(_, r)| copy || !r.is_user) .map(|(d, r)| { d.helper_ctx() .exec(|ctx| ctx.texture_subdata(r, &bx, user_ptr, stride, layer_stride)) @@ -134,44 +142,6 @@ impl Context { Ok(res) } - - pub fn create_texture_from_user( - &self, - desc: &cl_image_desc, - format: &cl_image_format, - user_ptr: *mut c_void, - ) -> CLResult, Arc>> { - let width = desc - .image_width - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)?; - let height = desc - .image_height - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)?; - let depth = desc - .image_depth - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)?; - let array_size = desc - .image_array_size - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)?; - let target = cl_mem_type_to_texture_target(desc.image_type); - let format = format.to_pipe_format().unwrap(); - - let mut res = HashMap::new(); - for dev in &self.devs { - let resource = dev - .screen() - .resource_create_texture_from_user( - width, height, depth, array_size, target, format, user_ptr, - ) - .ok_or(CL_OUT_OF_RESOURCES); - res.insert(Arc::clone(dev), Arc::new(resource?)); - } - Ok(res) - } } impl Drop for Context { diff --git a/src/gallium/frontends/rusticl/core/memory.rs b/src/gallium/frontends/rusticl/core/memory.rs index cb8067dd3b2..eb60c611787 100644 --- a/src/gallium/frontends/rusticl/core/memory.rs +++ b/src/gallium/frontends/rusticl/core/memory.rs @@ -70,6 +70,7 @@ pub trait CLImageDescInfo { fn bx(&self) -> CLResult; fn row_pitch(&self) -> CLResult; fn slice_pitch(&self) -> CLResult; + fn size(&self) -> CLVec; fn dims(&self) -> u8 { self.type_info().0 @@ -115,30 +116,29 @@ impl CLImageDescInfo for cl_image_desc { res } - fn bx(&self) -> CLResult { + fn size(&self) -> CLVec { let mut depth = if self.is_array() { self.image_array_size - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)? } else { self.image_depth - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)? }; let height = cmp::max(self.image_height, 1); depth = cmp::max(depth, 1); + CLVec::new([self.image_width, height, depth]) + } + + fn bx(&self) -> CLResult { + let size = self.size(); + Ok(pipe_box { x: 0, y: 0, z: 0, - width: self - .image_width - .try_into() - .map_err(|_| CL_OUT_OF_HOST_MEMORY)?, - height: height.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?, - depth: depth, + width: size[0].try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?, + height: size[1].try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?, + depth: size[2].try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?, }) } @@ -225,12 +225,8 @@ impl Mem { println!("host ptr semantics not implemented!"); } - let buffer = if bit_check(flags, CL_MEM_USE_HOST_PTR) { - context.create_buffer_from_user(size, host_ptr) - } else { - assert_eq!(bit_check(flags, CL_MEM_COPY_HOST_PTR), !host_ptr.is_null()); - context.create_buffer(size, host_ptr) - }?; + let buffer = + context.create_buffer(size, host_ptr, bit_check(flags, CL_MEM_COPY_HOST_PTR))?; let host_ptr = if bit_check(flags, CL_MEM_USE_HOST_PTR) { host_ptr @@ -316,12 +312,12 @@ impl Mem { image_desc.image_array_size = 1; } - let texture = if bit_check(flags, CL_MEM_USE_HOST_PTR) { - context.create_texture_from_user(&image_desc, image_format, host_ptr) - } else { - assert_eq!(bit_check(flags, CL_MEM_COPY_HOST_PTR), !host_ptr.is_null()); - context.create_texture(&image_desc, image_format, host_ptr) - }?; + let texture = context.create_texture( + &image_desc, + image_format, + host_ptr, + bit_check(flags, CL_MEM_COPY_HOST_PTR), + )?; let host_ptr = if bit_check(flags, CL_MEM_USE_HOST_PTR) { host_ptr @@ -443,6 +439,11 @@ impl Mem { } } + fn has_user_shadow_buffer(&self, d: &Device) -> CLResult { + let r = self.get_res()?.get(d).unwrap(); + Ok(!r.is_user && bit_check(self.flags, CL_MEM_USE_HOST_PTR)) + } + pub fn read_to_user( &self, q: &Arc, @@ -743,6 +744,54 @@ impl Mem { Ok(()) } + // TODO: only sync on unmap when memory is mapped for writing + pub fn sync_shadow_buffer(&self, q: &Arc, ctx: &PipeContext, map: bool) -> CLResult<()> { + if self.has_user_shadow_buffer(&q.device)? { + if map { + self.read_to_user(q, ctx, 0, self.host_ptr, self.size) + } else { + self.write_from_user(q, ctx, 0, self.host_ptr, self.size) + } + } else { + Ok(()) + } + } + + // TODO: only sync on unmap when memory is mapped for writing + pub fn sync_shadow_image(&self, q: &Arc, ctx: &PipeContext, map: bool) -> CLResult<()> { + if self.has_user_shadow_buffer(&q.device)? { + if map { + self.read_to_user_rect( + self.host_ptr, + q, + ctx, + &self.image_desc.size(), + &CLVec::default(), + 0, + 0, + &CLVec::default(), + self.image_desc.image_row_pitch, + self.image_desc.image_slice_pitch, + ) + } else { + self.write_from_user_rect( + self.host_ptr, + q, + ctx, + &self.image_desc.size(), + &CLVec::default(), + self.image_desc.image_row_pitch, + self.image_desc.image_slice_pitch, + &CLVec::default(), + self.image_desc.image_row_pitch, + self.image_desc.image_slice_pitch, + ) + } + } else { + Ok(()) + } + } + fn map<'a>( &self, q: &Arc, @@ -776,8 +825,19 @@ impl Mem { assert!(self.is_buffer()); let mut lock = self.maps.lock().unwrap(); - let tx = self.map(q, ctx, &mut lock)?; - let ptr = unsafe { tx.ptr().add(offset) }; + let ptr = if self.has_user_shadow_buffer(&q.device)? { + // copy to the host_ptr if we are blocking + if let Some(ctx) = ctx { + self.sync_shadow_buffer(q, ctx, true)?; + } + + self.host_ptr + } else { + let tx = self.map(q, ctx, &mut lock)?; + tx.ptr() + }; + + let ptr = unsafe { ptr.add(offset) }; if let Some(e) = lock.maps.get_mut(&ptr) { *e += 1; @@ -800,12 +860,29 @@ impl Mem { assert!(!self.is_buffer()); let mut lock = self.maps.lock().unwrap(); - let tx = self.map(q, ctx, &mut lock)?; - *row_pitch = tx.row_pitch() as usize; - *slice_pitch = tx.slice_pitch() as usize; + // we might have a host_ptr shadow buffer + let ptr = if self.has_user_shadow_buffer(&q.device)? { + *row_pitch = self.image_desc.image_row_pitch; + *slice_pitch = self.image_desc.image_slice_pitch; + + // copy to the host_ptr if we are blocking + if let Some(ctx) = ctx { + self.sync_shadow_image(q, ctx, true)?; + } + + self.host_ptr + } else { + let tx = self.map(q, ctx, &mut lock)?; + + *row_pitch = tx.row_pitch() as usize; + *slice_pitch = tx.slice_pitch() as usize; + + tx.ptr() + }; + let ptr = unsafe { - tx.ptr().add( + ptr.add( *origin * [ self.image_format.pixel_size().unwrap() as usize, @@ -828,12 +905,12 @@ impl Mem { self.maps.lock().unwrap().maps.contains_key(&ptr) } - pub fn unmap(&self, q: &Arc, ctx: &PipeContext, ptr: *mut c_void) { + pub fn unmap(&self, q: &Arc, ctx: &PipeContext, ptr: *mut c_void) -> CLResult<()> { let mut lock = self.maps.lock().unwrap(); let e = lock.maps.get_mut(&ptr).unwrap(); if *e == 0 { - return; + return Ok(()); } *e -= 1; @@ -841,12 +918,25 @@ impl Mem { lock.maps.remove(&ptr); } - let tx = lock.tx.get_mut(&q.device).unwrap(); - tx.1 -= 1; - - if tx.1 == 0 { - lock.tx.remove(&q.device).unwrap().0.with_ctx(ctx); + // TODO: only sync on last unmap and only mapped ranges + if self.has_user_shadow_buffer(&q.device)? { + if self.is_buffer() { + self.sync_shadow_buffer(q, ctx, false)?; + } else { + self.sync_shadow_image(q, ctx, false)?; + } } + + // shadow buffers don't get a tx option bound + if let Some(tx) = lock.tx.get_mut(&q.device) { + tx.1 -= 1; + + if tx.1 == 0 { + lock.tx.remove(&q.device).unwrap().0.with_ctx(ctx); + } + } + + Ok(()) } } diff --git a/src/gallium/frontends/rusticl/mesa/pipe/resource.rs b/src/gallium/frontends/rusticl/mesa/pipe/resource.rs index 6fda87d48e6..63cf919b25a 100644 --- a/src/gallium/frontends/rusticl/mesa/pipe/resource.rs +++ b/src/gallium/frontends/rusticl/mesa/pipe/resource.rs @@ -6,15 +6,19 @@ use std::ptr; pub struct PipeResource { pipe: *mut pipe_resource, + pub is_user: bool, } impl PipeResource { - pub fn new(res: *mut pipe_resource) -> Option { + pub fn new(res: *mut pipe_resource, is_user: bool) -> Option { if res.is_null() { return None; } - Some(Self { pipe: res }) + Some(Self { + pipe: res, + is_user: is_user, + }) } pub(super) fn pipe(&self) -> *mut pipe_resource { diff --git a/src/gallium/frontends/rusticl/mesa/pipe/screen.rs b/src/gallium/frontends/rusticl/mesa/pipe/screen.rs index 2e56ef4b651..78e55613cda 100644 --- a/src/gallium/frontends/rusticl/mesa/pipe/screen.rs +++ b/src/gallium/frontends/rusticl/mesa/pipe/screen.rs @@ -84,7 +84,10 @@ impl PipeScreen { } fn resource_create(&self, tmpl: &pipe_resource) -> Option { - PipeResource::new(unsafe { (*self.screen).resource_create.unwrap()(self.screen, tmpl) }) + PipeResource::new( + unsafe { (*self.screen).resource_create.unwrap()(self.screen, tmpl) }, + false, + ) } fn resource_create_from_user( @@ -92,9 +95,10 @@ impl PipeScreen { tmpl: &pipe_resource, mem: *mut c_void, ) -> Option { - PipeResource::new(unsafe { - (*self.screen).resource_from_user_memory.unwrap()(self.screen, tmpl, mem) - }) + PipeResource::new( + unsafe { (*self.screen).resource_from_user_memory.unwrap()(self.screen, tmpl, mem) }, + true, + ) } pub fn resource_create_buffer(&self, size: u32) -> Option {