rusticl/mem: implement clCopyBuffer

Signed-off-by: Karol Herbst <kherbst@redhat.com>
Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15439>

src/gallium/frontends/rusticl/api/icd.rs

@@ -923,18 +923,27 @@ extern "C" fn cl_enqueue_write_buffer(
 }
 
 extern "C" fn cl_enqueue_copy_buffer(
-    _command_queue: cl_command_queue,
-    _src_buffer: cl_mem,
-    _dst_buffer: cl_mem,
-    _src_offset: usize,
-    _dst_offset: usize,
-    _cb: usize,
-    _num_events_in_wait_list: cl_uint,
-    _event_wait_list: *const cl_event,
-    _event: *mut cl_event,
+    command_queue: cl_command_queue,
+    src_buffer: cl_mem,
+    dst_buffer: cl_mem,
+    src_offset: usize,
+    dst_offset: usize,
+    cb: usize,
+    num_events_in_wait_list: cl_uint,
+    event_wait_list: *const cl_event,
+    event: *mut cl_event,
 ) -> cl_int {
-    println!("cl_enqueue_copy_buffer not implemented");
-    CL_OUT_OF_HOST_MEMORY
+    match_err!(enqueue_copy_buffer(
+        command_queue,
+        src_buffer,
+        dst_buffer,
+        src_offset,
+        dst_offset,
+        cb,
+        num_events_in_wait_list,
+        event_wait_list,
+        event,
+    ))
 }
 
 extern "C" fn cl_enqueue_read_image(

src/gallium/frontends/rusticl/api/memory.rs

@@ -892,6 +892,64 @@ pub fn enqueue_write_buffer(
     // CL_MISALIGNED_SUB_BUFFER_OFFSET if buffer is a sub-buffer object and offset specified when the sub-buffer object is created is not aligned to CL_DEVICE_MEM_BASE_ADDR_ALIGN value for device associated with queue.
 }
 
+pub fn enqueue_copy_buffer(
+    command_queue: cl_command_queue,
+    src_buffer: cl_mem,
+    dst_buffer: cl_mem,
+    src_offset: usize,
+    dst_offset: usize,
+    size: usize,
+    num_events_in_wait_list: cl_uint,
+    event_wait_list: *const cl_event,
+    event: *mut cl_event,
+) -> CLResult<()> {
+    let q = command_queue.get_arc()?;
+    let src = src_buffer.get_arc()?;
+    let dst = dst_buffer.get_arc()?;
+    let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?;
+
+    // CL_INVALID_CONTEXT if the context associated with command_queue, src_buffer and dst_buffer
+    // are not the same
+    if q.context != src.context || q.context != dst.context {
+        return Err(CL_INVALID_CONTEXT);
+    }
+
+    // CL_INVALID_VALUE if src_offset, dst_offset, size, src_offset + size or dst_offset + size
+    // require accessing elements outside the src_buffer and dst_buffer buffer objects respectively.
+    if src_offset + size > src.size || dst_offset + size > dst.size {
+        return Err(CL_INVALID_VALUE);
+    }
+
+    // CL_MEM_COPY_OVERLAP if src_buffer and dst_buffer are the same buffer or sub-buffer object
+    // and the source and destination regions overlap or if src_buffer and dst_buffer are different
+    // sub-buffers of the same associated buffer object and they overlap. The regions overlap if
+    // src_offset ≤ dst_offset ≤ src_offset + size - 1 or if dst_offset ≤ src_offset ≤ dst_offset + size - 1.
+    if src.has_same_parent(&dst) {
+        let src_offset = src_offset + src.offset;
+        let dst_offset = dst_offset + dst.offset;
+
+        if (src_offset <= dst_offset && dst_offset < src_offset + size)
+            || (dst_offset <= src_offset && src_offset < dst_offset + size)
+        {
+            return Err(CL_MEM_COPY_OVERLAP);
+        }
+    }
+
+    create_and_queue(
+        q,
+        CL_COMMAND_COPY_BUFFER,
+        evs,
+        event,
+        false,
+        Box::new(move |q, ctx| src.copy_to(q, ctx, &dst, src_offset, dst_offset, size)),
+    )
+
+    // TODO
+    //• CL_MISALIGNED_SUB_BUFFER_OFFSET if src_buffer is a sub-buffer object and offset specified when the sub-buffer object is created is not aligned to CL_DEVICE_MEM_BASE_ADDR_ALIGN value for device associated with queue.
+    //• CL_MISALIGNED_SUB_BUFFER_OFFSET if dst_buffer is a sub-buffer object and offset specified when the sub-buffer object is created is not aligned to CL_DEVICE_MEM_BASE_ADDR_ALIGN value for device associated with queue.
+    //• CL_MEM_OBJECT_ALLOCATION_FAILURE if there is a failure to allocate memory for data store associated with src_buffer or dst_buffer.
+}
+
 pub fn enqueue_read_buffer_rect(
     command_queue: cl_command_queue,
     buffer: cl_mem,
@@ -1265,7 +1323,7 @@ pub fn enqueue_copy_buffer_rect(
         event,
         false,
         Box::new(move |q, ctx| {
-            src.copy_to(
+            src.copy_to_rect(
                 &dst,
                 q,
                 ctx,

src/gallium/frontends/rusticl/core/memory.rs

@@ -260,6 +260,31 @@ impl Mem {
         Ok(())
     }
 
+    pub fn copy_to(
+        &self,
+        q: &Arc<Queue>,
+        ctx: &PipeContext,
+        dst: &Arc<Mem>,
+        mut src_offset: usize,
+        mut dst_offset: usize,
+        size: usize,
+    ) -> CLResult<()> {
+        let src = self.to_parent(&mut src_offset);
+        let dst = dst.to_parent(&mut dst_offset);
+
+        let src_res = src.get_res().get(&q.device).unwrap();
+        let dst_res = dst.get_res().get(&q.device).unwrap();
+
+        ctx.resource_copy_region(
+            src_res,
+            src_offset.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?,
+            dst_res,
+            dst_offset.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?,
+            size.try_into().map_err(|_| CL_OUT_OF_HOST_MEMORY)?,
+        );
+        Ok(())
+    }
+
     pub fn write_from_user_rect(
         &self,
         src: *const c_void,
@@ -328,7 +353,7 @@ impl Mem {
         Ok(())
     }
 
-    pub fn copy_to(
+    pub fn copy_to_rect(
         &self,
         dst: &Self,
         q: &Arc<Queue>,

src/gallium/frontends/rusticl/mesa/pipe/context.rs

@@ -55,6 +55,37 @@ impl PipeContext {
         }
     }
 
+    pub fn resource_copy_region(
+        &self,
+        src: &PipeResource,
+        src_offset: i32,
+        dst: &PipeResource,
+        dst_offset: u32,
+        size: i32,
+    ) {
+        let b = pipe_box {
+            width: size,
+            height: 1,
+            depth: 1,
+            x: src_offset,
+            ..Default::default()
+        };
+
+        unsafe {
+            self.pipe.as_ref().resource_copy_region.unwrap()(
+                self.pipe.as_ptr(),
+                dst.pipe(),
+                0,
+                dst_offset,
+                0,
+                0,
+                src.pipe(),
+                0,
+                &b,
+            )
+        }
+    }
+
     pub fn buffer_map(
         &self,
         res: &PipeResource,
@@ -210,5 +241,6 @@ fn has_required_cbs(c: &pipe_context) -> bool {
         && c.flush.is_some()
         && c.launch_grid.is_some()
         && c.memory_barrier.is_some()
+        && c.resource_copy_region.is_some()
         && c.set_global_binding.is_some()
 }