diff --git a/src/nouveau/nil/cbindgen.toml b/src/nouveau/nil/cbindgen.toml index 7026139da39..e8f8d518284 100644 --- a/src/nouveau/nil/cbindgen.toml +++ b/src/nouveau/nil/cbindgen.toml @@ -16,6 +16,34 @@ renaming_overrides_prefixing = true "nv_device_info" = "struct nv_device_info" "pipe_format" = "enum pipe_format" "pipe_swizzle" = "enum pipe_swizzle" +"GOB_DEPTH" = "NIL_GOB_DEPTH" +"GOB_WIDTH_B" = "NIL_GOB_WIDTH_B" +"MAX_LEVELS" = "NIL_MAX_LEVELS" + +# This is annoying. rename_types doesn't seem to work +"Extent4D" = "nil_extent4d" +"Format" = "nil_format" +"Image" = "nil_image" +"ImageDim" = "nil_image_dim" +"ImageInitInfo" = "nil_image_init_info" +"ImageLevel" = "nil_image_level" +"Offset4D" = "nil_offset4d" +"SampleLayout" = "nil_sample_layout" +"Tiling" = "nil_tiling" +"View" = "nil_view" +"ViewType" = "nil_view_type" + +# There's just no good solution for this one +"ImageUsageFlags" = "nil_image_usage_flags" +"IMAGE_USAGE_RENDER_TARGET_BIT" = "NIL_IMAGE_USAGE_RENDER_TARGET_BIT" +"IMAGE_USAGE_DEPTH_BIT" = "NIL_IMAGE_USAGE_DEPTH_BIT" +"IMAGE_USAGE_STENCIL_BIT" = "NIL_IMAGE_USAGE_STENCIL_BIT" +"IMAGE_USAGE_TEXTURE_BIT" = "NIL_IMAGE_USAGE_TEXTURE_BIT" +"IMAGE_USAGE_STORAGE_BIT" = "NIL_IMAGE_USAGE_STORAGE_BIT" +"IMAGE_USAGE_CUBE_BIT" = "NIL_IMAGE_USAGE_CUBE_BIT" +"IMAGE_USAGE_2D_VIEW_BIT" = "NIL_IMAGE_USAGE_2D_VIEW_BIT" +"IMAGE_USAGE_LINEAR_BIT" = "NIL_IMAGE_USAGE_LINEAR_BIT" +"IMAGE_USAGE_SPARSE_RESIDENCY_BIT" = "NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT" [macro_expansion] bitflags = true # We need this for the bitflags crate diff --git a/src/nouveau/nil/extent.rs b/src/nouveau/nil/extent.rs new file mode 100644 index 00000000000..221f8de41a6 --- /dev/null +++ b/src/nouveau/nil/extent.rs @@ -0,0 +1,230 @@ +// Copyright © 2024 Collabora, Ltd. +// SPDX-License-Identifier: MIT + +use crate::format::Format; +use crate::image::SampleLayout; +use crate::tiling::{gob_height, Tiling, GOB_DEPTH, GOB_WIDTH_B}; +use crate::Minify; + +#[derive(Clone, Debug, Copy, PartialEq, Default)] +#[repr(C)] +pub struct Extent4D { + pub width: u32, + pub height: u32, + pub depth: u32, + pub array_len: u32, +} + +#[no_mangle] +pub extern "C" fn nil_extent4d( + width: u32, + height: u32, + depth: u32, + array_len: u32, +) -> Extent4D { + Extent4D { + width, + height, + depth, + array_len, + } +} + +impl Extent4D { + pub fn size(&self) -> u32 { + self.width * self.height * self.depth + } + + pub fn align(self, alignment: &Self) -> Self { + Self { + width: self.width.next_multiple_of(alignment.width), + height: self.height.next_multiple_of(alignment.height), + depth: self.depth.next_multiple_of(alignment.depth), + array_len: self.array_len.next_multiple_of(alignment.array_len), + } + } + + pub fn mul(self, other: Self) -> Self { + Self { + width: self.width * other.width, + height: self.height * other.height, + depth: self.depth * other.depth, + array_len: self.array_len * other.array_len, + } + } + + pub fn div_ceil(self, other: Self) -> Self { + Self { + width: self.width.div_ceil(other.width), + height: self.height.div_ceil(other.height), + depth: self.depth.div_ceil(other.depth), + array_len: self.array_len.div_ceil(other.array_len), + } + } + + pub fn px_to_sa(self, sample_layout: SampleLayout) -> Self { + self.mul(sample_layout.px_extent_sa()) + } + + #[no_mangle] + pub extern "C" fn nil_extent4d_px_to_el( + self, + format: Format, + sample_layout: SampleLayout, + ) -> Extent4D { + self.px_to_el(format, sample_layout) + } + + pub fn px_to_el(self, format: Format, sample_layout: SampleLayout) -> Self { + self.px_to_sa(sample_layout).div_ceil(format.el_extent_sa()) + } + + pub fn el_to_B(self, bytes_per_element: u32) -> Self { + Self { + width: self.width * bytes_per_element, + ..self + } + } + pub fn px_to_B(self, format: Format, sample_layout: SampleLayout) -> Self { + self.px_to_el(format, sample_layout) + .el_to_B(format.el_size_B()) + } + + pub fn B_to_GOB(self, gob_height_is_8: bool) -> Self { + let gob_extent_B = Self { + width: GOB_WIDTH_B, + height: gob_height(gob_height_is_8), + depth: GOB_DEPTH, + array_len: 1, + }; + + self.div_ceil(gob_extent_B) + } + + #[no_mangle] + pub extern "C" fn nil_extent4d_px_to_tl( + self, + tiling: &Tiling, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + self.px_to_tl(tiling, format, sample_layout) + } + + pub fn px_to_tl( + self, + tiling: &Tiling, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + let tl_extent_B = tiling.extent_B(); + self.px_to_B(format, sample_layout).div_ceil(tl_extent_B) + } +} + +#[derive(Clone, Debug, Copy, PartialEq)] +#[repr(C)] +pub struct Offset4D { + pub x: u32, + pub y: u32, + pub z: u32, + pub a: u32, +} + +#[no_mangle] +pub extern "C" fn nil_offset4d(x: u32, y: u32, z: u32, a: u32) -> Offset4D { + Offset4D { x, y, z, a } +} + +impl Offset4D { + fn div_floor(self, other: Extent4D) -> Self { + Self { + x: self.x / other.width, + y: self.y / other.height, + z: self.z / other.depth, + a: self.a / other.array_len, + } + } + + fn mul(self, other: Extent4D) -> Self { + Self { + x: self.x * other.width, + y: self.y * other.height, + z: self.z * other.depth, + a: self.a * other.array_len, + } + } + + #[no_mangle] + pub extern "C" fn nil_offset4d_px_to_el( + self, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + self.px_to_el(format, sample_layout) + } + + pub fn px_to_el(self, format: Format, sample_layout: SampleLayout) -> Self { + self.mul(sample_layout.px_extent_sa()) + .div_floor(format.el_extent_sa()) + } + + #[no_mangle] + pub extern "C" fn nil_offset4d_px_to_tl( + self, + tiling: &Tiling, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + self.px_to_tl(tiling, format, sample_layout) + } + + pub fn px_to_tl( + self, + tiling: &Tiling, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + self.px_to_B(format, sample_layout) + .div_floor(tiling.extent_B()) + } + + #[no_mangle] + pub extern "C" fn nil_offset4d_px_to_B( + self, + format: Format, + sample_layout: SampleLayout, + ) -> Self { + self.px_to_B(format, sample_layout) + } + + pub fn px_to_B(self, format: Format, sample_layout: SampleLayout) -> Self { + self.px_to_el(format, sample_layout) + .el_to_B(format.el_size_B()) + } + + #[no_mangle] + pub extern "C" fn nil_offset4d_el_to_B( + self, + bytes_per_element: u32, + ) -> Self { + self.el_to_B(bytes_per_element) + } + + pub fn el_to_B(self, bytes_per_element: u32) -> Self { + let mut offset_B = self; + offset_B.x *= bytes_per_element; + offset_B + } +} + +impl Minify for Extent4D { + fn minify(self, level: u32) -> Self { + Self { + width: self.width.minify(level), + height: self.height.minify(level), + depth: self.depth.minify(level), + ..self + } + } +} diff --git a/src/nouveau/nil/format.rs b/src/nouveau/nil/format.rs index 3212ecc6b56..a208ac2dd04 100644 --- a/src/nouveau/nil/format.rs +++ b/src/nouveau/nil/format.rs @@ -4,6 +4,8 @@ use nil_rs_bindings::*; use nvidia_headers::{cla297, clb097}; +use crate::extent::Extent4D; + #[repr(C)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Format { @@ -37,6 +39,22 @@ impl Format { unsafe { &*util_format_description((*self).into()) } } + pub(crate) fn el_size_B(&self) -> u32 { + let bits = self.description().block.bits; + debug_assert!(bits % 8 == 0); + bits / 8 + } + + pub(crate) fn el_extent_sa(&self) -> Extent4D { + let desc = self.description(); + Extent4D { + width: desc.block.width, + height: desc.block.height, + depth: desc.block.depth, + array_len: 1, + } + } + fn info(&self) -> &nil_format_info { unsafe { &nil_format_table[self.p_format as usize] } } diff --git a/src/nouveau/nil/image.rs b/src/nouveau/nil/image.rs new file mode 100644 index 00000000000..b44344d54ae --- /dev/null +++ b/src/nouveau/nil/image.rs @@ -0,0 +1,728 @@ +// Copyright © 2024 Collabora, Ltd. +// SPDX-License-Identifier: MIT + +use crate::extent::Extent4D; +use crate::format::Format; +use crate::tiling::Tiling; +use crate::Minify; + +use nil_rs_bindings::*; +use nvidia_headers::{cl9097, clc597}; + +pub const MAX_LEVELS: usize = 16; + +pub type ImageUsageFlags = u16; +pub const IMAGE_USAGE_RENDER_TARGET_BIT: ImageUsageFlags = 1 << 0; +pub const IMAGE_USAGE_DEPTH_BIT: ImageUsageFlags = 1 << 1; +pub const IMAGE_USAGE_STENCIL_BIT: ImageUsageFlags = 1 << 2; +pub const IMAGE_USAGE_TEXTURE_BIT: ImageUsageFlags = 1 << 3; +pub const IMAGE_USAGE_STORAGE_BIT: ImageUsageFlags = 1 << 4; +pub const IMAGE_USAGE_CUBE_BIT: ImageUsageFlags = 1 << 5; +pub const IMAGE_USAGE_2D_VIEW_BIT: ImageUsageFlags = 1 << 6; +pub const IMAGE_USAGE_LINEAR_BIT: ImageUsageFlags = 1 << 7; +pub const IMAGE_USAGE_SPARSE_RESIDENCY_BIT: ImageUsageFlags = 1 << 8; + +#[derive(Clone, Debug, Copy, PartialEq, Default)] +#[repr(u8)] +pub enum ImageDim { + #[default] + _1D = 1, + _2D = 2, + _3D = 3, +} + +#[derive(Clone, Debug, Copy, PartialEq, Default)] +#[repr(u8)] +pub enum SampleLayout { + _1x1 = 0, + _2x1 = 1, + _2x2 = 2, + _4x2 = 3, + _4x4 = 4, + #[default] + Invalid = 5, +} + +impl SampleLayout { + #[no_mangle] + pub extern "C" fn nil_choose_sample_layout(samples: u32) -> SampleLayout { + Self::choose_sample_layout(samples) + } + + pub fn choose_sample_layout(samples: u32) -> SampleLayout { + match samples { + 1 => SampleLayout::_1x1, + 2 => SampleLayout::_2x1, + 4 => SampleLayout::_2x2, + 8 => SampleLayout::_4x2, + 16 => SampleLayout::_4x4, + _ => SampleLayout::Invalid, + } + } + + pub fn px_extent_sa(&self) -> Extent4D { + let (width, height) = match self { + SampleLayout::_1x1 => (1, 1), + SampleLayout::_2x1 => (2, 1), + SampleLayout::_2x2 => (2, 2), + SampleLayout::_4x2 => (4, 2), + SampleLayout::_4x4 => (4, 4), + SampleLayout::Invalid => panic!("Invalid sample layout"), + }; + + Extent4D { + width, + height, + depth: 1, + array_len: 1, + } + } + + #[no_mangle] + pub extern "C" fn nil_px_extent_sa(self) -> Extent4D { + self.px_extent_sa() + } +} + +#[derive(Clone, Debug, Copy, PartialEq)] +#[repr(C)] +pub struct ImageInitInfo { + pub dim: ImageDim, + pub format: Format, + pub extent_px: Extent4D, + pub levels: u32, + pub samples: u32, + pub usage: ImageUsageFlags, +} + +/// Represents the data layout of a single slice (level + lod) of an image. +#[repr(C)] +#[derive(Clone, Debug, Copy, PartialEq, Default)] +pub struct ImageLevel { + pub offset_B: u64, + pub tiling: Tiling, + pub row_stride_B: u32, +} + +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +pub struct Image { + pub dim: ImageDim, + pub format: Format, + pub extent_px: Extent4D, + pub sample_layout: SampleLayout, + pub num_levels: u32, + pub mip_tail_first_lod: u32, + pub levels: [ImageLevel; MAX_LEVELS], + pub array_stride_B: u64, + pub align_B: u32, + pub size_B: u64, + pub tile_mode: u16, + pub pte_kind: u8, +} + +impl Image { + #[no_mangle] + pub extern "C" fn nil_image_new( + dev: &nil_rs_bindings::nv_device_info, + info: &ImageInitInfo, + ) -> Self { + Self::new(dev, info) + } + + pub fn new( + dev: &nil_rs_bindings::nv_device_info, + info: &ImageInitInfo, + ) -> Self { + match info.dim { + ImageDim::_1D => { + assert!(info.extent_px.height == 1); + assert!(info.extent_px.depth == 1); + assert!(info.samples == 1); + } + ImageDim::_2D => { + assert!(info.extent_px.depth == 1); + } + ImageDim::_3D => { + assert!(info.extent_px.array_len == 1); + assert!(info.samples == 1); + } + } + + let sample_layout = SampleLayout::choose_sample_layout(info.samples); + + let tiling = if (info.usage & IMAGE_USAGE_SPARSE_RESIDENCY_BIT) != 0 { + Tiling::sparse(info.format, info.dim) + } else { + Tiling::choose( + info.extent_px, + info.format, + sample_layout, + info.usage, + ) + }; + + let mut image = Self { + dim: info.dim, + format: info.format, + extent_px: info.extent_px, + sample_layout, + num_levels: info.levels, + levels: [ImageLevel::default(); MAX_LEVELS as usize], + array_stride_B: 0, + align_B: 0, + size_B: 0, + tile_mode: 0, + pte_kind: 0, + mip_tail_first_lod: 0, + }; + + if (info.usage & IMAGE_USAGE_SPARSE_RESIDENCY_BIT) != 0 { + image.mip_tail_first_lod = info.levels; + } + + let mut layer_size_B = 0; + for level in 0..info.levels { + let mut lvl_ext_B = image.level_extent_B(level); + if tiling.is_tiled { + let lvl_tiling = tiling.clamp(lvl_ext_B); + + if tiling != lvl_tiling { + image.mip_tail_first_lod = + std::cmp::min(image.mip_tail_first_lod, level); + } + + // Align the size to tiles + let lvl_tiling_ext_B = lvl_tiling.extent_B(); + lvl_ext_B = lvl_ext_B.align(&lvl_tiling_ext_B); + + image.levels[level as usize] = ImageLevel { + offset_B: layer_size_B, + tiling: lvl_tiling, + row_stride_B: lvl_ext_B.width, + }; + } else { + // Linear images need to be 2D + assert!(image.dim == ImageDim::_2D); + // NVIDIA can't do linear and mipmapping + assert!(image.num_levels == 1); + // NVIDIA can't do linear and multisampling + assert!(image.sample_layout == SampleLayout::_1x1); + + image.levels[level as usize] = ImageLevel { + offset_B: layer_size_B, + tiling, + // Row stride needs to be aligned to 128B for render to work + row_stride_B: lvl_ext_B.width.next_multiple_of(128), + }; + + assert!(lvl_ext_B.depth == 1); + } + layer_size_B += image.level_size_B(level); + } + + // We use the tiling for level 0 instead of the tiling selected above + // because, in the case of sparse residency with small images, level 0 may + // have a smaller tiling than what we tried to use. However, the level 0 + // tiling is the one we program in the hardware so that's the one we need + // to use for array stride calculations and the like. + + let lvl0_tiling_size_B = image.levels[0].tiling.size_B(); + + // The array stride has to be aligned to the size of a level 0 tile + image.array_stride_B = + layer_size_B.next_multiple_of(lvl0_tiling_size_B.into()); + + image.size_B = + image.array_stride_B * u64::from(image.extent_px.array_len); + image.align_B = lvl0_tiling_size_B; + + // If the client requested sparse residency, we need a 64K alignment + // or else sparse binding may fail. This is true regardless of + // whether or not we actually select a 64K tile format. + if (info.usage & IMAGE_USAGE_SPARSE_RESIDENCY_BIT) != 0 { + image.align_B = std::cmp::max(image.align_B, 1 << 16); + } + + if image.levels[0].tiling.is_tiled { + image.tile_mode = u16::from(image.levels[0].tiling.y_log2) << 4 + | u16::from(image.levels[0].tiling.z_log2) << 8; + + // TODO: compressed + image.pte_kind = + Self::choose_pte_kind(dev, info.format, info.samples, false); + + image.align_B = std::cmp::max(image.align_B, 4096); + if image.pte_kind >= 0xb && image.pte_kind <= 0xe { + image.align_B = std::cmp::max(image.align_B, 1 << 16); + } + } else { + // Linear images need to be aligned to 128B for render to work + image.align_B = std::cmp::max(image.align_B, 128); + } + + image.size_B = image.size_B.next_multiple_of(image.align_B.into()); + + image + } + + /// The size in bytes of an extent at a given level. + fn level_extent_B(&self, level: u32) -> Extent4D { + let level_extent_px = self.level_extent_px(level); + let level_extent_el = + level_extent_px.px_to_el(self.format, self.sample_layout); + level_extent_el.el_to_B(self.format.el_size_B()) + } + + #[no_mangle] + pub extern "C" fn nil_image_level_extent_px(&self, level: u32) -> Extent4D { + self.level_extent_px(level) + } + + pub fn level_extent_px(&self, level: u32) -> Extent4D { + assert!(level == 0 || self.sample_layout == SampleLayout::_1x1); + self.extent_px.minify(level) + } + + #[no_mangle] + pub extern "C" fn nil_image_level_layer_offset_B( + &self, + level: u32, + layer: u32, + ) -> u64 { + self.level_layer_offset_B(level, layer) + } + + pub fn level_layer_offset_B(&self, level: u32, layer: u32) -> u64 { + assert!(level < self.num_levels); + assert!(layer < self.extent_px.array_len); + self.levels[level as usize].offset_B + + u64::from(layer) * self.array_stride_B + } + + #[no_mangle] + pub extern "C" fn nil_image_mip_tail_offset_B(&self) -> u64 { + self.mip_tail_offset_B() + } + + pub fn mip_tail_offset_B(&self) -> u64 { + assert!(self.mip_tail_first_lod > 0); + self.levels[self.mip_tail_first_lod as usize].offset_B + } + + #[no_mangle] + pub extern "C" fn nil_image_mip_tail_size_B(&self) -> u32 { + self.mip_tail_size_B() + } + + pub fn mip_tail_size_B(&self) -> u32 { + (self.array_stride_B - self.mip_tail_offset_B()) + .try_into() + .unwrap() + } + + #[no_mangle] + pub extern "C" fn nil_image_level_extent_sa(&self, level: u32) -> Extent4D { + self.level_extent_sa(level) + } + + pub fn level_extent_sa(&self, level: u32) -> Extent4D { + let level_extent_px = self.level_extent_px(level); + level_extent_px.px_to_sa(self.sample_layout) + } + + #[no_mangle] + pub extern "C" fn nil_image_level_size_B(&self, level: u32) -> u64 { + self.level_size_B(level) + } + + pub fn level_size_B(&self, level: u32) -> u64 { + assert!(level < self.num_levels); + let lvl_ext_B = self.level_extent_B(level); + let level = &self.levels[level as usize]; + + if level.tiling.is_tiled { + let lvl_tiling_ext_B = level.tiling.extent_B(); + lvl_ext_B.align(&&lvl_tiling_ext_B).size().into() + } else { + assert!(lvl_ext_B.depth == 1); + let row_stride = level.row_stride_B * lvl_ext_B.height; + row_stride.into() + } + } + + #[no_mangle] + pub extern "C" fn nil_image_level_depth_stride_B(&self, level: u32) -> u64 { + self.level_depth_stride_B(level) + } + + pub fn level_depth_stride_B(&self, level: u32) -> u64 { + assert!(level < self.num_levels); + + let lvl_ext_B = self.level_extent_B(level); + let level = &self.levels[level as usize]; + let lvl_tiling_ext_B = level.tiling.extent_B(); + let lvl_ext_B = lvl_ext_B.align(&lvl_tiling_ext_B); + + (lvl_ext_B.width * lvl_ext_B.height).into() + } + + #[no_mangle] + pub extern "C" fn nil_image_for_level( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + self.image_for_level(level, offset_in_bytes_out) + } + + pub fn image_for_level( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + assert!(level < self.num_levels); + let lvl_extent_px = self.level_extent_px(level); + let lvl = self.levels[level as usize]; + let align_B = lvl.tiling.size_B(); + + let mut size_B = self.size_B - lvl.offset_B; + if (level + 1) < self.num_levels { + // This assumes levels are sequential, tightly packed and that each + // level has a higher alignment than the next one. All of this is + // currently true. + let next_lvl_offset_in_bytes = + self.levels[level as usize + 1].offset_B; + assert!(next_lvl_offset_in_bytes > lvl.offset_B); + size_B -= next_lvl_offset_in_bytes - lvl.offset_B; + } + + let mut levels: [ImageLevel; MAX_LEVELS as usize] = Default::default(); + levels[0] = lvl; + + *offset_in_bytes_out = lvl.offset_B; + levels[0].offset_B = 0; + + Self { + extent_px: lvl_extent_px, + num_levels: 1, + levels, + align_B, + size_B, + mip_tail_first_lod: if level < self.mip_tail_first_lod { + 1 + } else { + 0 + }, + ..*self + } + } + + #[no_mangle] + pub extern "C" fn nil_image_level_as_uncompressed( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + self.level_as_uncompressed(level, offset_in_bytes_out) + } + + pub fn level_as_uncompressed( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + assert!(self.sample_layout == SampleLayout::_1x1); + + // Format is arbitrary. Pick one that has the right number of bits. + let uc_format = match self.format.el_size_B() { + 4 => PIPE_FORMAT_R32_UINT, + 8 => PIPE_FORMAT_R32G32_UINT, + 16 => PIPE_FORMAT_R32G32B32A32_UINT, + _ => panic!("No compressed PIPE_FORMAT with this size"), + }; + + let lvl_image = self.image_for_level(level, offset_in_bytes_out); + let mut image_out = lvl_image.clone(); + + image_out.format = uc_format.try_into().unwrap(); + image_out.extent_px = lvl_image + .extent_px + .px_to_el(lvl_image.format, lvl_image.sample_layout); + + image_out + } + + #[no_mangle] + pub extern "C" fn nil_image_3d_level_as_2d_array( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + self._3d_level_as_2d_array(level, offset_in_bytes_out) + } + + pub fn _3d_level_as_2d_array( + &self, + level: u32, + offset_in_bytes_out: &mut u64, + ) -> Self { + assert!(self.dim == ImageDim::_3D); + assert!(self.extent_px.array_len == 1); + assert!(self.sample_layout == SampleLayout::_1x1); + + let mut image_2d_out = self.image_for_level(level, offset_in_bytes_out); + let lvl0 = &image_2d_out.levels[0]; + + assert!(image_2d_out.num_levels == 1); + assert!(!lvl0.tiling.is_tiled || lvl0.tiling.z_log2 == 0); + + let lvl_tiling_ext_B = lvl0.tiling.extent_B(); + let lvl_ext_B = image_2d_out.level_extent_B(0); + let lvl_ext_B = lvl_ext_B.align(&lvl_tiling_ext_B); + let z_stride = u64::from(lvl_ext_B.width * lvl_ext_B.height); + + image_2d_out.dim = ImageDim::_2D; + image_2d_out.extent_px.array_len = image_2d_out.extent_px.depth; + image_2d_out.extent_px.depth = 1; + image_2d_out.array_stride_B = z_stride; + + image_2d_out + } + + fn choose_pte_kind( + dev: &nil_rs_bindings::nv_device_info, + format: Format, + samples: u32, + compressed: bool, + ) -> u8 { + if dev.cls_eng3d >= clc597::TURING_A { + Self::tu102_choose_pte_kind(format, compressed) + } else if dev.cls_eng3d >= cl9097::FERMI_A { + Self::nvc0_choose_pte_kind(format, samples, compressed) + } else { + panic!("Unsupported 3d engine class") + } + } + + fn tu102_choose_pte_kind(format: Format, compressed: bool) -> u8 { + match pipe_format::from(format) { + PIPE_FORMAT_Z16_UNORM => { + if compressed { + 0x0b // NV_MMU_PTE_KIND_Z16_COMPRESSIBLE_DISABLE_PLC + } else { + 0x01 // NV_MMU_PTE_KIND_Z16 + } + } + PIPE_FORMAT_X8Z24_UNORM + | PIPE_FORMAT_S8X24_UINT + | PIPE_FORMAT_S8_UINT_Z24_UNORM => { + if compressed { + 0x0e // NV_MMU_PTE_KIND_Z24S8_COMPRESSIBLE_DISABLE_PLC + } else { + 0x05 // NV_MMU_PTE_KIND_Z24S8 + } + } + PIPE_FORMAT_X24S8_UINT + | PIPE_FORMAT_Z24X8_UNORM + | PIPE_FORMAT_Z24_UNORM_S8_UINT => { + if compressed { + 0x0c // NV_MMU_PTE_KIND_S8Z24_COMPRESSIBLE_DISABLE_PLC + } else { + 0x03 // NV_MMU_PTE_KIND_S8Z24 + } + } + PIPE_FORMAT_X32_S8X24_UINT | PIPE_FORMAT_Z32_FLOAT_S8X24_UINT => { + if compressed { + 0x0d // NV_MMU_PTE_KIND_ZF32_X24S8_COMPRESSIBLE_DISABLE_PLC + } else { + 0x04 // NV_MMU_PTE_KIND_ZF32_X24S8 + } + } + PIPE_FORMAT_Z32_FLOAT => 0x06, + _ => 0, + } + } + + fn nvc0_choose_pte_kind( + format: Format, + samples: u32, + compressed: bool, + ) -> u8 { + let ms = samples.ilog2() as u8; + match pipe_format::from(format) { + PIPE_FORMAT_Z16_UNORM => { + if compressed { + 0x02 + ms + } else { + 0x01 + } + } + PIPE_FORMAT_X8Z24_UNORM + | PIPE_FORMAT_S8X24_UINT + | PIPE_FORMAT_S8_UINT_Z24_UNORM => { + if compressed { + 0x51 + ms + } else { + 0x46 + } + } + PIPE_FORMAT_X24S8_UINT + | PIPE_FORMAT_Z24X8_UNORM + | PIPE_FORMAT_Z24_UNORM_S8_UINT => { + if compressed { + 0x17 + ms + } else { + 0x11 + } + } + PIPE_FORMAT_X32_S8X24_UINT | PIPE_FORMAT_Z32_FLOAT_S8X24_UINT => { + if compressed { + 0xce + ms + } else { + 0xc3 + } + } + _ => { + let blocksize_bits = format.el_size_B() * 8; + match blocksize_bits { + 128 => { + if compressed { + 0xf4 + ms * 2 + } else { + 0xfe + } + } + 64 => { + if compressed { + match samples { + 1 => 0xe6, + 2 => 0xeb, + 4 => 0xed, + 8 => 0xf2, + _ => panic!("Unsupported sample count"), + } + } else { + 0xfe + } + } + 32 => { + if compressed && ms != 0 { + match samples { + // This one makes things blurry: + // 1 => 0xdb + 2 => 0xdd, + 4 => 0xdf, + 8 => 0xe4, + _ => 0, + } + } else { + 0xfe + } + } + 16 | 8 => 0xfe, + _ => 0, + } + } + } + } + + #[no_mangle] + pub extern "C" fn nil_msaa_image_as_sa(&self) -> Self { + self.msaa_as_samples() + } + + /// For a multisampled image, returns an image of samples + /// + /// The resulting image is supersampled with each pixel in the original + /// consuming some number pixels in the supersampled images according to the + /// original image's sample layout + pub fn msaa_as_samples(&self) -> Self { + assert!(self.dim == ImageDim::_2D); + assert!(self.num_levels == 1); + + let extent_in_samples = self.extent_px.px_to_sa(self.sample_layout); + let mut out = self.clone(); + out.extent_px = extent_in_samples; + out.sample_layout = SampleLayout::_1x1; + out + } + + #[no_mangle] + pub extern "C" fn nil_image_level_z_offset_B( + &self, + level: u32, + z: u32, + ) -> u64 { + self.level_z_offset_B(level, z) + } + + pub fn level_z_offset_B(&self, level: u32, z: u32) -> u64 { + assert!(level < self.num_levels); + let lvl_extent_px = self.level_extent_px(level); + assert!(z < lvl_extent_px.depth); + + let lvl_tiling = &self.levels[level as usize].tiling; + let z_tl = z >> lvl_tiling.z_log2; + let z_gob = z & ((1 << lvl_tiling.z_log2) - 1); + + let lvl_extent_tl = + lvl_extent_px.px_to_tl(lvl_tiling, self.format, self.sample_layout); + let offset_B = u64::from( + lvl_extent_tl.width + * lvl_extent_tl.height + * z_tl + * lvl_tiling.size_B(), + ); + + let tiling_extent_B = lvl_tiling.extent_B(); + let offset_B = offset_B + + u64::from(tiling_extent_B.width * tiling_extent_B.height * z_gob); + offset_B + } +} + +#[allow(dead_code)] +#[derive(Clone, Debug, Copy, PartialEq)] +#[repr(u8)] +pub enum ViewType { + _1D, + _2D, + _3D, + _3DSliced, + Cube, + _1DArray, + _2DArray, + CubeArray, +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq)] +pub struct View { + pub view_type: ViewType, + + /// The format to use in the view + /// + /// This may differ from the format of the actual isl_surf but must have the + /// same block size. + pub format: Format, + + pub base_level: u32, + pub num_levels: u32, + + /// Base array layer + /// + /// For cube maps, both base_array_layer and array_len should be specified in + /// terms of 2-D layers and must be a multiple of 6. + pub base_array_layer: u32, + + /// Array Length + /// + /// Indicates the number of array elements starting at Base Array Layer. + pub array_len: u32, + + pub swizzle: [nil_rs_bindings::pipe_swizzle; 4], + + // VK_EXT_image_view_min_lod + pub min_lod_clamp: f32, +} diff --git a/src/nouveau/nil/lib.rs b/src/nouveau/nil/lib.rs index 0982fe77e36..8c41f7de7b3 100644 --- a/src/nouveau/nil/lib.rs +++ b/src/nouveau/nil/lib.rs @@ -4,4 +4,32 @@ extern crate nil_rs_bindings; extern crate nvidia_headers; +mod extent; mod format; +mod image; +mod tiling; + +pub trait ILog2Ceil { + fn ilog2_ceil(self) -> Self; +} + +impl ILog2Ceil for u32 { + fn ilog2_ceil(self) -> Self { + if self <= 1 { + 0 + } else { + (self - 1).ilog2() + 1 + } + } +} + +pub trait Minify { + // Required method + fn minify(self, rhs: Rhs) -> Self; +} + +impl Minify for u32 { + fn minify(self, level: u32) -> u32 { + std::cmp::max(1, self >> level) + } +} diff --git a/src/nouveau/nil/meson.build b/src/nouveau/nil/meson.build index 054afc48efb..0f20d6400bb 100644 --- a/src/nouveau/nil/meson.build +++ b/src/nouveau/nil/meson.build @@ -16,7 +16,6 @@ prog_cbindgen = find_program('cbindgen', required : false, native : true) libnil_files = files( 'nil.h', - 'nil_image.c', 'nil_image_tic.c', ) @@ -38,7 +37,10 @@ libnil_deps = [ _libnil_rs_files = files( 'lib.rs', # lib.rs has to come first + 'extent.rs', 'format.rs', + 'image.rs', + 'tiling.rs', ) _libnil_rs_deps = [ diff --git a/src/nouveau/nil/nil.h b/src/nouveau/nil/nil.h index 2f41bced33f..71a376a92d7 100644 --- a/src/nouveau/nil/nil.h +++ b/src/nouveau/nil/nil.h @@ -16,35 +16,6 @@ struct nv_device_info; -enum ENUM_PACKED nil_image_dim { - NIL_IMAGE_DIM_1D = 1, - NIL_IMAGE_DIM_2D = 2, - NIL_IMAGE_DIM_3D = 3, -}; - -enum ENUM_PACKED nil_sample_layout { - NIL_SAMPLE_LAYOUT_1X1, - NIL_SAMPLE_LAYOUT_2X1, - NIL_SAMPLE_LAYOUT_2X2, - NIL_SAMPLE_LAYOUT_4X2, - NIL_SAMPLE_LAYOUT_4X4, - NIL_SAMPLE_LAYOUT_INVALID, -}; - -enum nil_sample_layout nil_choose_sample_layout(uint32_t samples); - -enum nil_image_usage_flags { - NIL_IMAGE_USAGE_RENDER_TARGET_BIT = BITFIELD_BIT(0), - NIL_IMAGE_USAGE_DEPTH_BIT = BITFIELD_BIT(1), - NIL_IMAGE_USAGE_STENCIL_BIT = BITFIELD_BIT(2), - NIL_IMAGE_USAGE_TEXTURE_BIT = BITFIELD_BIT(3), - NIL_IMAGE_USAGE_STORAGE_BIT = BITFIELD_BIT(4), - NIL_IMAGE_USAGE_CUBE_BIT = BITFIELD_BIT(5), - NIL_IMAGE_USAGE_2D_VIEW_BIT = BITFIELD_BIT(6), - NIL_IMAGE_USAGE_LINEAR_BIT = BITFIELD_BIT(7), - NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT = BITFIELD_BIT(8), -}; - enum ENUM_PACKED nil_view_type { NIL_VIEW_TYPE_1D, NIL_VIEW_TYPE_2D, @@ -56,116 +27,6 @@ enum ENUM_PACKED nil_view_type { NIL_VIEW_TYPE_CUBE_ARRAY, }; -struct nil_extent4d { - union { uint32_t w, width; }; - union { uint32_t h, height; }; - union { uint32_t d, depth; }; - union { uint32_t a, array_len; }; -}; - -static inline struct nil_extent4d -nil_extent4d(uint32_t w, uint32_t h, uint32_t d, uint32_t a) -{ - struct nil_extent4d e; - e.w = w; - e.h = h; - e.d = d; - e.a = a; - return e; -} - -struct nil_extent4d nil_px_extent_sa(enum nil_sample_layout sample_layout); - -struct nil_extent4d -nil_extent4d_px_to_el(struct nil_extent4d extent_px, - enum pipe_format format, - enum nil_sample_layout sample_layout); - -struct nil_offset4d { - uint32_t x; - uint32_t y; - uint32_t z; - uint32_t a; -}; - -static inline struct nil_offset4d -nil_offset4d(uint32_t x, uint32_t y, uint32_t z, uint32_t a) -{ - struct nil_offset4d o; - o.x = x; - o.y = y; - o.z = z; - o.a = a; - return o; -} - -struct nil_offset4d -nil_offset4d_px_to_el(struct nil_offset4d offset_px, - enum pipe_format format, - enum nil_sample_layout sample_layout); - -#define NIL_GOB_WIDTH_B 64 -#define NIL_GOB_HEIGHT(gob_height_is_8) ((gob_height_is_8) ? 8 : 4) -#define NIL_GOB_DEPTH 1 -#define NIL_MAX_LEVELS 16 - -struct nil_tiling { - bool is_tiled:1; - bool gob_height_is_8:1; /**< GOB height is 4 or 8 */ - uint8_t x_log2:3; /**< log2 of the Y tile dimension in GOBs */ - uint8_t y_log2:3; /**< log2 of the Y tile dimension in GOBs */ - uint8_t z_log2:3; /**< log2 of the Z tile dimension in GOBs */ - uint8_t pad:5; -}; -static_assert(sizeof(struct nil_tiling) == 2, "This struct has no holes"); - -struct nil_image_init_info { - enum nil_image_dim dim; - enum pipe_format format; - - struct nil_extent4d extent_px; - uint32_t levels; - uint32_t samples; - - enum nil_image_usage_flags usage; -}; - -/** Represents the data layout of a single slice (level+lod) of an image */ -struct nil_image_level { - /** Offset into the image of this level in bytes */ - uint64_t offset_B; - - /** Tiling for this level */ - struct nil_tiling tiling; - - /** Stride between rows in bytes - * - * For linear images, this is the obvious stride. For tiled images, it's - * the width of the mip level rounded up to the tile size. - */ - uint32_t row_stride_B; -}; - -struct nil_image { - enum nil_image_dim dim; - enum pipe_format format; - - struct nil_extent4d extent_px; - enum nil_sample_layout sample_layout; - uint8_t num_levels; - uint8_t mip_tail_first_lod; - - struct nil_image_level levels[NIL_MAX_LEVELS]; - - uint32_t array_stride_B; - - uint32_t align_B; - uint64_t size_B; - - uint16_t tile_mode; - uint8_t pte_kind; -}; - struct nil_view { enum nil_view_type type; @@ -201,82 +62,6 @@ struct nil_view { float min_lod_clamp; }; -struct nil_extent4d -nil_tiling_extent_B(struct nil_tiling tiling); - -uint32_t -nil_tiling_size_B(struct nil_tiling tiling); - -struct nil_extent4d -nil_tiling_extent_px(struct nil_tiling tiling, enum pipe_format format, - enum nil_sample_layout sample_layout); - -bool nil_image_init(struct nv_device_info *dev, - struct nil_image *image, - const struct nil_image_init_info *restrict info); - -static inline uint64_t -nil_image_level_layer_offset_B(const struct nil_image *image, - uint32_t level, uint32_t layer) -{ - assert(level < image->num_levels); - assert(layer < image->extent_px.array_len); - return image->levels[level].offset_B + (layer * image->array_stride_B); -} - -uint64_t nil_image_level_z_offset_B(const struct nil_image *image, - uint32_t level, uint32_t z); - -static inline uint32_t -nil_image_mip_tail_offset_B(const struct nil_image *image) -{ - assert(image->mip_tail_first_lod > 0); - return image->levels[image->mip_tail_first_lod].offset_B; -} - -static inline uint32_t -nil_image_mip_tail_size_B(const struct nil_image *image) -{ - return image->array_stride_B - nil_image_mip_tail_offset_B(image); -} - -struct nil_extent4d -nil_extent4d_px_to_tl(struct nil_extent4d extent_px, - struct nil_tiling tiling, enum pipe_format format, - enum nil_sample_layout sample_layout); - -struct nil_offset4d -nil_offset4d_px_to_tl(struct nil_offset4d offset_px, - struct nil_tiling tiling, enum pipe_format format, - enum nil_sample_layout sample_layout); - -struct nil_extent4d nil_image_level_extent_px(const struct nil_image *image, - uint32_t level); -struct nil_extent4d nil_image_level_extent_sa(const struct nil_image *image, - uint32_t level); -uint64_t nil_image_level_size_B(const struct nil_image *image, - uint32_t level); -uint64_t nil_image_level_depth_stride_B(const struct nil_image *image, - uint32_t level); - -void nil_image_for_level(const struct nil_image *image_in, - uint32_t level, - struct nil_image *level_image_out, - uint64_t *offset_B_out); - -void nil_image_level_as_uncompressed(const struct nil_image *image_3d, - uint32_t level, - struct nil_image *uc_image_out, - uint64_t *offset_B_out); - -void nil_image_3d_level_as_2d_array(const struct nil_image *image_3d, - uint32_t level, - struct nil_image *image_2d_out, - uint64_t *offset_B_out); - -void nil_msaa_image_as_sa(const struct nil_image *image_msaa, - struct nil_image *image_sa_out); - void nil_image_fill_tic(struct nv_device_info *dev, const struct nil_image *image, const struct nil_view *view, @@ -289,9 +74,4 @@ void nil_buffer_fill_tic(struct nv_device_info *dev, uint32_t num_elements, void *desc_out); -struct nil_extent4d -nil_sparse_block_extent_px(enum pipe_format format, - enum nil_image_dim dim, - enum nil_sample_layout sample_layout); - #endif /* NIL_H */ diff --git a/src/nouveau/nil/nil_image.c b/src/nouveau/nil/nil_image.c deleted file mode 100644 index d8596fe7c92..00000000000 --- a/src/nouveau/nil/nil_image.c +++ /dev/null @@ -1,874 +0,0 @@ -/* - * Copyright © 2022 Collabora Ltd. - * SPDX-License-Identifier: MIT - */ -#include "nil.h" - -#include "util/u_math.h" - -#include "nouveau_device.h" - -#include "cl9097.h" -#include "clc597.h" - -static struct nil_extent4d -nil_minify_extent4d(struct nil_extent4d extent, uint32_t level) -{ - return (struct nil_extent4d) { - .w = u_minify(extent.w, level), - .h = u_minify(extent.h, level), - .d = u_minify(extent.d, level), - .a = extent.a, - }; -} - -static struct nil_extent4d -nil_extent4d_div_round_up(struct nil_extent4d num, struct nil_extent4d denom) -{ - return (struct nil_extent4d) { - .w = DIV_ROUND_UP(num.w, denom.w), - .h = DIV_ROUND_UP(num.h, denom.h), - .d = DIV_ROUND_UP(num.d, denom.d), - .a = DIV_ROUND_UP(num.a, denom.a), - }; -} - -static struct nil_extent4d -nil_extent4d_mul(struct nil_extent4d a, struct nil_extent4d b) -{ - return (struct nil_extent4d) { - .w = a.w * b.w, - .h = a.h * b.h, - .d = a.d * b.d, - .a = a.a * b.a, - }; -} - -static struct nil_offset4d -nil_offset4d_div_round_down(struct nil_offset4d num, struct nil_extent4d denom) -{ - return (struct nil_offset4d) { - .x = num.x / denom.w, - .y = num.y / denom.h, - .z = num.z / denom.d, - .a = num.a / denom.a, - }; -} - -static struct nil_offset4d -nil_offset4d_mul(struct nil_offset4d a, struct nil_extent4d b) -{ - return (struct nil_offset4d) { - .x = a.x * b.w, - .y = a.y * b.h, - .z = a.z * b.d, - .a = a.a * b.a, - }; -} - -static struct nil_extent4d -nil_extent4d_align(struct nil_extent4d ext, struct nil_extent4d alignment) -{ - return (struct nil_extent4d) { - .w = align(ext.w, alignment.w), - .h = align(ext.h, alignment.h), - .d = align(ext.d, alignment.d), - .a = align(ext.a, alignment.a), - }; -} - -struct nil_extent4d -nil_px_extent_sa(enum nil_sample_layout sample_layout) -{ - switch (sample_layout) { - case NIL_SAMPLE_LAYOUT_1X1: return nil_extent4d(1, 1, 1, 1); - case NIL_SAMPLE_LAYOUT_2X1: return nil_extent4d(2, 1, 1, 1); - case NIL_SAMPLE_LAYOUT_2X2: return nil_extent4d(2, 2, 1, 1); - case NIL_SAMPLE_LAYOUT_4X2: return nil_extent4d(4, 2, 1, 1); - case NIL_SAMPLE_LAYOUT_4X4: return nil_extent4d(4, 4, 1, 1); - default: unreachable("Invalid sample layout"); - } -} - -static inline struct nil_extent4d -nil_el_extent_sa(enum pipe_format format) -{ - const struct util_format_description *fmt = - util_format_description(format); - - return (struct nil_extent4d) { - .w = fmt->block.width, - .h = fmt->block.height, - .d = fmt->block.depth, - .a = 1, - }; -} - -static struct nil_extent4d -nil_extent4d_px_to_sa(struct nil_extent4d extent_px, - enum nil_sample_layout sample_layout) -{ - return nil_extent4d_mul(extent_px, nil_px_extent_sa(sample_layout)); -} - -struct nil_extent4d -nil_extent4d_px_to_el(struct nil_extent4d extent_px, - enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_extent4d extent_sa = - nil_extent4d_px_to_sa(extent_px, sample_layout); - - return nil_extent4d_div_round_up(extent_sa, nil_el_extent_sa(format)); -} - -struct nil_offset4d -nil_offset4d_px_to_el(struct nil_offset4d offset_px, - enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_offset4d offset_sa = - nil_offset4d_mul(offset_px, nil_px_extent_sa(sample_layout)); - - return nil_offset4d_div_round_down(offset_sa, nil_el_extent_sa(format)); -} - -static struct nil_extent4d -nil_extent4d_el_to_B(struct nil_extent4d extent_el, - uint32_t B_per_el) -{ - struct nil_extent4d extent_B = extent_el; - extent_B.w *= B_per_el; - return extent_B; -} - -static struct nil_offset4d -nil_offset4d_el_to_B(struct nil_offset4d offset_el, - uint32_t B_per_el) -{ - struct nil_offset4d offset_B = offset_el; - offset_B.x *= B_per_el; - return offset_B; -} - -static struct nil_extent4d -nil_extent4d_px_to_B(struct nil_extent4d extent_px, - enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_extent4d extent_el = - nil_extent4d_px_to_el(extent_px, format, sample_layout); - const uint32_t B_per_el = util_format_get_blocksize(format); - - return nil_extent4d_el_to_B(extent_el, B_per_el); -} - -static struct nil_offset4d -nil_offset4d_px_to_B(struct nil_offset4d offset_px, - enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_offset4d offset_el = - nil_offset4d_px_to_el(offset_px, format, sample_layout); - const uint32_t B_per_el = util_format_get_blocksize(format); - - return nil_offset4d_el_to_B(offset_el, B_per_el); -} - -static struct nil_extent4d -nil_extent4d_B_to_GOB(struct nil_extent4d extent_B, - bool gob_height_is_8) -{ - const struct nil_extent4d gob_extent_B = { - .w = NIL_GOB_WIDTH_B, - .h = NIL_GOB_HEIGHT(gob_height_is_8), - .d = NIL_GOB_DEPTH, - .a = 1, - }; - - return nil_extent4d_div_round_up(extent_B, gob_extent_B); -} - -struct nil_extent4d -nil_tiling_extent_B(struct nil_tiling tiling) -{ - if (tiling.is_tiled) { - return (struct nil_extent4d) { - .w = NIL_GOB_WIDTH_B << tiling.x_log2, - .h = NIL_GOB_HEIGHT(tiling.gob_height_is_8) << tiling.y_log2, - .d = NIL_GOB_DEPTH << tiling.z_log2, - .a = 1, - }; - } else { - /* We handle linear images in nil_image_create */ - return nil_extent4d(1, 1, 1, 1); - } -} - -/** Clamps the tiling to less than 2x the given extent in each dimension - * - * This operation is done by the hardware at each LOD. - */ -static struct nil_tiling -nil_tiling_clamp(struct nil_tiling tiling, struct nil_extent4d extent_B) -{ - if (!tiling.is_tiled) - return tiling; - - const struct nil_extent4d tiling_extent_B = nil_tiling_extent_B(tiling); - - /* The moment the LOD is smaller than a tile, tiling.x_log2 goes to 0 */ - if (extent_B.w < tiling_extent_B.w || - extent_B.h < tiling_extent_B.h || - extent_B.d < tiling_extent_B.d) - tiling.x_log2 = 0; - - const struct nil_extent4d extent_GOB = - nil_extent4d_B_to_GOB(extent_B, tiling.gob_height_is_8); - - tiling.y_log2 = MIN2(tiling.y_log2, util_logbase2_ceil(extent_GOB.h)); - tiling.z_log2 = MIN2(tiling.z_log2, util_logbase2_ceil(extent_GOB.d)); - - return tiling; -} - -static bool -nil_tiling_eq(struct nil_tiling a, struct nil_tiling b) -{ - return memcmp(&a, &b, sizeof(b)) == 0; -} - -enum nil_sample_layout -nil_choose_sample_layout(uint32_t samples) -{ - switch (samples) { - case 1: return NIL_SAMPLE_LAYOUT_1X1; - case 2: return NIL_SAMPLE_LAYOUT_2X1; - case 4: return NIL_SAMPLE_LAYOUT_2X2; - case 8: return NIL_SAMPLE_LAYOUT_4X2; - case 16: return NIL_SAMPLE_LAYOUT_4X4; - default: - unreachable("Unsupported sample count"); - } -} - -static struct nil_tiling -choose_tiling(struct nil_extent4d extent_px, - enum pipe_format format, - enum nil_sample_layout sample_layout, - enum nil_image_usage_flags usage) -{ - if (usage & NIL_IMAGE_USAGE_LINEAR_BIT) - return (struct nil_tiling) { .is_tiled = false }; - - struct nil_tiling tiling = { - .is_tiled = true, - .gob_height_is_8 = true, - .y_log2 = 5, - .z_log2 = 5, - }; - - if (usage & NIL_IMAGE_USAGE_2D_VIEW_BIT) - tiling.z_log2 = 0; - - const struct nil_extent4d extent_B = - nil_extent4d_px_to_B(extent_px, format, sample_layout); - - return nil_tiling_clamp(tiling, extent_B); -} - -static struct nil_extent4d -nil_sparse_block_extent_el(enum pipe_format format, - enum nil_image_dim dim) -{ - /* Taken from Vulkan 1.3.279 spec section entitled "Standard Sparse Image - * Block Shapes". - */ - switch (dim) { - case NIL_IMAGE_DIM_2D: - switch (util_format_get_blocksizebits(format)) { - case 8: return nil_extent4d(256, 256, 1, 1); - case 16: return nil_extent4d(256, 128, 1, 1); - case 32: return nil_extent4d(128, 128, 1, 1); - case 64: return nil_extent4d(128, 64, 1, 1); - case 128: return nil_extent4d(64, 64, 1, 1); - default: unreachable("Invalid texel size"); - } - case NIL_IMAGE_DIM_3D: - switch (util_format_get_blocksizebits(format)) { - case 8: return nil_extent4d(64, 32, 32, 1); - case 16: return nil_extent4d(32, 32, 32, 1); - case 32: return nil_extent4d(32, 32, 16, 1); - case 64: return nil_extent4d(32, 16, 16, 1); - case 128: return nil_extent4d(16, 16, 16, 1); - default: unreachable("Invalid texel size"); - } - default: - unreachable("Invalid dimension"); - } -} - -struct nil_extent4d -nil_sparse_block_extent_px(enum pipe_format format, - enum nil_image_dim dim, - enum nil_sample_layout sample_layout) -{ - struct nil_extent4d block_extent_el = - nil_sparse_block_extent_el(format, dim); - const struct nil_extent4d el_extent_sa = nil_el_extent_sa(format); - struct nil_extent4d block_extent_sa = - nil_extent4d_mul(block_extent_el, el_extent_sa); - - return nil_extent4d_div_round_up(block_extent_sa, - nil_px_extent_sa(sample_layout)); -} - -static struct nil_extent4d -nil_sparse_block_extent_B(enum pipe_format format, - enum nil_image_dim dim) -{ - const struct nil_extent4d block_extent_el = - nil_sparse_block_extent_el(format, dim); - const uint32_t B_per_el = util_format_get_blocksize(format); - return nil_extent4d_el_to_B(block_extent_el, B_per_el); -} - -static struct nil_tiling -sparse_tiling(enum pipe_format format, enum nil_image_dim dim) -{ - const struct nil_extent4d sparse_block_extent_B = - nil_sparse_block_extent_B(format, dim); - - assert(util_is_power_of_two_or_zero(sparse_block_extent_B.w)); - assert(util_is_power_of_two_or_zero(sparse_block_extent_B.h)); - assert(util_is_power_of_two_or_zero(sparse_block_extent_B.d)); - - const bool gob_height_is_8 = true; - const struct nil_extent4d sparse_block_extent_GOB = - nil_extent4d_B_to_GOB(sparse_block_extent_B, gob_height_is_8); - - return (struct nil_tiling) { - .is_tiled = true, - .gob_height_is_8 = gob_height_is_8, - .x_log2 = util_logbase2(sparse_block_extent_GOB.w), - .y_log2 = util_logbase2(sparse_block_extent_GOB.h), - .z_log2 = util_logbase2(sparse_block_extent_GOB.d), - }; -} - -uint32_t -nil_tiling_size_B(struct nil_tiling tiling) -{ - const struct nil_extent4d extent_B = nil_tiling_extent_B(tiling); - return extent_B.w * extent_B.h * extent_B.d * extent_B.a; -} - -static struct nil_extent4d -nil_extent4d_B_to_tl(struct nil_extent4d extent_B, - struct nil_tiling tiling) -{ - return nil_extent4d_div_round_up(extent_B, nil_tiling_extent_B(tiling)); -} - -struct nil_extent4d -nil_extent4d_px_to_tl(struct nil_extent4d extent_px, - struct nil_tiling tiling, enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_extent4d extent_B = - nil_extent4d_px_to_B(extent_px, format, sample_layout); - - const struct nil_extent4d tiling_extent_B = nil_tiling_extent_B(tiling); - - return nil_extent4d_div_round_up(extent_B, tiling_extent_B); -} - -struct nil_offset4d -nil_offset4d_px_to_tl(struct nil_offset4d offset_px, - struct nil_tiling tiling, enum pipe_format format, - enum nil_sample_layout sample_layout) -{ - const struct nil_offset4d offset_B = - nil_offset4d_px_to_B(offset_px, format, sample_layout); - - const struct nil_extent4d tiling_extent_B = nil_tiling_extent_B(tiling); - - return nil_offset4d_div_round_down(offset_B, tiling_extent_B); -} - -struct nil_extent4d -nil_image_level_extent_px(const struct nil_image *image, uint32_t level) -{ - assert(level == 0 || image->sample_layout == NIL_SAMPLE_LAYOUT_1X1); - - return nil_minify_extent4d(image->extent_px, level); -} - -struct nil_extent4d -nil_image_level_extent_sa(const struct nil_image *image, uint32_t level) -{ - const struct nil_extent4d level_extent_px = - nil_image_level_extent_px(image, level); - - return nil_extent4d_px_to_sa(level_extent_px, image->sample_layout); -} - -static struct nil_extent4d -image_level_extent_B(const struct nil_image *image, uint32_t level) -{ - const struct nil_extent4d level_extent_px = - nil_image_level_extent_px(image, level); - - return nil_extent4d_px_to_B(level_extent_px, image->format, - image->sample_layout); -} - -uint64_t -nil_image_level_size_B(const struct nil_image *image, - uint32_t level) -{ - assert(level < image->num_levels); - - /* See the nil_image::levels[] computations */ - struct nil_extent4d lvl_ext_B = image_level_extent_B(image, level); - - if (image->levels[level].tiling.is_tiled) { - struct nil_extent4d lvl_tiling_ext_B = - nil_tiling_extent_B(image->levels[level].tiling); - lvl_ext_B = nil_extent4d_align(lvl_ext_B, lvl_tiling_ext_B); - - return (uint64_t)lvl_ext_B.w * - (uint64_t)lvl_ext_B.h * - (uint64_t)lvl_ext_B.d; - } else { - assert(lvl_ext_B.d == 1); - return (uint64_t)image->levels[level].row_stride_B * - (uint64_t)lvl_ext_B.h; - } -} - -static uint8_t -tu102_choose_pte_kind(enum pipe_format format, bool compressed) -{ - switch (format) { - case PIPE_FORMAT_Z16_UNORM: - if (compressed) - return 0x0b; // NV_MMU_PTE_KIND_Z16_COMPRESSIBLE_DISABLE_PLC - else - return 0x01; // NV_MMU_PTE_KIND_Z16 - case PIPE_FORMAT_X8Z24_UNORM: - case PIPE_FORMAT_S8X24_UINT: - case PIPE_FORMAT_S8_UINT_Z24_UNORM: - if (compressed) - return 0x0e; // NV_MMU_PTE_KIND_Z24S8_COMPRESSIBLE_DISABLE_PLC - else - return 0x05; // NV_MMU_PTE_KIND_Z24S8 - case PIPE_FORMAT_X24S8_UINT: - case PIPE_FORMAT_Z24X8_UNORM: - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - if (compressed) - return 0x0c; // NV_MMU_PTE_KIND_S8Z24_COMPRESSIBLE_DISABLE_PLC - else - return 0x03; // NV_MMU_PTE_KIND_S8Z24 - case PIPE_FORMAT_X32_S8X24_UINT: - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: - if (compressed) - return 0x0d; // NV_MMU_PTE_KIND_ZF32_X24S8_COMPRESSIBLE_DISABLE_PLC - else - return 0x04; // NV_MMU_PTE_KIND_ZF32_X24S8 - case PIPE_FORMAT_Z32_FLOAT: - return 0x06; - default: - return 0; - } -} - -static uint8_t -nvc0_choose_pte_kind(enum pipe_format format, - uint32_t samples, bool compressed) -{ - const unsigned ms = util_logbase2(samples); - - switch (format) { - case PIPE_FORMAT_Z16_UNORM: - if (compressed) - return 0x02 + ms; - else - return 0x01; - case PIPE_FORMAT_X8Z24_UNORM: - case PIPE_FORMAT_S8X24_UINT: - case PIPE_FORMAT_S8_UINT_Z24_UNORM: - if (compressed) - return 0x51 + ms; - else - return 0x46; - case PIPE_FORMAT_X24S8_UINT: - case PIPE_FORMAT_Z24X8_UNORM: - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - if (compressed) - return 0x17 + ms; - else - return 0x11; - break; - case PIPE_FORMAT_Z32_FLOAT: - if (compressed) - return 0x86 + ms; - else - return 0x7b; - break; - case PIPE_FORMAT_X32_S8X24_UINT: - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: - if (compressed) - return 0xce + ms; - else - return 0xc3; - default: - switch (util_format_get_blocksizebits(format)) { - case 128: - if (compressed) - return 0xf4 + ms * 2; - else - return 0xfe; - break; - case 64: - if (compressed) { - switch (samples) { - case 1: return 0xe6; - case 2: return 0xeb; - case 4: return 0xed; - case 8: return 0xf2; - default: return 0; - } - } else { - return 0xfe; - } - break; - case 32: - if (compressed && ms) { - switch (samples) { - /* This one makes things blurry: - case 1: return 0xdb; - */ - case 2: return 0xdd; - case 4: return 0xdf; - case 8: return 0xe4; - default: return 0; - } - } else { - return 0xfe; - } - break; - case 16: - case 8: - return 0xfe; - default: - return 0; - } - } -} - -static uint8_t -nil_choose_pte_kind(struct nv_device_info *dev, - enum pipe_format format, - uint32_t samples, bool compressed) -{ - if (dev->cls_eng3d >= TURING_A) - return tu102_choose_pte_kind(format, compressed); - else if (dev->cls_eng3d >= FERMI_A) - return nvc0_choose_pte_kind(format, samples, compressed); - else - unreachable("Unsupported 3D engine class"); -} - -bool -nil_image_init(struct nv_device_info *dev, - struct nil_image *image, - const struct nil_image_init_info *restrict info) -{ - switch (info->dim) { - case NIL_IMAGE_DIM_1D: - assert(info->extent_px.h == 1); - assert(info->extent_px.d == 1); - assert(info->samples == 1); - break; - case NIL_IMAGE_DIM_2D: - assert(info->extent_px.d == 1); - break; - case NIL_IMAGE_DIM_3D: - assert(info->extent_px.a == 1); - assert(info->samples == 1); - break; - } - - const enum nil_sample_layout sample_layout = - nil_choose_sample_layout(info->samples); - - struct nil_tiling tiling; - if (info->usage & NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT) { - tiling = sparse_tiling(info->format, info->dim); - } else { - tiling = choose_tiling(info->extent_px, info->format, - sample_layout, info->usage); - } - - *image = (struct nil_image) { - .dim = info->dim, - .format = info->format, - .extent_px = info->extent_px, - .sample_layout = sample_layout, - .num_levels = info->levels, - }; - - /* If the client requested sparse, default mip_tail_firs_lod to the number - * of mip levels and we'll clamp it as needed in the loop below. - */ - if (info->usage & NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT) - image->mip_tail_first_lod = info->levels; - - uint64_t layer_size_B = 0; - for (uint32_t l = 0; l < info->levels; l++) { - struct nil_extent4d lvl_ext_B = image_level_extent_B(image, l); - if (tiling.is_tiled) { - struct nil_tiling lvl_tiling = nil_tiling_clamp(tiling, lvl_ext_B); - - if (!nil_tiling_eq(tiling, lvl_tiling)) - image->mip_tail_first_lod = MIN2(image->mip_tail_first_lod, l); - - /* Align the size to tiles */ - struct nil_extent4d lvl_tiling_ext_B = nil_tiling_extent_B(lvl_tiling); - lvl_ext_B = nil_extent4d_align(lvl_ext_B, lvl_tiling_ext_B); - - image->levels[l] = (struct nil_image_level) { - .offset_B = layer_size_B, - .tiling = lvl_tiling, - .row_stride_B = lvl_ext_B.width, - }; - } else { - /* Linear images need to be 2D */ - assert(image->dim == NIL_IMAGE_DIM_2D); - /* NVIDIA can't do linear and mipmapping */ - assert(image->num_levels == 1); - /* NVIDIA can't do linear and multisampling*/ - assert(image->sample_layout == NIL_SAMPLE_LAYOUT_1X1); - - image->levels[l] = (struct nil_image_level) { - .offset_B = layer_size_B, - .tiling = tiling, - /* Row stride needs to be aligned to 128B for render to work */ - .row_stride_B = align(lvl_ext_B.width, 128), - }; - } - layer_size_B += nil_image_level_size_B(image, l); - } - - /* We use the tiling for level 0 instead of the tiling selected above - * because, in the case of sparse residency with small images, level 0 may - * have a smaller tiling than what we tried to use. However, the level 0 - * tiling is the one we program in the hardware so that's the one we need - * to use for array stride calculations and the like. - */ - const uint32_t lvl0_tiling_size_B = - nil_tiling_size_B(image->levels[0].tiling); - - /* The array stride has to be aligned to the size of a level 0 tile */ - image->array_stride_B = align(layer_size_B, lvl0_tiling_size_B); - - image->size_B = (uint64_t)image->array_stride_B * image->extent_px.a; - image->align_B = lvl0_tiling_size_B; - - /* If the client requested sparse residency, we need a 64K alignment or - * else sparse binding may fail. This is true regardless of whether or - * not we actually select a 64K tile format. - */ - if (info->usage & NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT) - image->align_B = MAX2(image->align_B, (1 << 16)); - - if (image->levels[0].tiling.is_tiled) { - image->tile_mode = (uint16_t)image->levels[0].tiling.y_log2 << 4 | - (uint16_t)image->levels[0].tiling.z_log2 << 8; - - image->pte_kind = nil_choose_pte_kind(dev, info->format, info->samples, - false /* TODO: compressed */); - - image->align_B = MAX2(image->align_B, 4096); - if (image->pte_kind >= 0xb && image->pte_kind <= 0xe) - image->align_B = MAX2(image->align_B, (1 << 16)); - } else { - /* Linear images need to be aligned to 128B for render to work */ - image->align_B = MAX2(image->align_B, 128); - } - - image->size_B = align64(image->size_B, image->align_B); - return true; -} - -/** Offset of the given Z slice within the level */ -uint64_t -nil_image_level_z_offset_B(const struct nil_image *image, - uint32_t level, uint32_t z) -{ - assert(level < image->num_levels); - const struct nil_extent4d lvl_extent_px = - nil_image_level_extent_px(image, level); - assert(z < lvl_extent_px.d); - - const struct nil_tiling *lvl_tiling = &image->levels[level].tiling; - - const uint32_t z_tl = z >> lvl_tiling->z_log2; - const uint32_t z_GOB = z & BITFIELD_MASK(lvl_tiling->z_log2); - - const struct nil_extent4d lvl_extent_tl = - nil_extent4d_px_to_tl(lvl_extent_px, *lvl_tiling, - image->format, image->sample_layout); - uint64_t offset_B = lvl_extent_tl.w * lvl_extent_tl.h * (uint64_t)z_tl * - nil_tiling_size_B(*lvl_tiling); - - const struct nil_extent4d tiling_extent_B = - nil_tiling_extent_B(*lvl_tiling); - offset_B += tiling_extent_B.w * tiling_extent_B.h * z_GOB; - - return offset_B; -} - -uint64_t -nil_image_level_depth_stride_B(const struct nil_image *image, uint32_t level) -{ - assert(level < image->num_levels); - - /* See the nil_image::levels[] computations */ - struct nil_extent4d lvl_ext_B = image_level_extent_B(image, level); - struct nil_extent4d lvl_tiling_ext_B = - nil_tiling_extent_B(image->levels[level].tiling); - lvl_ext_B = nil_extent4d_align(lvl_ext_B, lvl_tiling_ext_B); - - return (uint64_t)lvl_ext_B.w * (uint64_t)lvl_ext_B.h; -} - -void -nil_image_for_level(const struct nil_image *image_in, - uint32_t level, - struct nil_image *lvl_image_out, - uint64_t *offset_B_out) -{ - assert(level < image_in->num_levels); - - const struct nil_extent4d lvl_extent_px = - nil_image_level_extent_px(image_in, level); - struct nil_image_level lvl = image_in->levels[level]; - const uint32_t align_B = nil_tiling_size_B(lvl.tiling); - - uint64_t size_B = image_in->size_B - lvl.offset_B; - if (level + 1 < image_in->num_levels) { - /* This assumes levels are sequential, tightly packed, and that each - * level has a higher alignment than the next one. All of this is - * currently true - */ - const uint64_t next_lvl_offset_B = image_in->levels[level + 1].offset_B; - assert(next_lvl_offset_B > lvl.offset_B); - size_B -= next_lvl_offset_B - lvl.offset_B; - } - - *offset_B_out = lvl.offset_B; - lvl.offset_B = 0; - - *lvl_image_out = (struct nil_image) { - .dim = image_in->dim, - .format = image_in->format, - .extent_px = lvl_extent_px, - .sample_layout = image_in->sample_layout, - .num_levels = 1, - .levels[0] = lvl, - .array_stride_B = image_in->array_stride_B, - .align_B = align_B, - .size_B = size_B, - .tile_mode = image_in->tile_mode, - .pte_kind = image_in->pte_kind, - .mip_tail_first_lod = level < image_in->mip_tail_first_lod ? 1 : 0, - }; -} - -static enum pipe_format -pipe_format_for_bits(uint32_t bits) -{ - switch (bits) { - case 32: return PIPE_FORMAT_R32_UINT; - case 64: return PIPE_FORMAT_R32G32_UINT; - case 128: return PIPE_FORMAT_R32G32B32A32_UINT; - default: - unreachable("No PIPE_FORMAT with this size"); - } -} - -void -nil_image_level_as_uncompressed(const struct nil_image *image_in, - uint32_t level, - struct nil_image *uc_image_out, - uint64_t *offset_B_out) -{ - assert(image_in->sample_layout == NIL_SAMPLE_LAYOUT_1X1); - - /* Format is arbitrary. Pick one that has the right number of bits. */ - const enum pipe_format uc_format = - pipe_format_for_bits(util_format_get_blocksizebits(image_in->format)); - - struct nil_image lvl_image; - nil_image_for_level(image_in, level, &lvl_image, offset_B_out); - - *uc_image_out = lvl_image; - uc_image_out->format = uc_format; - uc_image_out->extent_px = - nil_extent4d_px_to_el(lvl_image.extent_px, lvl_image.format, - lvl_image.sample_layout); -} - -void -nil_image_3d_level_as_2d_array(const struct nil_image *image_3d, - uint32_t level, - struct nil_image *image_2d_out, - uint64_t *offset_B_out) -{ - assert(image_3d->dim == NIL_IMAGE_DIM_3D); - assert(image_3d->extent_px.array_len == 1); - assert(image_3d->sample_layout == NIL_SAMPLE_LAYOUT_1X1); - - struct nil_image lvl_image; - nil_image_for_level(image_3d, level, &lvl_image, offset_B_out); - - assert(lvl_image.num_levels == 1); - assert(!lvl_image.levels[0].tiling.is_tiled || - lvl_image.levels[0].tiling.z_log2 == 0); - - struct nil_extent4d lvl_tiling_ext_B = - nil_tiling_extent_B(lvl_image.levels[0].tiling); - struct nil_extent4d lvl_ext_B = image_level_extent_B(&lvl_image, 0); - lvl_ext_B = nil_extent4d_align(lvl_ext_B, lvl_tiling_ext_B); - uint64_t z_stride = (uint64_t)lvl_ext_B.w * (uint64_t)lvl_ext_B.h; - - *image_2d_out = lvl_image; - image_2d_out->dim = NIL_IMAGE_DIM_2D; - image_2d_out->extent_px.d = 1; - image_2d_out->extent_px.a = lvl_image.extent_px.d; - image_2d_out->array_stride_B = z_stride; -} - -/** For a multisampled image, returns an image of samples - * - * The resulting image is supersampled with each pixel in the original - * consuming some number pixels in the supersampled images according to the - * original image's sample layout - */ -void -nil_msaa_image_as_sa(const struct nil_image *image_msaa, - struct nil_image *image_sa_out) -{ - assert(image_msaa->dim == NIL_IMAGE_DIM_2D); - assert(image_msaa->num_levels == 1); - - const struct nil_extent4d extent_sa = - nil_extent4d_px_to_sa(image_msaa->extent_px, - image_msaa->sample_layout); - - *image_sa_out = *image_msaa; - image_sa_out->extent_px = extent_sa; - image_sa_out->sample_layout = NIL_SAMPLE_LAYOUT_1X1; -} diff --git a/src/nouveau/nil/nil_image_tic.c b/src/nouveau/nil/nil_image_tic.c index 97a5e6f976c..af52dbd5f84 100644 --- a/src/nouveau/nil/nil_image_tic.c +++ b/src/nouveau/nil/nil_image_tic.c @@ -326,7 +326,7 @@ nv9097_nil_image_fill_tic(const struct nil_image *image, uint64_t base_address, void *desc_out) { - assert(util_format_get_blocksize(image->format) == + assert(util_format_get_blocksize(image->format.p_format) == util_format_get_blocksize(view->format)); assert(view->base_level + view->num_levels <= image->num_levels); assert(view->base_array_layer + view->array_len <= @@ -417,7 +417,7 @@ nvb097_nil_image_fill_tic(struct nv_device_info *dev, uint64_t base_address, void *desc_out) { - assert(util_format_get_blocksize(image->format) == + assert(util_format_get_blocksize(image->format.p_format) == util_format_get_blocksize(view->format)); assert(view->base_level + view->num_levels <= image->num_levels); diff --git a/src/nouveau/nil/tiling.rs b/src/nouveau/nil/tiling.rs new file mode 100644 index 00000000000..fbd6631e3c1 --- /dev/null +++ b/src/nouveau/nil/tiling.rs @@ -0,0 +1,192 @@ +// Copyright © 2024 Collabora, Ltd. +// SPDX-License-Identifier: MIT + +use crate::extent::{nil_extent4d, Extent4D}; +use crate::format::Format; +use crate::image::{ + ImageDim, ImageUsageFlags, SampleLayout, IMAGE_USAGE_2D_VIEW_BIT, + IMAGE_USAGE_LINEAR_BIT, +}; +use crate::ILog2Ceil; + +pub const GOB_WIDTH_B: u32 = 64; +pub const GOB_DEPTH: u32 = 1; + +pub fn gob_height(gob_height_is_8: bool) -> u32 { + if gob_height_is_8 { + 8 + } else { + 4 + } +} + +#[derive(Clone, Debug, Default, Copy, PartialEq)] +#[repr(C)] +pub struct Tiling { + pub is_tiled: bool, + /// Whether the GOB height is 4 or 8 + pub gob_height_is_8: bool, + /// log2 of the X tile dimension in GOBs + pub x_log2: u8, + /// log2 of the Y tile dimension in GOBs + pub y_log2: u8, + /// log2 of the z tile dimension in GOBs + pub z_log2: u8, +} + +impl Tiling { + /// Clamps the tiling to less than 2x the given extent in each dimension. + /// + /// This operation is done by the hardware at each LOD. + pub fn clamp(&self, extent_B: Extent4D) -> Self { + let mut tiling = *self; + + if !self.is_tiled { + return tiling; + } + + let tiling_extent_B = self.extent_B(); + + if extent_B.width < tiling_extent_B.width + || extent_B.height < tiling_extent_B.height + || extent_B.depth < tiling_extent_B.depth + { + tiling.x_log2 = 0; + } + + let extent_GOB = extent_B.B_to_GOB(tiling.gob_height_is_8); + + let ceil_h = extent_GOB.height.ilog2_ceil() as u8; + let ceil_d = extent_GOB.depth.ilog2_ceil() as u8; + + tiling.y_log2 = std::cmp::min(tiling.y_log2, ceil_h); + tiling.z_log2 = std::cmp::min(tiling.z_log2, ceil_d); + tiling + } + + pub fn size_B(&self) -> u32 { + let extent_B = self.extent_B(); + extent_B.width * extent_B.height * extent_B.depth * extent_B.array_len + } + + #[no_mangle] + pub extern "C" fn nil_tiling_size_B(&self) -> u32 { + self.size_B() + } + + pub fn extent_B(&self) -> Extent4D { + if self.is_tiled { + Extent4D { + width: GOB_WIDTH_B << self.x_log2, + height: gob_height(self.gob_height_is_8) << self.y_log2, + depth: GOB_DEPTH << self.z_log2, + array_len: 1, + } + } else { + // We handle linear images in Image::new() + Extent4D { + width: 1, + height: 1, + depth: 1, + array_len: 1, + } + } + } +} + +pub fn sparse_block_extent_el(format: Format, dim: ImageDim) -> Extent4D { + let bits = format.el_size_B() * 8; + + // Taken from Vulkan 1.3.279 spec section entitled "Standard Sparse + // Image Block Shapes". + match dim { + ImageDim::_2D => match bits { + 8 => nil_extent4d(256, 256, 1, 1), + 16 => nil_extent4d(256, 128, 1, 1), + 32 => nil_extent4d(128, 128, 1, 1), + 64 => nil_extent4d(128, 64, 1, 1), + 128 => nil_extent4d(64, 64, 1, 1), + other => panic!("Invalid texel size {other}"), + }, + ImageDim::_3D => match bits { + 8 => nil_extent4d(64, 32, 32, 1), + 16 => nil_extent4d(32, 32, 32, 1), + 32 => nil_extent4d(32, 32, 16, 1), + 64 => nil_extent4d(32, 16, 16, 1), + 128 => nil_extent4d(16, 16, 16, 1), + _ => panic!("Invalid texel size"), + }, + _ => panic!("Invalid sparse image dimension"), + } +} + +pub fn sparse_block_extent_px( + format: Format, + dim: ImageDim, + sample_layout: SampleLayout, +) -> Extent4D { + sparse_block_extent_el(format, dim) + .mul(format.el_extent_sa()) + .div_ceil(sample_layout.px_extent_sa()) +} + +pub fn sparse_block_extent_B(format: Format, dim: ImageDim) -> Extent4D { + sparse_block_extent_el(format, dim).el_to_B(format.el_size_B()) +} + +#[no_mangle] +pub extern "C" fn nil_sparse_block_extent_px( + format: Format, + dim: ImageDim, + sample_layout: SampleLayout, +) -> Extent4D { + sparse_block_extent_px(format, dim, sample_layout) +} + +impl Tiling { + pub fn sparse(format: Format, dim: ImageDim) -> Self { + let sparse_block_extent_B = sparse_block_extent_B(format, dim); + + assert!(sparse_block_extent_B.width.is_power_of_two()); + assert!(sparse_block_extent_B.height.is_power_of_two()); + assert!(sparse_block_extent_B.depth.is_power_of_two()); + + let gob_height_is_8 = true; + let sparse_block_extent_gob = + sparse_block_extent_B.B_to_GOB(gob_height_is_8); + + Self { + is_tiled: true, + gob_height_is_8, + x_log2: sparse_block_extent_gob.width.ilog2().try_into().unwrap(), + y_log2: sparse_block_extent_gob.height.ilog2().try_into().unwrap(), + z_log2: sparse_block_extent_gob.depth.ilog2().try_into().unwrap(), + } + } + + pub fn choose( + extent_px: Extent4D, + format: Format, + sample_layout: SampleLayout, + usage: ImageUsageFlags, + ) -> Tiling { + if (usage & IMAGE_USAGE_LINEAR_BIT) != 0 { + return Default::default(); + } + + let mut tiling = Tiling { + is_tiled: true, + gob_height_is_8: true, + x_log2: 0, + y_log2: 5, + z_log2: 5, + }; + + if (usage & IMAGE_USAGE_2D_VIEW_BIT) != 0 { + tiling.z_log2 = 0; + } + + let extent_B = extent_px.px_to_B(format, sample_layout); + tiling.clamp(extent_B) + } +} diff --git a/src/nouveau/vulkan/nvk_cmd_copy.c b/src/nouveau/vulkan/nvk_cmd_copy.c index dd940b09a69..820f1c7e64b 100644 --- a/src/nouveau/vulkan/nvk_cmd_copy.c +++ b/src/nouveau/vulkan/nvk_cmd_copy.c @@ -98,7 +98,7 @@ nouveau_copy_rect_image(struct nvk_image *img, plane->nil.sample_layout), .extent_el = nil_extent4d_px_to_el(lvl_extent4d_px, plane->nil.format, plane->nil.sample_layout), - .bpp = util_format_get_blocksize(plane->nil.format), + .bpp = util_format_get_blocksize(plane->nil.format.p_format), .row_stride = plane->nil.levels[sub_res->mipLevel].row_stride_B, .array_stride = plane->nil.array_stride_B, .tiling = plane->nil.levels[sub_res->mipLevel].tiling, diff --git a/src/nouveau/vulkan/nvk_cmd_draw.c b/src/nouveau/vulkan/nvk_cmd_draw.c index 89ce9d5f60e..a3b0c2e7bd0 100644 --- a/src/nouveau/vulkan/nvk_cmd_draw.c +++ b/src/nouveau/vulkan/nvk_cmd_draw.c @@ -800,8 +800,8 @@ nvk_CmdBeginRendering(VkCommandBuffer commandBuffer, if (nil_image.dim == NIL_IMAGE_DIM_3D) { uint64_t level_offset_B; - nil_image_3d_level_as_2d_array(&nil_image, mip_level, - &nil_image, &level_offset_B); + nil_image = nil_image_3d_level_as_2d_array(&nil_image, mip_level, + &level_offset_B); addr += level_offset_B; mip_level = 0; base_array_layer = 0; diff --git a/src/nouveau/vulkan/nvk_image.c b/src/nouveau/vulkan/nvk_image.c index bac496bc90b..133ed699b53 100644 --- a/src/nouveau/vulkan/nvk_image.c +++ b/src/nouveau/vulkan/nvk_image.c @@ -465,7 +465,7 @@ nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects, const enum nil_sample_layout sample_layout) { struct nil_extent4d sparse_block_extent_px = - nil_sparse_block_extent_px(format, dim, sample_layout); + nil_sparse_block_extent_px(nil_format(format), dim, sample_layout); assert(sparse_block_extent_px.array_len == 1); @@ -545,6 +545,8 @@ nvk_image_init(struct nvk_device *dev, struct nvk_image *image, const VkImageCreateInfo *pCreateInfo) { + struct nvk_physical_device *pdev = nvk_device_physical(dev); + vk_image_init(&dev->vk, &image->vk, pCreateInfo); if ((image->vk.usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | @@ -559,7 +561,7 @@ nvk_image_init(struct nvk_device *dev, if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT) image->vk.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - enum nil_image_usage_flags usage = 0; /* TODO */ + nil_image_usage_flags usage = 0; if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) usage |= NIL_IMAGE_USAGE_LINEAR_BIT; if (pCreateInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) @@ -596,7 +598,7 @@ nvk_image_init(struct nvk_device *dev, ycbcr_info->planes[plane].denominator_scales[1] : 1; struct nil_image_init_info nil_info = { .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType), - .format = vk_format_to_pipe_format(format), + .format = nil_format(vk_format_to_pipe_format(format)), .extent_px = { .width = pCreateInfo->extent.width / width_scale, .height = pCreateInfo->extent.height / height_scale, @@ -608,15 +610,13 @@ nvk_image_init(struct nvk_device *dev, .usage = usage, }; - ASSERTED bool ok = nil_image_init(&nvk_device_physical(dev)->info, - &image->planes[plane].nil, &nil_info); - assert(ok); + image->planes[plane].nil = nil_image_new(&pdev->info, &nil_info); } if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) { struct nil_image_init_info stencil_nil_info = { .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType), - .format = PIPE_FORMAT_R32_UINT, + .format = nil_format(PIPE_FORMAT_R32_UINT), .extent_px = { .width = pCreateInfo->extent.width, .height = pCreateInfo->extent.height, @@ -628,10 +628,8 @@ nvk_image_init(struct nvk_device *dev, .usage = usage, }; - ASSERTED bool ok = nil_image_init(&nvk_device_physical(dev)->info, - &image->stencil_copy_temp.nil, - &stencil_nil_info); - assert(ok); + image->stencil_copy_temp.nil = + nil_image_new(&pdev->info, &stencil_nil_info); } return VK_SUCCESS; @@ -872,7 +870,7 @@ nvk_fill_sparse_image_memory_reqs(const struct nil_image *nil, VkImageAspectFlags aspects) { VkSparseImageFormatProperties sparse_format_props = - nvk_fill_sparse_image_fmt_props(aspects, nil->format, + nvk_fill_sparse_image_fmt_props(aspects, nil->format.p_format, nil->dim, nil->sample_layout); assert(nil->mip_tail_first_lod <= nil->num_levels); diff --git a/src/nouveau/vulkan/nvk_image_view.c b/src/nouveau/vulkan/nvk_image_view.c index 219530b22c5..a470fe96b01 100644 --- a/src/nouveau/vulkan/nvk_image_view.c +++ b/src/nouveau/vulkan/nvk_image_view.c @@ -51,7 +51,7 @@ image_single_level_view(struct nil_image *image, assert(view->num_levels == 1); uint64_t offset_B; - nil_image_for_level(image, view->base_level, image, &offset_B); + *image = nil_image_for_level(image, view->base_level, &offset_B); *base_addr += offset_B; view->base_level = 0; } @@ -64,7 +64,7 @@ image_uncompressed_view(struct nil_image *image, assert(view->num_levels == 1); uint64_t offset_B; - nil_image_level_as_uncompressed(image, view->base_level, image, &offset_B); + *image = nil_image_level_as_uncompressed(image, view->base_level, &offset_B); *base_addr += offset_B; view->base_level = 0; } @@ -79,7 +79,7 @@ image_3d_view_as_2d_array(struct nil_image *image, assert(view->num_levels == 1); uint64_t offset_B; - nil_image_3d_level_as_2d_array(image, view->base_level, image, &offset_B); + *image = nil_image_3d_level_as_2d_array(image, view->base_level, &offset_B); *base_addr += offset_B; view->base_level = 0; } @@ -164,7 +164,7 @@ nvk_image_view_init(struct nvk_device *dev, .min_lod_clamp = view->vk.min_lod, }; - if (util_format_is_compressed(nil_image.format) && + if (util_format_is_compressed(nil_image.format.p_format) && !util_format_is_compressed(nil_view.format)) image_uncompressed_view(&nil_image, &nil_view, &base_addr); @@ -212,7 +212,7 @@ nvk_image_view_init(struct nvk_device *dev, } if (image->vk.samples != VK_SAMPLE_COUNT_1_BIT) - nil_msaa_image_as_sa(&nil_image, &nil_image); + nil_image = nil_msaa_image_as_sa(&nil_image); uint32_t tic[8]; nil_image_fill_tic(&nvk_device_physical(dev)->info, diff --git a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c index 208b2280e92..63dcb8c169d 100644 --- a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c +++ b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c @@ -161,7 +161,7 @@ push_add_image_plane_bind(struct push_builder *pb, const uint32_t level = bind->subresource.mipLevel; const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling; - const uint32_t tile_size_B = nil_tiling_size_B(plane_tiling); + const uint32_t tile_size_B = nil_tiling_size_B(&plane_tiling); const struct nil_extent4d bind_extent_px = { .width = bind->extent.width, @@ -179,17 +179,17 @@ push_add_image_plane_bind(struct push_builder *pb, const struct nil_extent4d level_extent_px = nil_image_level_extent_px(&plane->nil, level); const struct nil_extent4d level_extent_tl = - nil_extent4d_px_to_tl(level_extent_px, plane_tiling, + nil_extent4d_px_to_tl(level_extent_px, &plane_tiling, plane->nil.format, plane->nil.sample_layout); /* Convert the extent and offset to tiles */ struct nil_extent4d bind_extent_tl = - nil_extent4d_px_to_tl(bind_extent_px, plane_tiling, + nil_extent4d_px_to_tl(bind_extent_px, &plane_tiling, plane->nil.format, plane->nil.sample_layout); struct nil_offset4d bind_offset_tl = - nil_offset4d_px_to_tl(bind_offset_px, plane_tiling, + nil_offset4d_px_to_tl(bind_offset_px, &plane_tiling, plane->nil.format, plane->nil.sample_layout);