diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c index 35bf3e82e4c..2b0118a6cb6 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c @@ -180,7 +180,7 @@ lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base : offsetof(struct lp_texture_functions, sample_functions); LLVMValueRef texture_base_ptr = load_texture_functions_ptr( - gallivm, params->texture_resource, offsetof(struct lp_descriptor, functions), functions_offset); + gallivm, params->texture_resource, offsetof(struct lp_image_descriptor, functions), functions_offset); LLVMTypeRef texture_function_type = lp_build_sample_function_type(gallivm, params->sample_key); LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0); @@ -199,7 +199,7 @@ lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base } else { sampler_desc_ptr = params->sampler_resource; - LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.sampler_index)); + LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_sampler_descriptor, sampler_index)); LLVMValueRef sampler_index_ptr = LLVMBuildAdd(builder, sampler_desc_ptr, sampler_index_offset, ""); LLVMTypeRef sampler_index_type = LLVMInt32TypeInContext(gallivm->context); @@ -370,7 +370,7 @@ lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base, : offsetof(struct lp_texture_functions, size_function); LLVMValueRef texture_base_ptr = load_texture_functions_ptr( - gallivm, params->resource, offsetof(struct lp_descriptor, functions), functions_offset); + gallivm, params->resource, offsetof(struct lp_image_descriptor, functions), functions_offset); LLVMTypeRef texture_function_type = lp_build_size_function_type(gallivm, params); LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0); @@ -485,7 +485,7 @@ lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base, } LLVMValueRef image_base_ptr = load_texture_functions_ptr( - gallivm, params->resource, offsetof(struct lp_descriptor, functions), + gallivm, params->resource, offsetof(struct lp_image_descriptor, functions), offsetof(struct lp_texture_functions, image_functions)); LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, ms, is64); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c index fd9026f9b5a..6b4f6b42719 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c @@ -365,7 +365,7 @@ lp_build_llvm_texture_member(struct gallivm_state *gallivm, } } } else if (gallivm->texture_descriptor) { - static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset!"); + static_assert(offsetof(struct lp_image_descriptor, texture) == 0, "Invalid texture offset!"); LLVMValueRef texture_ptr = gallivm->texture_descriptor; LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES); @@ -432,7 +432,7 @@ lp_build_llvm_texture_residency(struct gallivm_state *gallivm, LLVMValueRef residency_ptr_ptr = gallivm->texture_descriptor; residency_ptr_ptr = LLVMBuildAdd(builder, residency_ptr_ptr, - lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.residency)), ""); + lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, texture.residency)), ""); LLVMTypeRef residency_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); residency_ptr_ptr = LLVMBuildIntToPtr(builder, residency_ptr_ptr, LLVMPointerType(residency_type, 0), ""); @@ -451,7 +451,7 @@ lp_build_llvm_texture_base_offset(struct gallivm_state *gallivm, LLVMValueRef base_offset_ptr = gallivm->texture_descriptor; base_offset_ptr = LLVMBuildAdd(builder, base_offset_ptr, - lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.base_offset)), ""); + lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, texture.base_offset)), ""); LLVMTypeRef base_offset_type = LLVMInt32TypeInContext(gallivm->context); base_offset_ptr = LLVMBuildIntToPtr(builder, base_offset_ptr, LLVMPointerType(base_offset_type, 0), ""); @@ -527,7 +527,7 @@ lp_build_llvm_sampler_member(struct gallivm_state *gallivm, LLVMValueRef ptr; if (gallivm->sampler_descriptor) { - LLVMValueRef sampler_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, sampler)); + LLVMValueRef sampler_offset = lp_build_const_int64(gallivm, offsetof(struct lp_sampler_descriptor, jit)); LLVMValueRef sampler_ptr = LLVMBuildAdd(builder, gallivm->sampler_descriptor, sampler_offset, ""); LLVMTypeRef sampler_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS); @@ -611,7 +611,7 @@ lp_build_llvm_image_member(struct gallivm_state *gallivm, LLVMValueRef ptr; if (gallivm->texture_descriptor) { - LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, image)); + LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, image)); LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, ""); LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.h b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.h index 09c74fa80ca..4456fb1992e 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.h @@ -237,27 +237,30 @@ struct lp_jit_bindless_texture { const void *base; const void *residency; - uint32_t sampler_index; uint32_t base_offset; }; -struct lp_descriptor { +struct lp_image_descriptor { union { - struct { - struct lp_jit_bindless_texture texture; - struct lp_jit_sampler sampler; - }; - struct { - struct lp_jit_image image; - }; - struct lp_jit_buffer buffer; - uint64_t accel_struct; + struct lp_jit_bindless_texture texture; + struct lp_jit_image image; }; /* Store sample/image functions in the same location since some d3d12 games * rely on mismatched descriptor types with null descriptors. */ void *functions; + + uint32_t padding[2]; +}; + +struct lp_sampler_descriptor { + struct lp_jit_sampler jit; + uint32_t sampler_index; +}; + +struct lp_buffer_descriptor { + struct lp_jit_buffer jit; }; #define LP_MAX_TEX_FUNC_ARGS 32 diff --git a/src/gallium/drivers/llvmpipe/lp_texture_handle.c b/src/gallium/drivers/llvmpipe/lp_texture_handle.c index 103635f4d54..91ce7eb2efd 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture_handle.c +++ b/src/gallium/drivers/llvmpipe/lp_texture_handle.c @@ -730,8 +730,8 @@ static uint64_t get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key) { struct lp_sampler_matrix *matrix = (void *)(uintptr_t)_matrix; - struct lp_descriptor *sampler_desc = (void *)(uintptr_t)_sampler_desc; - uint32_t sampler_index = sampler_desc->texture.sampler_index; + struct lp_sampler_descriptor *sampler_desc = (void *)(uintptr_t)_sampler_desc; + uint32_t sampler_index = sampler_desc->sampler_index; struct lp_texture_functions *texture_functions = (void *)(uintptr_t)_texture_functions; struct sample_function_cache_key key = { @@ -910,7 +910,7 @@ compile_jit_sample_function(struct llvmpipe_context *ctx, uint32_t sample_key) LLVMPositionBuilderAtEnd(gallivm->builder, block); LLVMValueRef functions_offset = - lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions)); + lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, functions)); LLVMValueRef functions_ptr = LLVMBuildAdd(builder, texture_descriptor, functions_offset, ""); @@ -1029,7 +1029,7 @@ compile_jit_fetch_function(struct llvmpipe_context *ctx, uint32_t sample_key) LLVMPositionBuilderAtEnd(gallivm->builder, block); LLVMValueRef functions_offset = - lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions)); + lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, functions)); LLVMValueRef functions_ptr = LLVMBuildAdd(builder, texture_descriptor, functions_offset, ""); @@ -1153,7 +1153,7 @@ compile_jit_size_function(struct llvmpipe_context *ctx, bool samples) LLVMPositionBuilderAtEnd(gallivm->builder, block); LLVMValueRef functions_offset = - lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions)); + lp_build_const_int64(gallivm, offsetof(struct lp_image_descriptor, functions)); LLVMValueRef functions_ptr = LLVMBuildAdd(builder, texture_descriptor, functions_offset, ""); diff --git a/src/gallium/frontends/lavapipe/lvp_descriptor_set.c b/src/gallium/frontends/lavapipe/lvp_descriptor_set.c index 82d07641a7e..180e7e4d827 100644 --- a/src/gallium/frontends/lavapipe/lvp_descriptor_set.c +++ b/src/gallium/frontends/lavapipe/lvp_descriptor_set.c @@ -55,6 +55,49 @@ lvp_descriptor_set_layout_destroy(struct vk_device *_device, struct vk_descripto vk_descriptor_set_layout_destroy(_device, _layout); } +uint32_t +lvp_get_descriptor_size(VkDescriptorType type) +{ + switch (type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: + return sizeof(struct lp_buffer_descriptor); + case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: + return 0; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + return sizeof(struct lp_image_descriptor); + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + return sizeof(struct lvp_image_sampler_descriptor); + case VK_DESCRIPTOR_TYPE_SAMPLER: + return sizeof(struct lp_sampler_descriptor); + case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: + return sizeof(uint64_t); + case VK_DESCRIPTOR_TYPE_MUTABLE_EXT: + return sizeof(struct lvp_image_sampler_descriptor); + default: + break; + } + + return 0; +} + + +uint32_t +lvp_get_sampler_descriptor_offset(VkDescriptorType type) +{ + if (type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) + return offsetof(struct lvp_image_sampler_descriptor, sampler); + + return 0; +} + + VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( VkDevice _device, const VkDescriptorSetLayoutCreateInfo* pCreateInfo, @@ -64,6 +107,9 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( VK_FROM_HANDLE(lvp_device, device, _device); struct lvp_descriptor_set_layout *set_layout; + const VkMutableDescriptorTypeCreateInfoEXT *mutable_info = + vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT); + assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO); uint32_t num_bindings = 0; uint32_t immutable_sampler_count = 0; @@ -86,7 +132,7 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( size_t size = sizeof(struct lvp_descriptor_set_layout) + num_bindings * sizeof(set_layout->binding[0]) + - immutable_sampler_count * sizeof(struct lp_descriptor) + + immutable_sampler_count * sizeof(struct lp_sampler_descriptor) + immutable_sampler_count * sizeof(struct vk_ycbcr_conversion_state); set_layout = vk_descriptor_set_layout_zalloc(&device->vk, size, pCreateInfo); @@ -95,8 +141,8 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( set_layout->immutable_sampler_count = immutable_sampler_count; /* We just allocate all the samplers at the end of the struct */ - struct lp_descriptor *samplers = - (struct lp_descriptor *)&set_layout->binding[num_bindings]; + struct lp_sampler_descriptor *samplers = + (struct lp_sampler_descriptor *)&set_layout->binding[num_bindings]; struct vk_ycbcr_conversion_state *ycbcr = (void*)(samplers + immutable_sampler_count); set_layout->binding_count = num_bindings; @@ -124,7 +170,7 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( descriptor_count = 1; set_layout->binding[b].array_size = descriptor_count; - set_layout->binding[b].descriptor_index = set_layout->size; + set_layout->binding[b].offset = set_layout->size; set_layout->binding[b].type = binding->descriptorType; set_layout->binding[b].valid = true; set_layout->binding[b].uniform_block_offset = 0; @@ -156,39 +202,30 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateDescriptorSetLayout( } } - set_layout->binding[b].stride = max_plane_count; - set_layout->size += descriptor_count * max_plane_count; + set_layout->binding[b].max_plane_count = max_plane_count; - switch (binding->descriptorType) { - case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: - case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: - break; - case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: + uint32_t descriptor_size = lvp_get_descriptor_size(binding->descriptorType); + if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_EXT && mutable_info) { + descriptor_size = 0; + const VkMutableDescriptorTypeListEXT *type_list = &mutable_info->pMutableDescriptorTypeLists[j]; + for (uint32_t k = 0; k < type_list->descriptorTypeCount; k++) + descriptor_size = MAX2(descriptor_size, lvp_get_descriptor_size(type_list->pDescriptorTypes[k])); + } + + set_layout->binding[b].stride = max_plane_count * descriptor_size; + set_layout->size += descriptor_count * max_plane_count * descriptor_size; + + if (binding->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) { set_layout->binding[b].uniform_block_offset = uniform_block_size; set_layout->binding[b].uniform_block_size = binding->descriptorCount; uniform_block_size += binding->descriptorCount; - break; - case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: - case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: - break; - - case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: - case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: - case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: - break; - case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: - case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: - case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: - break; - default: - break; } set_layout->shader_stages |= binding->stageFlags & MESA_VK_SHADER_STAGE_ALL; } for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) - set_layout->binding[i].uniform_block_offset += set_layout->size * sizeof(struct lp_descriptor); + set_layout->binding[i].uniform_block_offset += set_layout->size; free(bindings); @@ -338,7 +375,7 @@ lvp_descriptor_set_create(struct lvp_device *device, set->layout = layout; vk_descriptor_set_layout_ref(&layout->vk); - uint64_t bo_size = layout->size * sizeof(struct lp_descriptor); + uint64_t bo_size = layout->size; for (unsigned i = 0; i < layout->binding_count; i++) bo_size += layout->binding[i].uniform_block_size; @@ -370,13 +407,12 @@ lvp_descriptor_set_create(struct lvp_device *device, if (!bind_layout->immutable_samplers) continue; - struct lp_descriptor *desc = set->map; - desc += bind_layout->descriptor_index; - + uint8_t *desc = set->map + bind_layout->offset; for (uint32_t sampler_index = 0; sampler_index < bind_layout->array_size; sampler_index++) { - for (uint32_t s = 0; s < bind_layout->stride; s++) { - int idx = sampler_index * bind_layout->stride + s; - desc[idx] = bind_layout->immutable_samplers[sampler_index]; + for (uint32_t s = 0; s < bind_layout->max_plane_count; s++) { + memcpy(desc + sampler_index * bind_layout->stride + s * lvp_get_descriptor_size(bind_layout->type) + + lvp_get_sampler_descriptor_offset(bind_layout->type), + &bind_layout->immutable_samplers[sampler_index], sizeof(struct lp_sampler_descriptor)); } } } @@ -470,20 +506,16 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( continue; } - struct lp_descriptor *desc = set->map; - desc += bind_layout->descriptor_index + (write->dstArrayElement * bind_layout->stride); + uint8_t *descs = set->map + bind_layout->offset + (write->dstArrayElement * bind_layout->stride); switch (write->descriptorType) { case VK_DESCRIPTOR_TYPE_SAMPLER: if (!bind_layout->immutable_samplers) { for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_sampler, sampler, write->pImageInfo[j].sampler); - uint32_t didx = j * bind_layout->stride; - - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[didx + k].sampler = sampler->desc.sampler; - desc[didx + k].texture.sampler_index = sampler->desc.texture.sampler_index; - } + struct lp_sampler_descriptor *desc = (struct lp_sampler_descriptor *)(descs + j * bind_layout->stride); + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + desc[k] = sampler->desc; } } break; @@ -492,13 +524,13 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_image_view, iview, write->pImageInfo[j].imageView); - uint32_t didx = j * bind_layout->stride; + struct lvp_image_sampler_descriptor *desc = (struct lvp_image_sampler_descriptor *)(descs + j * bind_layout->stride); if (iview) { unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[didx + p].texture, iview->planes[p].sv); - desc[didx + p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&desc[p].image.texture, iview->planes[p].sv); + desc[p].image.functions = iview->planes[p].texture_handle->functions; } if (!bind_layout->immutable_samplers) { @@ -506,14 +538,13 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( write->pImageInfo[j].sampler); for (unsigned p = 0; p < plane_count; p++) { - desc[didx + p].sampler = sampler->desc.sampler; - desc[didx + p].texture.sampler_index = sampler->desc.texture.sampler_index; + desc[p].sampler = sampler->desc; } } } else { - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[didx + k].functions = device->null_texture_handle->functions; - desc[didx + k].texture.sampler_index = 0; + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) { + desc[k].image.functions = device->null_texture_handle->functions; + desc[k].sampler.sampler_index = 0; } } } @@ -523,19 +554,17 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_image_view, iview, write->pImageInfo[j].imageView); - uint32_t didx = j * bind_layout->stride; + struct lp_image_descriptor *desc = (struct lp_image_descriptor *)(descs + j * bind_layout->stride); if (iview) { unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[didx + p].texture, iview->planes[p].sv); - desc[didx + p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&desc[p].texture, iview->planes[p].sv); + desc[p].functions = iview->planes[p].texture_handle->functions; } } else { - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[didx + k].functions = device->null_texture_handle->functions; - desc[didx + k].texture.sampler_index = 0; - } + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + desc[k].functions = device->null_texture_handle->functions; } } break; @@ -544,57 +573,59 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_image_view, iview, write->pImageInfo[j].imageView); - uint32_t didx = j * bind_layout->stride; + struct lp_image_descriptor *desc = (struct lp_image_descriptor *)(descs + j * bind_layout->stride); if (iview) { unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_image_from_pipe(&desc[didx + p].image, &iview->planes[p].iv); - desc[didx + p].functions = iview->planes[p].image_handle->functions; + lp_jit_image_from_pipe(&desc[p].image, &iview->planes[p].iv); + desc[p].functions = iview->planes[p].image_handle->functions; } } else { - memset(&desc[didx], 0, sizeof(desc[didx]) * bind_layout->stride); - for (unsigned k = 0; k < bind_layout->stride; k++) - desc[didx + k].functions = device->null_image_handle->functions; + memset(desc, 0, bind_layout->stride); + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + desc[k].functions = device->null_image_handle->functions; } } break; - case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: { for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_buffer_view, bview, write->pTexelBufferView[j]); - assert(bind_layout->stride == 1); + struct lp_image_descriptor *desc = + (struct lp_image_descriptor *)(descs + j * bind_layout->stride); if (bview) { - lp_jit_bindless_texture_from_pipe(&desc[j].texture, bview->sv); - desc[j].functions = bview->texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&desc->texture, bview->sv); + desc->functions = bview->texture_handle->functions; } else { - desc[j].functions = device->null_texture_handle->functions; - desc[j].texture.sampler_index = 0; + desc->functions = device->null_texture_handle->functions; } } break; - - case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + } + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: { for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_buffer_view, bview, - write->pTexelBufferView[j]); - assert(bind_layout->stride == 1); + write->pTexelBufferView[j]); + struct lp_image_descriptor *desc = + (struct lp_image_descriptor *)(descs + j * bind_layout->stride); if (bview) { - lp_jit_image_from_pipe(&desc[j].image, &bview->iv); - desc[j].functions = bview->image_handle->functions; + lp_jit_image_from_pipe(&desc->image, &bview->iv); + desc->functions = bview->image_handle->functions; } else { - memset(&desc[j].image, 0, sizeof(desc[j].image)); - desc[j].functions = device->null_image_handle->functions; + memset(&desc->image, 0, sizeof(desc->image)); + desc->functions = device->null_image_handle->functions; } } break; - + } case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: - case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: { for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_buffer, buffer, write->pBufferInfo[j].buffer); - assert(bind_layout->stride == 1); + struct lp_buffer_descriptor *desc = + (struct lp_buffer_descriptor *)(descs + j * bind_layout->stride); if (buffer) { struct pipe_constant_buffer ubo = { .buffer = buffer->bo, @@ -605,18 +636,19 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( if (write->pBufferInfo[j].range == VK_WHOLE_SIZE) ubo.buffer_size = buffer->bo->width0 - ubo.buffer_offset; - lp_jit_buffer_from_pipe_const(&desc[j].buffer, &ubo, device->pscreen); + lp_jit_buffer_from_pipe_const(&desc->jit, &ubo, device->pscreen); } else { - lp_jit_buffer_from_pipe_const(&desc[j].buffer, &((struct pipe_constant_buffer){0}), device->pscreen); + lp_jit_buffer_from_pipe_const(&desc->jit, &((struct pipe_constant_buffer){0}), device->pscreen); } } break; - + } case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: - case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { for (uint32_t j = 0; j < write->descriptorCount; j++) { VK_FROM_HANDLE(lvp_buffer, buffer, write->pBufferInfo[j].buffer); - assert(bind_layout->stride == 1); + struct lp_buffer_descriptor *desc = + (struct lp_buffer_descriptor *)(descs + j * bind_layout->stride); if (buffer) { struct pipe_shader_buffer ubo = { .buffer = buffer->bo, @@ -627,22 +659,23 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( if (write->pBufferInfo[j].range == VK_WHOLE_SIZE) ubo.buffer_size = buffer->bo->width0 - ubo.buffer_offset; - lp_jit_buffer_from_pipe(&desc[j].buffer, &ubo); + lp_jit_buffer_from_pipe(&desc->jit, &ubo); } else { - lp_jit_buffer_from_pipe(&desc[j].buffer, &((struct pipe_shader_buffer){0})); + lp_jit_buffer_from_pipe(&desc->jit, &((struct pipe_shader_buffer){0})); } } break; - - case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: + } + case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: { for (uint32_t j = 0; j < write->descriptorCount; j++) { const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs = vk_find_struct_const(write->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR); VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, accel_structs->pAccelerationStructures[j]); - - desc[j].accel_struct = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0; + uint64_t *desc = (uint64_t *)(descs + j * bind_layout->stride); + *desc = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0; } break; + } default: UNREACHABLE("Unsupported descriptor type"); @@ -657,24 +690,21 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets( const struct lvp_descriptor_set_binding_layout *src_layout = &src->layout->binding[copy->srcBinding]; - struct lp_descriptor *src_desc = src->map; - src_desc += src_layout->descriptor_index; - const struct lvp_descriptor_set_binding_layout *dst_layout = &dst->layout->binding[copy->dstBinding]; - struct lp_descriptor *dst_desc = dst->map; - dst_desc += dst_layout->descriptor_index; if (src_layout->type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) { memcpy((uint8_t *)dst->map + dst_layout->uniform_block_offset + copy->dstArrayElement, (uint8_t *)src->map + src_layout->uniform_block_offset + copy->srcArrayElement, copy->descriptorCount); } else { - src_desc += copy->srcArrayElement; - dst_desc += copy->dstArrayElement; - - for (uint32_t j = 0; j < copy->descriptorCount; j++) - dst_desc[j] = src_desc[j]; + /* The sizes can be different if mutable descriptors are used. */ + uint32_t desc_size = MIN2(src_layout->stride, dst_layout->stride); + for (uint32_t j = 0; j < copy->descriptorCount; j++) { + uint8_t *src_desc = src->map + src_layout->offset + (copy->srcArrayElement + j) * src_layout->stride; + uint8_t *dst_desc = dst->map + dst_layout->offset + (copy->dstArrayElement + j) * dst_layout->stride; + memcpy(dst_desc, src_desc, desc_size); + } } } } @@ -747,6 +777,9 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorSetLayoutSupport(VkDevice device, vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO); VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count = vk_find_struct(pSupport->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT); + const VkMutableDescriptorTypeCreateInfoEXT *mutable_info = + vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT); + if (variable_count) { variable_count->maxVariableDescriptorCount = 0; if (variable_flags) { @@ -756,7 +789,23 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorSetLayoutSupport(VkDevice device, } } } + pSupport->supported = true; + if (mutable_info) { + for (uint32_t i = 0; i < mutable_info->mutableDescriptorTypeListCount; i++) { + const VkMutableDescriptorTypeListEXT *type_list = &mutable_info->pMutableDescriptorTypeLists[i]; + for (uint32_t j = 0; j < type_list->descriptorTypeCount; j++) { + /* Combined image samplers use a different offset for sampler descriptors which + * lvp_nir_lower_pipeline_layout would has to account for. For mutable descriptors, + * the pass cannot know the descriptor type and apply the offset. + */ + if (type_list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) { + pSupport->supported = false; + break; + } + } + } + } } void @@ -782,46 +831,41 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri continue; } - struct lp_descriptor *desc = set->map; - desc += bind_layout->descriptor_index; - for (j = 0; j < entry->array_count; ++j) { - unsigned idx = j + entry->array_element; + uint8_t *descs = set->map + bind_layout->offset + (entry->array_element + j) * bind_layout->stride; - idx *= bind_layout->stride; switch (entry->type) { case VK_DESCRIPTOR_TYPE_SAMPLER: { VkDescriptorImageInfo *info = (VkDescriptorImageInfo *)pSrc; VK_FROM_HANDLE(lvp_sampler, sampler, info->sampler); - - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[idx + k].sampler = sampler->desc.sampler; - desc[idx + k].texture.sampler_index = sampler->desc.texture.sampler_index; - } + struct lp_sampler_descriptor *sampler_desc = (struct lp_sampler_descriptor *)descs; + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + sampler_desc[k] = sampler->desc; break; } case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: { VkDescriptorImageInfo *info = (VkDescriptorImageInfo *)pSrc; VK_FROM_HANDLE(lvp_image_view, iview, info->imageView); + struct lvp_image_sampler_descriptor *desc = (struct lvp_image_sampler_descriptor *)descs; + if (iview) { for (unsigned p = 0; p < iview->plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[idx + p].texture, iview->planes[p].sv); - desc[idx + p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&desc[p].image.texture, iview->planes[p].sv); + desc[p].image.functions = iview->planes[p].texture_handle->functions; } if (!bind_layout->immutable_samplers) { VK_FROM_HANDLE(lvp_sampler, sampler, info->sampler); for (unsigned p = 0; p < iview->plane_count; p++) { - desc[idx + p].sampler = sampler->desc.sampler; - desc[idx + p].texture.sampler_index = sampler->desc.texture.sampler_index; + desc[p].sampler = sampler->desc; } } } else { - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[idx + k].functions = device->null_texture_handle->functions; - desc[idx + k].texture.sampler_index = 0; + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) { + desc[k].image.functions = device->null_texture_handle->functions; + desc[k].sampler.sampler_index = 0; } } break; @@ -830,16 +874,16 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri VkDescriptorImageInfo *info = (VkDescriptorImageInfo *)pSrc; VK_FROM_HANDLE(lvp_image_view, iview, info->imageView); + struct lp_image_descriptor *image_desc = (struct lp_image_descriptor *)descs; + if (iview) { for (unsigned p = 0; p < iview->plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[idx + p].texture, iview->planes[p].sv); - desc[idx + p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&image_desc[p].texture, iview->planes[p].sv); + image_desc[p].functions = iview->planes[p].texture_handle->functions; } } else { - for (unsigned k = 0; k < bind_layout->stride; k++) { - desc[idx + k].functions = device->null_texture_handle->functions; - desc[idx + k].texture.sampler_index = 0; - } + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + image_desc[k].functions = device->null_texture_handle->functions; } break; } @@ -848,41 +892,42 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri VK_FROM_HANDLE(lvp_image_view, iview, ((VkDescriptorImageInfo *)pSrc)->imageView); + struct lp_image_descriptor *image_desc = (struct lp_image_descriptor *)descs; + if (iview) { for (unsigned p = 0; p < iview->plane_count; p++) { - lp_jit_image_from_pipe(&desc[idx + p].image, &iview->planes[p].iv); - desc[idx + p].functions = iview->planes[p].image_handle->functions; + lp_jit_image_from_pipe(&image_desc[p].image, &iview->planes[p].iv); + image_desc[p].functions = iview->planes[p].image_handle->functions; } } else { - memset(&desc[idx], 0, sizeof(desc[idx]) * bind_layout->stride); - for (unsigned k = 0; k < bind_layout->stride; k++) - desc[idx + k].functions = device->null_image_handle->functions; + memset(image_desc, 0, bind_layout->stride); + for (unsigned k = 0; k < bind_layout->max_plane_count; k++) + image_desc[k].functions = device->null_image_handle->functions; } break; } case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: { VK_FROM_HANDLE(lvp_buffer_view, bview, *(VkBufferView *)pSrc); - assert(bind_layout->stride == 1); + struct lp_image_descriptor *image_desc = (struct lp_image_descriptor *)descs; if (bview) { - lp_jit_bindless_texture_from_pipe(&desc[idx].texture, bview->sv); - desc[idx].functions = bview->texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&image_desc->texture, bview->sv); + image_desc->functions = bview->texture_handle->functions; } else { - desc[j].functions = device->null_texture_handle->functions; - desc[j].texture.sampler_index = 0; + image_desc->functions = device->null_texture_handle->functions; } break; } case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: { VK_FROM_HANDLE(lvp_buffer_view, bview, *(VkBufferView *)pSrc); - assert(bind_layout->stride == 1); + struct lp_image_descriptor *image_desc = (struct lp_image_descriptor *)descs; if (bview) { - lp_jit_image_from_pipe(&desc[idx].image, &bview->iv); - desc[idx].functions = bview->image_handle->functions; + lp_jit_image_from_pipe(&image_desc->image, &bview->iv); + image_desc->functions = bview->image_handle->functions; } else { - memset(&desc[idx].image, 0, sizeof(desc[idx].image)); - desc[idx].functions = device->null_image_handle->functions; + memset(&image_desc->image, 0, sizeof(image_desc->image)); + image_desc->functions = device->null_image_handle->functions; } break; } @@ -891,7 +936,7 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: { VkDescriptorBufferInfo *info = (VkDescriptorBufferInfo *)pSrc; VK_FROM_HANDLE(lvp_buffer, buffer, info->buffer); - assert(bind_layout->stride == 1); + struct lp_buffer_descriptor *buffer_desc = (struct lp_buffer_descriptor *)descs; if (buffer) { struct pipe_constant_buffer ubo = { .buffer = buffer->bo, @@ -902,9 +947,9 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri if (info->range == VK_WHOLE_SIZE) ubo.buffer_size = buffer->bo->width0 - ubo.buffer_offset; - lp_jit_buffer_from_pipe_const(&desc[idx].buffer, &ubo, device->pscreen); + lp_jit_buffer_from_pipe_const(&buffer_desc->jit, &ubo, device->pscreen); } else { - lp_jit_buffer_from_pipe_const(&desc[idx].buffer, &((struct pipe_constant_buffer){0}), device->pscreen); + lp_jit_buffer_from_pipe_const(&buffer_desc->jit, &((struct pipe_constant_buffer){0}), device->pscreen); } break; } @@ -913,8 +958,7 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { VkDescriptorBufferInfo *info = (VkDescriptorBufferInfo *)pSrc; VK_FROM_HANDLE(lvp_buffer, buffer, info->buffer); - assert(bind_layout->stride == 1); - + struct lp_buffer_descriptor *buffer_desc = (struct lp_buffer_descriptor *)descs; if (buffer) { struct pipe_shader_buffer ubo = { .buffer = buffer->bo, @@ -925,16 +969,17 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri if (info->range == VK_WHOLE_SIZE) ubo.buffer_size = buffer->bo->width0 - ubo.buffer_offset; - lp_jit_buffer_from_pipe(&desc[idx].buffer, &ubo); + lp_jit_buffer_from_pipe(&buffer_desc->jit, &ubo); } else { - lp_jit_buffer_from_pipe(&desc[idx].buffer, &((struct pipe_shader_buffer){0})); + lp_jit_buffer_from_pipe(&buffer_desc->jit, &((struct pipe_shader_buffer){0})); } break; } case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: { VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, *(VkAccelerationStructureKHR *)pSrc); - desc[idx].accel_struct = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0; + uint64_t *accel_struct_desc = (uint64_t *)descs; + *accel_struct_desc = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0; break; } @@ -963,7 +1008,7 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorSetLayoutSizeEXT( { VK_FROM_HANDLE(lvp_descriptor_set_layout, layout, _layout); - *pSize = layout->size * sizeof(struct lp_descriptor); + *pSize = layout->size; for (unsigned i = 0; i < layout->binding_count; i++) *pSize += layout->binding[i].uniform_block_size; @@ -982,7 +1027,7 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorSetLayoutBindingOffsetEXT( if (bind_layout->type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) *pOffset = bind_layout->uniform_block_offset; else - *pOffset = bind_layout->descriptor_index * sizeof(struct lp_descriptor); + *pOffset = bind_layout->offset; } VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( @@ -993,7 +1038,11 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( { VK_FROM_HANDLE(lvp_device, device, _device); - struct lp_descriptor *desc = pDescriptor; + struct lp_image_descriptor *image_desc = pDescriptor; + struct lvp_image_sampler_descriptor *image_sampler_desc = pDescriptor; + struct lp_sampler_descriptor *sampler_desc = pDescriptor; + struct lp_buffer_descriptor *buffer_desc = pDescriptor; + uint64_t *accel_struct_desc = pDescriptor; struct pipe_sampler_state sampler = { .seamless_cube_map = 1, @@ -1008,11 +1057,10 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( case VK_DESCRIPTOR_TYPE_SAMPLER: { if (pCreateInfo->data.pSampler) { VK_FROM_HANDLE(lvp_sampler, sampler, pCreateInfo->data.pSampler[0]); - desc->sampler = sampler->desc.sampler; - desc->texture.sampler_index = sampler->desc.texture.sampler_index; + *sampler_desc = sampler->desc; } else { - lp_jit_sampler_from_pipe(&desc->sampler, &sampler); - desc->texture.sampler_index = 0; + lp_jit_sampler_from_pipe(&sampler_desc->jit, &sampler); + sampler_desc->sampler_index = 0; } break; } @@ -1025,24 +1073,23 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[p].texture, iview->planes[p].sv); - desc[p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&image_sampler_desc[p].image.texture, iview->planes[p].sv); + image_sampler_desc[p].image.functions = iview->planes[p].texture_handle->functions; if (info->sampler) { VK_FROM_HANDLE(lvp_sampler, sampler, info->sampler); - desc[p].sampler = sampler->desc.sampler; - desc[p].texture.sampler_index = sampler->desc.texture.sampler_index; + image_sampler_desc[p].sampler = sampler->desc; } else { - lp_jit_sampler_from_pipe(&desc->sampler, &sampler); - desc[p].texture.sampler_index = 0; + lp_jit_sampler_from_pipe(&image_sampler_desc[p].sampler.jit, &sampler); + image_sampler_desc[p].sampler.sampler_index = 0; } } } else { - unsigned plane_count = size / sizeof(struct lp_descriptor); + unsigned plane_count = size / sizeof(struct lvp_image_sampler_descriptor); for (unsigned p = 0; p < plane_count; p++) { - desc[p].functions = device->null_texture_handle->functions; - desc[p].texture.sampler_index = 0; + image_sampler_desc[p].image.functions = device->null_texture_handle->functions; + image_sampler_desc[p].sampler.sampler_index = 0; } } @@ -1056,16 +1103,13 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_bindless_texture_from_pipe(&desc[p].texture, iview->planes[p].sv); - desc[p].functions = iview->planes[p].texture_handle->functions; + lp_jit_bindless_texture_from_pipe(&image_desc[p].texture, iview->planes[p].sv); + image_desc[p].functions = iview->planes[p].texture_handle->functions; } } else { - unsigned plane_count = size / sizeof(struct lp_descriptor); - - for (unsigned p = 0; p < plane_count; p++) { - desc[p].functions = device->null_texture_handle->functions; - desc[p].texture.sampler_index = 0; - } + unsigned plane_count = size / sizeof(struct lp_image_descriptor); + for (unsigned p = 0; p < plane_count; p++) + image_desc[p].functions = device->null_texture_handle->functions; } break; } @@ -1079,15 +1123,15 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( unsigned plane_count = iview->plane_count; for (unsigned p = 0; p < plane_count; p++) { - lp_jit_image_from_pipe(&desc[p].image, &iview->planes[p].iv); - desc[p].functions = iview->planes[p].image_handle->functions; + lp_jit_image_from_pipe(&image_desc[p].image, &iview->planes[p].iv); + image_desc[p].functions = iview->planes[p].image_handle->functions; } } else { - memset(desc, 0, size); + memset(image_desc, 0, size); - unsigned plane_count = size / sizeof(struct lp_descriptor); + unsigned plane_count = size / sizeof(struct lp_image_descriptor); for (unsigned p = 0; p < plane_count; p++) - desc[p].functions = device->null_image_handle->functions; + image_desc[p].functions = device->null_image_handle->functions; } break; } @@ -1095,12 +1139,11 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( const VkDescriptorAddressInfoEXT *bda = pCreateInfo->data.pUniformTexelBuffer; if (bda && bda->address) { enum pipe_format pformat = vk_format_to_pipe_format(bda->format); - lp_jit_bindless_texture_buffer_from_bda(&desc->texture, (void*)(uintptr_t)bda->address); - desc->functions = get_texture_handle_bda(device, bda->address, bda->range, pformat).functions; + lp_jit_bindless_texture_buffer_from_bda(&image_desc->texture, (void*)(uintptr_t)bda->address); + image_desc->functions = get_texture_handle_bda(device, bda->address, bda->range, pformat).functions; } else { - memset(desc, 0, size); - desc->functions = device->null_texture_handle->functions; - desc->texture.sampler_index = 0; + memset(image_desc, 0, size); + image_desc->functions = device->null_texture_handle->functions; } break; } @@ -1108,11 +1151,11 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( const VkDescriptorAddressInfoEXT *bda = pCreateInfo->data.pStorageTexelBuffer; if (bda && bda->address) { enum pipe_format pformat = vk_format_to_pipe_format(bda->format); - lp_jit_image_buffer_from_bda(&desc->image, (void *)(uintptr_t)bda->address, bda->range, pformat); - desc->functions = get_image_handle_bda(device, bda->address, bda->range, pformat).functions; + lp_jit_image_buffer_from_bda(&image_desc->image, (void *)(uintptr_t)bda->address, bda->range, pformat); + image_desc->functions = get_image_handle_bda(device, bda->address, bda->range, pformat).functions; } else { - memset(desc, 0, size); - desc->functions = device->null_image_handle->functions; + memset(image_desc, 0, size); + image_desc->functions = device->null_image_handle->functions; } break; } @@ -1124,23 +1167,23 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT( .buffer_size = bda->range, }; - lp_jit_buffer_from_pipe_const(&desc->buffer, &ubo, device->pscreen); + lp_jit_buffer_from_pipe_const(&buffer_desc->jit, &ubo, device->pscreen); } else { - lp_jit_buffer_from_pipe_const(&desc->buffer, &((struct pipe_constant_buffer){0}), device->pscreen); + lp_jit_buffer_from_pipe_const(&buffer_desc->jit, &((struct pipe_constant_buffer){0}), device->pscreen); } break; } case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: { const VkDescriptorAddressInfoEXT *bda = pCreateInfo->data.pStorageBuffer; if (bda && bda->address) { - lp_jit_buffer_from_bda(&desc->buffer, (void *)(uintptr_t)bda->address, bda->range); + lp_jit_buffer_from_bda(&buffer_desc->jit, (void *)(uintptr_t)bda->address, bda->range); } else { - lp_jit_buffer_from_pipe(&desc->buffer, &((struct pipe_shader_buffer){0})); + lp_jit_buffer_from_pipe(&buffer_desc->jit, &((struct pipe_shader_buffer){0})); } break; } case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: { - desc->accel_struct = pCreateInfo->data.accelerationStructure; + *accel_struct_desc = pCreateInfo->data.accelerationStructure; break; } default: diff --git a/src/gallium/frontends/lavapipe/lvp_device.c b/src/gallium/frontends/lavapipe/lvp_device.c index 4e40b8d2b40..1f6b8f79907 100644 --- a/src/gallium/frontends/lavapipe/lvp_device.c +++ b/src/gallium/frontends/lavapipe/lvp_device.c @@ -936,7 +936,6 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie int texel_buffer_alignment = device->pscreen->caps.texture_buffer_offset_alignment; - STATIC_ASSERT(sizeof(struct lp_descriptor) <= 256); *p = (struct vk_properties) { /* Vulkan 1.0 */ .apiVersion = LVP_API_VERSION, @@ -1241,20 +1240,20 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie .imageViewCaptureReplayDescriptorDataSize = 0, .samplerCaptureReplayDescriptorDataSize = 0, .accelerationStructureCaptureReplayDescriptorDataSize = 0, - .EDBsamplerDescriptorSize = sizeof(struct lp_descriptor), - .combinedImageSamplerDescriptorSize = sizeof(struct lp_descriptor), - .sampledImageDescriptorSize = sizeof(struct lp_descriptor), - .storageImageDescriptorSize = sizeof(struct lp_descriptor), - .uniformTexelBufferDescriptorSize = sizeof(struct lp_descriptor), - .robustUniformTexelBufferDescriptorSize = sizeof(struct lp_descriptor), - .storageTexelBufferDescriptorSize = sizeof(struct lp_descriptor), - .robustStorageTexelBufferDescriptorSize = sizeof(struct lp_descriptor), - .uniformBufferDescriptorSize = sizeof(struct lp_descriptor), - .robustUniformBufferDescriptorSize = sizeof(struct lp_descriptor), - .storageBufferDescriptorSize = sizeof(struct lp_descriptor), - .robustStorageBufferDescriptorSize = sizeof(struct lp_descriptor), - .inputAttachmentDescriptorSize = sizeof(struct lp_descriptor), - .accelerationStructureDescriptorSize = sizeof(struct lp_descriptor), + .EDBsamplerDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_SAMPLER), + .combinedImageSamplerDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER), + .sampledImageDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE), + .storageImageDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE), + .uniformTexelBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER), + .robustUniformTexelBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER), + .storageTexelBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER), + .robustStorageTexelBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER), + .uniformBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER), + .robustUniformBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER), + .storageBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER), + .robustStorageBufferDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER), + .inputAttachmentDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT), + .accelerationStructureDescriptorSize = lvp_get_descriptor_size(VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR), .maxSamplerDescriptorBufferRange = UINT32_MAX, .maxResourceDescriptorBufferRange = UINT32_MAX, .resourceDescriptorBufferAddressSpaceSize = UINT32_MAX, @@ -2658,7 +2657,7 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_ResetEvent( } void -lvp_sampler_init(struct lvp_device *device, struct lp_descriptor *desc, const VkSamplerCreateInfo *pCreateInfo, const struct vk_sampler *sampler) +lvp_sampler_init(struct lvp_device *device, struct lp_sampler_descriptor *desc, const VkSamplerCreateInfo *pCreateInfo, const struct vk_sampler *sampler) { struct pipe_sampler_state state = {0}; VkClearColorValue border_color = @@ -2690,11 +2689,11 @@ lvp_sampler_init(struct lvp_device *device, struct lp_descriptor *desc, const Vk simple_mtx_lock(&device->queue.lock); struct lp_texture_handle *texture_handle = (void *)(uintptr_t)device->queue.ctx->create_texture_handle(device->queue.ctx, NULL, &state); - desc->texture.sampler_index = texture_handle->sampler_index; + desc->sampler_index = texture_handle->sampler_index; device->queue.ctx->delete_texture_handle(device->queue.ctx, (uint64_t)(uintptr_t)texture_handle); simple_mtx_unlock(&device->queue.lock); - lp_jit_sampler_from_pipe(&desc->sampler, &state); + lp_jit_sampler_from_pipe(&desc->jit, &state); } VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateSampler( diff --git a/src/gallium/frontends/lavapipe/lvp_execute.c b/src/gallium/frontends/lavapipe/lvp_execute.c index 0c431bde482..1ff0c0ff838 100644 --- a/src/gallium/frontends/lavapipe/lvp_execute.c +++ b/src/gallium/frontends/lavapipe/lvp_execute.c @@ -1368,15 +1368,13 @@ apply_dynamic_offsets(struct lvp_descriptor_set **out_set, const uint32_t *offse if (!vk_descriptor_type_is_dynamic(binding->type)) continue; - struct lp_descriptor *desc = set->map; - desc += binding->descriptor_index; - + struct lp_buffer_descriptor *desc = (struct lp_buffer_descriptor *)(set->map + binding->offset); for (uint32_t j = 0; j < binding->array_size; j++) { uint32_t offset_index = binding->dynamic_index + j; if (offset_index >= offset_count) return; - desc[j].buffer.u = (uint32_t *)((uint8_t *)desc[j].buffer.u + offsets[offset_index]); + desc[j].jit.u = (uint32_t *)((uint8_t *)desc[j].jit.u + offsets[offset_index]); } } } @@ -4796,13 +4794,10 @@ bind_db_samplers(struct rendering_state *state, enum lvp_pipeline_type pipeline_ if (!bind_layout->immutable_samplers) continue; - struct lp_descriptor *desc = (void*)db; - desc += bind_layout->descriptor_index; + struct lp_sampler_descriptor *desc = (struct lp_sampler_descriptor *)(db + bind_layout->offset); for (uint32_t sampler_index = 0; sampler_index < bind_layout->array_size; sampler_index++) { - struct lp_descriptor *immutable_desc = &bind_layout->immutable_samplers[sampler_index]; - desc[sampler_index].sampler = immutable_desc->sampler; - desc[sampler_index].texture.sampler_index = immutable_desc->texture.sampler_index; + desc[sampler_index] = bind_layout->immutable_samplers[sampler_index]; if (pipeline_type == LVP_PIPELINE_RAY_TRACING) { did_update |= BITFIELD_BIT(MESA_SHADER_RAYGEN); } else { diff --git a/src/gallium/frontends/lavapipe/lvp_pipeline.c b/src/gallium/frontends/lavapipe/lvp_pipeline.c index e62eb1135f7..70dddd786a9 100644 --- a/src/gallium/frontends/lavapipe/lvp_pipeline.c +++ b/src/gallium/frontends/lavapipe/lvp_pipeline.c @@ -753,7 +753,7 @@ layouts_equal(const struct lvp_descriptor_set_layout *a, const struct lvp_descri } if (!a->immutable_sampler_count) return true; - if (memcmp(la->immutable_samplers, lb->immutable_samplers, a->immutable_sampler_count * sizeof(struct lp_descriptor))) + if (memcmp(la->immutable_samplers, lb->immutable_samplers, a->immutable_sampler_count * sizeof(struct lp_sampler_descriptor))) return false; return !memcmp(la->immutable_ycbcr, lb->immutable_ycbcr, a->immutable_sampler_count * sizeof(struct vk_ycbcr_conversion_state)); } diff --git a/src/gallium/frontends/lavapipe/lvp_private.h b/src/gallium/frontends/lavapipe/lvp_private.h index c87d54f2aa1..fd7ade4033c 100644 --- a/src/gallium/frontends/lavapipe/lvp_private.h +++ b/src/gallium/frontends/lavapipe/lvp_private.h @@ -311,14 +311,15 @@ struct lvp_image_view { struct lvp_sampler { struct vk_sampler vk; - struct lp_descriptor desc; + struct lp_sampler_descriptor desc; }; struct lvp_descriptor_set_binding_layout { - uint32_t descriptor_index; + uint32_t offset; /* Number of array elements in this binding */ VkDescriptorType type; uint32_t stride; /* used for planar samplers */ + uint32_t max_plane_count; uint32_t array_size; bool valid; @@ -328,7 +329,7 @@ struct lvp_descriptor_set_binding_layout { uint32_t uniform_block_size; /* Immutable samplers (or NULL if no immutable samplers) */ - struct lp_descriptor *immutable_samplers; + struct lp_sampler_descriptor *immutable_samplers; struct vk_ycbcr_conversion_state *immutable_ycbcr; }; @@ -372,7 +373,7 @@ struct lvp_descriptor_set { /* Buffer holding the descriptors. */ struct pipe_memory_allocation *pmem; struct pipe_resource *bo; - void *map; + uint8_t *map; }; struct lvp_descriptor_pool { @@ -397,6 +398,15 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri VkDescriptorUpdateTemplate descriptorUpdateTemplate, const void *pData); +struct lvp_image_sampler_descriptor { + struct lp_image_descriptor image; + struct lp_sampler_descriptor sampler; +}; + +uint32_t lvp_get_descriptor_size(VkDescriptorType type); + +uint32_t lvp_get_sampler_descriptor_offset(VkDescriptorType type); + struct lvp_pipeline_layout { struct vk_pipeline_layout vk; @@ -756,7 +766,7 @@ void lvp_nir_lower_blend(nir_shader *nir, const nir_lower_blend_options *opts); void -lvp_sampler_init(struct lvp_device *device, struct lp_descriptor *desc, const VkSamplerCreateInfo *pCreateInfo, const struct vk_sampler *sampler); +lvp_sampler_init(struct lvp_device *device, struct lp_sampler_descriptor *desc, const VkSamplerCreateInfo *pCreateInfo, const struct vk_sampler *sampler); static inline uint8_t lvp_image_aspects_to_plane(ASSERTED const struct lvp_image *image, diff --git a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c index 16ae3cea6d7..567427df69d 100644 --- a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c +++ b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c @@ -34,7 +34,7 @@ static nir_def *lower_vri_intrin_vri(struct nir_builder *b, get_binding_layout(data_cb, desc_set_idx, binding_idx); return nir_vec3(b, nir_imm_int(b, desc_set_idx + 1), - nir_iadd_imm(b, intrin->src[0].ssa, binding->descriptor_index), + nir_iadd_imm(b, nir_imul_imm(b, intrin->src[0].ssa, binding->stride), binding->offset), nir_imm_int(b, 0)); } @@ -42,8 +42,23 @@ static nir_def *lower_vri_intrin_vrri(struct nir_builder *b, nir_instr *instr, void *data_cb) { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + nir_descriptor_type desc_type = nir_intrinsic_desc_type(intrin); + + uint32_t stride = 0; + switch (desc_type) { + case nir_descriptor_type_uniform_buffer: + case nir_descriptor_type_storage_buffer: + stride = sizeof(struct lp_buffer_descriptor); + break; + case nir_descriptor_type_acceleration_structure: + stride = sizeof(uint64_t); + break; + default: + UNREACHABLE("Unimplemented nir_descriptor_type"); + } + nir_def *old_index = intrin->src[0].ssa; - nir_def *delta = intrin->src[1].ssa; + nir_def *delta = nir_imul_imm(b, intrin->src[1].ssa, stride); return nir_vec3(b, nir_channel(b, old_index, 0), nir_iadd(b, nir_channel(b, old_index, 1), delta), nir_channel(b, old_index, 2)); @@ -63,10 +78,9 @@ lower_buffer(nir_builder *b, nir_intrinsic_instr *intr, uint32_t src_index) return; nir_def *set = nir_channel(b, intr->src[src_index].ssa, 0); - nir_def *binding = nir_channel(b, intr->src[src_index].ssa, 1); + nir_def *offset = nir_channel(b, intr->src[src_index].ssa, 1); nir_def *base = nir_load_const_buf_base_addr_lvp(b, set); - nir_def *offset = nir_imul_imm(b, binding, sizeof(struct lp_descriptor)); nir_def *descriptor = nir_iadd(b, base, nir_u2u64(b, offset)); nir_src_rewrite(&intr->src[src_index], descriptor); } @@ -78,15 +92,13 @@ lower_accel_struct(nir_builder *b, nir_intrinsic_instr *intr, uint32_t src_index return; nir_def *set = nir_channel(b, intr->src[src_index].ssa, 0); - nir_def *binding = nir_channel(b, intr->src[src_index].ssa, 1); - - nir_def *offset = nir_imul_imm(b, binding, sizeof(struct lp_descriptor)); + nir_def *offset = nir_channel(b, intr->src[src_index].ssa, 1); nir_src_rewrite(&intr->src[src_index], nir_load_ubo(b, 1, 64, set, offset, .range = UINT32_MAX)); } static nir_def * vulkan_resource_from_deref(nir_builder *b, nir_deref_instr *deref, const struct lvp_pipeline_layout *layout, - unsigned plane) + unsigned plane, bool is_sampler) { nir_def *index = nir_imm_int(b, 0); @@ -102,11 +114,14 @@ vulkan_resource_from_deref(nir_builder *b, nir_deref_instr *deref, const struct nir_variable *var = deref->var; const struct lvp_descriptor_set_binding_layout *binding = get_binding_layout(layout, var->data.descriptor_set, var->data.binding); - uint32_t binding_base = binding->descriptor_index + plane; - index = nir_iadd_imm(b, nir_imul_imm(b, index, binding->stride), binding_base); + + uint32_t binding_base = binding->offset + plane * lvp_get_descriptor_size(binding->type); + if (is_sampler) + binding_base += lvp_get_sampler_descriptor_offset(binding->type); + + nir_def *offset = nir_iadd_imm(b, nir_imul_imm(b, index, binding->stride), binding_base); nir_def *set = nir_load_const_buf_base_addr_lvp(b, nir_imm_int(b, var->data.descriptor_set + 1)); - nir_def *offset = nir_imul_imm(b, index, sizeof(struct lp_descriptor)); return nir_iadd(b, set, nir_u2u64(b, offset)); } @@ -119,6 +134,7 @@ static void lower_vri_instr_tex(struct nir_builder *b, plane_ssa ? nir_src_as_uint(nir_src_for_ssa(plane_ssa)) : 0; for (unsigned i = 0; i < tex->num_srcs; i++) { + bool is_sampler = false; nir_deref_instr *deref; switch (tex->src[i].src_type) { case nir_tex_src_texture_deref: @@ -128,12 +144,13 @@ static void lower_vri_instr_tex(struct nir_builder *b, case nir_tex_src_sampler_deref: tex->src[i].src_type = nir_tex_src_sampler_handle; deref = nir_src_as_deref(tex->src[i].src); + is_sampler = true; break; default: continue; } - nir_def *resource = vulkan_resource_from_deref(b, deref, layout, plane); + nir_def *resource = vulkan_resource_from_deref(b, deref, layout, plane, is_sampler); nir_src_rewrite(&tex->src[i].src, resource); } } @@ -147,7 +164,7 @@ lower_image_intrinsic(nir_builder *b, nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); - nir_def *resource = vulkan_resource_from_deref(b, deref, layout, 0); + nir_def *resource = vulkan_resource_from_deref(b, deref, layout, 0, false); nir_rewrite_image_intrinsic(intrin, resource, nir_image_intrinsic_type_bindless); }