turnip: Implement and enable VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT

Tested by
dEQP-VK.transform_feedback.simple.query*

Signed-off-by: Hyunjun Ko <zzoon@igalia.com>
Reviewed-by: Brian Ho <brian@brkho.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4604>

src/freedreno/vulkan/tu_device.c

@@ -921,8 +921,7 @@ tu_GetPhysicalDeviceProperties2(VkPhysicalDevice physicalDevice,
          properties->maxTransformFeedbackStreamDataSize = 512;
          properties->maxTransformFeedbackBufferDataSize = 512;
          properties->maxTransformFeedbackBufferDataStride = 512;
-         /* TODO: enable xfb query */
+         properties->transformFeedbackQueries = true;
-         properties->transformFeedbackQueries = false;
          properties->transformFeedbackStreamsLinesTriangles = false;
          properties->transformFeedbackRasterizationStreamSelect = false;
          properties->transformFeedbackDraw = true;

src/freedreno/vulkan/tu_query.c

@@ -62,6 +62,17 @@ struct PACKED occlusion_query_slot {
    struct slot_value end;
 };
 
+/* The result of transform feedback queries is two integer values:
+ *   common.result.values[0] is the count of primitives written,
+ *   common.result.values[1] is the count of primitives generated.
+ * Also a result for each stream is stored at 4 slots respectively.
+ */
+struct PACKED primitive_query_slot {
+   struct query_slot common;
+   struct slot_value begin[4];
+   struct slot_value end[4];
+};
+
 /* Returns the IOVA of a given uint64_t field in a given slot of a query
  * pool. */
 #define query_iova(type, pool, query, field, value_index)            \
@@ -71,6 +82,9 @@ struct PACKED occlusion_query_slot {
 #define occlusion_query_iova(pool, query, field)                     \
    query_iova(struct occlusion_query_slot, pool, query, field, 0)
 
+#define primitive_query_iova(pool, query, field, i)                  \
+   query_iova(struct primitive_query_slot, pool, query, field, i)
+
 #define query_available_iova(pool, query)                            \
    query_iova(struct query_slot, pool, query, available, 0)
 
@@ -105,6 +119,9 @@ tu_CreateQueryPool(VkDevice _device,
    case VK_QUERY_TYPE_TIMESTAMP:
       slot_size = sizeof(struct query_slot);
       break;
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      slot_size = sizeof(struct primitive_query_slot);
+      break;
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
       unreachable("Unimplemented query type");
    default:
@@ -297,6 +314,7 @@ tu_GetQueryPoolResults(VkDevice _device,
    switch (pool->type) {
    case VK_QUERY_TYPE_OCCLUSION:
    case VK_QUERY_TYPE_TIMESTAMP:
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
       return get_query_pool_results(device, pool, firstQuery, queryCount,
                                     dataSize, pData, stride, flags);
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
@@ -428,6 +446,7 @@ tu_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer,
    switch (pool->type) {
    case VK_QUERY_TYPE_OCCLUSION:
    case VK_QUERY_TYPE_TIMESTAMP:
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
       return emit_copy_query_pool_results(cmdbuf, cs, pool, firstQuery,
                queryCount, buffer, dstOffset, stride, flags);
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
@@ -473,6 +492,7 @@ tu_CmdResetQueryPool(VkCommandBuffer commandBuffer,
    switch (pool->type) {
    case VK_QUERY_TYPE_TIMESTAMP:
    case VK_QUERY_TYPE_OCCLUSION:
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
       emit_reset_query_pool(cmdbuf, pool, firstQuery, queryCount);
       break;
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
@@ -516,6 +536,19 @@ emit_begin_occlusion_query(struct tu_cmd_buffer *cmdbuf,
    tu_cs_emit(cs, ZPASS_DONE);
 }
 
+static void
+emit_begin_xfb_query(struct tu_cmd_buffer *cmdbuf,
+                     struct tu_query_pool *pool,
+                     uint32_t query,
+                     uint32_t stream_id)
+{
+   struct tu_cs *cs = cmdbuf->state.pass ? &cmdbuf->draw_cs : &cmdbuf->cs;
+   uint64_t begin_iova = primitive_query_iova(pool, query, begin[0], 0);
+
+   tu_cs_emit_regs(cs, A6XX_VPC_SO_STREAM_COUNTS_LO(begin_iova));
+   tu6_emit_event_write(cmdbuf, cs, WRITE_PRIMITIVE_COUNTS, false);
+}
+
 void
 tu_CmdBeginQuery(VkCommandBuffer commandBuffer,
                  VkQueryPool queryPool,
@@ -534,6 +567,9 @@ tu_CmdBeginQuery(VkCommandBuffer commandBuffer,
        */
       emit_begin_occlusion_query(cmdbuf, pool, query);
       break;
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      emit_begin_xfb_query(cmdbuf, pool, query, 0);
+      break;
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
    case VK_QUERY_TYPE_TIMESTAMP:
       unreachable("Unimplemented query type");
@@ -544,6 +580,28 @@ tu_CmdBeginQuery(VkCommandBuffer commandBuffer,
    tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
 }
 
+void
+tu_CmdBeginQueryIndexedEXT(VkCommandBuffer commandBuffer,
+                           VkQueryPool queryPool,
+                           uint32_t query,
+                           VkQueryControlFlags flags,
+                           uint32_t index)
+{
+   TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
+   TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
+   assert(query < pool->size);
+
+   switch (pool->type) {
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      emit_begin_xfb_query(cmdbuf, pool, query, index);
+      break;
+   default:
+      assert(!"Invalid query type");
+   }
+
+   tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
+}
+
 static void
 emit_end_occlusion_query(struct tu_cmd_buffer *cmdbuf,
                          struct tu_query_pool *pool,
@@ -617,6 +675,55 @@ emit_end_occlusion_query(struct tu_cmd_buffer *cmdbuf,
    tu_cs_emit_qw(cs, 0x1);
 }
 
+static void
+emit_end_xfb_query(struct tu_cmd_buffer *cmdbuf,
+                   struct tu_query_pool *pool,
+                   uint32_t query,
+                   uint32_t stream_id)
+{
+   struct tu_cs *cs = cmdbuf->state.pass ? &cmdbuf->draw_cs : &cmdbuf->cs;
+
+   uint64_t end_iova = primitive_query_iova(pool, query, end[0], 0);
+   uint64_t result_written_iova = query_result_iova(pool, query, 0);
+   uint64_t result_generated_iova = query_result_iova(pool, query, 1);
+   uint64_t begin_written_iova = primitive_query_iova(pool, query, begin[stream_id], 0);
+   uint64_t begin_generated_iova = primitive_query_iova(pool, query, begin[stream_id], 1);
+   uint64_t end_written_iova = primitive_query_iova(pool, query, end[stream_id], 0);
+   uint64_t end_generated_iova = primitive_query_iova(pool, query, end[stream_id], 1);
+   uint64_t available_iova = query_available_iova(pool, query);
+
+   tu_cs_emit_regs(cs, A6XX_VPC_SO_STREAM_COUNTS_LO(end_iova));
+   tu6_emit_event_write(cmdbuf, cs, WRITE_PRIMITIVE_COUNTS, false);
+
+   tu_cs_emit_wfi(cs);
+   tu6_emit_event_write(cmdbuf, cs, CACHE_FLUSH_TS, true);
+
+   /* Set the count of written primitives */
+   tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 9);
+   tu_cs_emit(cs, CP_MEM_TO_MEM_0_DOUBLE | CP_MEM_TO_MEM_0_NEG_C |
+                  CP_MEM_TO_MEM_0_WAIT_FOR_MEM_WRITES | 0x80000000);
+   tu_cs_emit_qw(cs, result_written_iova);
+   tu_cs_emit_qw(cs, result_written_iova);
+   tu_cs_emit_qw(cs, end_written_iova);
+   tu_cs_emit_qw(cs, begin_written_iova);
+
+   tu6_emit_event_write(cmdbuf, cs, CACHE_FLUSH_TS, true);
+
+   /* Set the count of generated primitives */
+   tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 9);
+   tu_cs_emit(cs, CP_MEM_TO_MEM_0_DOUBLE | CP_MEM_TO_MEM_0_NEG_C |
+                  CP_MEM_TO_MEM_0_WAIT_FOR_MEM_WRITES | 0x80000000);
+   tu_cs_emit_qw(cs, result_generated_iova);
+   tu_cs_emit_qw(cs, result_generated_iova);
+   tu_cs_emit_qw(cs, end_generated_iova);
+   tu_cs_emit_qw(cs, begin_generated_iova);
+
+   /* Set the availability to 1 */
+   tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 4);
+   tu_cs_emit_qw(cs, available_iova);
+   tu_cs_emit_qw(cs, 0x1);
+}
+
 void
 tu_CmdEndQuery(VkCommandBuffer commandBuffer,
                VkQueryPool queryPool,
@@ -630,6 +737,9 @@ tu_CmdEndQuery(VkCommandBuffer commandBuffer,
    case VK_QUERY_TYPE_OCCLUSION:
       emit_end_occlusion_query(cmdbuf, pool, query);
       break;
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      emit_end_xfb_query(cmdbuf, pool, query, 0);
+      break;
    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
    case VK_QUERY_TYPE_TIMESTAMP:
       unreachable("Unimplemented query type");
@@ -640,6 +750,28 @@ tu_CmdEndQuery(VkCommandBuffer commandBuffer,
    tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
 }
 
+void
+tu_CmdEndQueryIndexedEXT(VkCommandBuffer commandBuffer,
+                         VkQueryPool queryPool,
+                         uint32_t query,
+                         uint32_t index)
+{
+   TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
+   TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
+   assert(query < pool->size);
+
+   switch (pool->type) {
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      assert(index <= 4);
+      emit_end_xfb_query(cmdbuf, pool, query, index);
+      break;
+   default:
+      assert(!"Invalid query type");
+   }
+
+   tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
+}
+
 void
 tu_CmdWriteTimestamp(VkCommandBuffer commandBuffer,
                      VkPipelineStageFlagBits pipelineStage,