diff --git a/src/freedreno/vulkan/tu_drm.c b/src/freedreno/vulkan/tu_drm.c
index d23aab87509..7edf7c68fdf 100644
--- a/src/freedreno/vulkan/tu_drm.c
+++ b/src/freedreno/vulkan/tu_drm.c
@@ -436,40 +436,12 @@ tu_enumerate_devices(struct tu_instance *instance)
    return result;
 }
 
-// Queue semaphore functions
-
 static void
-tu_semaphore_part_destroy(struct tu_device *device,
-                          struct tu_semaphore_part *part)
+semaphore_set_temporary(struct tu_device *device, struct tu_semaphore *sem, uint32_t syncobj)
 {
-   switch(part->kind) {
-   case TU_SEMAPHORE_NONE:
-      break;
-   case TU_SEMAPHORE_SYNCOBJ:
-      drmSyncobjDestroy(device->fd, part->syncobj);
-      break;
-   }
-   part->kind = TU_SEMAPHORE_NONE;
-}
-
-static void
-tu_semaphore_remove_temp(struct tu_device *device,
-                         struct tu_semaphore *sem)
-{
-   if (sem->temporary.kind != TU_SEMAPHORE_NONE) {
-      tu_semaphore_part_destroy(device, &sem->temporary);
-   }
-}
-
-static void
-tu_semaphores_remove_temp(struct tu_device *device,
-                          const VkSemaphore *sems,
-                          uint32_t sem_count)
-{
-   for (uint32_t i = 0; i  < sem_count; ++i) {
-      TU_FROM_HANDLE(tu_semaphore, sem, sems[i]);
-      tu_semaphore_remove_temp(device, sem);
-   }
+   if (sem->temporary)
+      drmSyncobjDestroy(device->fd, sem->temporary);
+   sem->temporary = syncobj;
 }
 
 VkResult
@@ -486,14 +458,13 @@ tu_CreateSemaphore(VkDevice _device,
    if (!sem)
       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 
-   sem->temporary.kind = TU_SEMAPHORE_NONE;
-   sem->permanent.kind = TU_SEMAPHORE_SYNCOBJ;
-
-   if (drmSyncobjCreate(device->fd, 0, &sem->permanent.syncobj) < 0) {
+   if (drmSyncobjCreate(device->fd, 0, &sem->permanent) < 0) {
          vk_free2(&device->vk.alloc, pAllocator, sem);
          return VK_ERROR_OUT_OF_HOST_MEMORY;
    }
 
+   sem->temporary = 0;
+
    *pSemaphore = tu_semaphore_to_handle(sem);
    return VK_SUCCESS;
 }
@@ -508,8 +479,8 @@ tu_DestroySemaphore(VkDevice _device,
    if (!_semaphore)
       return;
 
-   tu_semaphore_part_destroy(device, &sem->permanent);
-   tu_semaphore_part_destroy(device, &sem->temporary);
+   semaphore_set_temporary(device, sem, 0);
+   drmSyncobjDestroy(device->fd, sem->permanent);
 
    vk_object_free(&device->vk, pAllocator, sem);
 }
@@ -521,7 +492,7 @@ tu_ImportSemaphoreFdKHR(VkDevice _device,
    TU_FROM_HANDLE(tu_device, device, _device);
    TU_FROM_HANDLE(tu_semaphore, sem, pImportSemaphoreFdInfo->semaphore);
    int ret;
-   struct tu_semaphore_part *dst = NULL;
+   uint32_t *dst = NULL;
 
    if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) {
       dst = &sem->temporary;
@@ -529,7 +500,7 @@ tu_ImportSemaphoreFdKHR(VkDevice _device,
       dst = &sem->permanent;
    }
 
-   uint32_t syncobj = dst->kind == TU_SEMAPHORE_SYNCOBJ ? dst->syncobj : 0;
+   uint32_t syncobj = *dst;
 
    switch(pImportSemaphoreFdInfo->handleType) {
       case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: {
@@ -564,8 +535,7 @@ tu_ImportSemaphoreFdKHR(VkDevice _device,
    if (ret) {
       return VK_ERROR_INVALID_EXTERNAL_HANDLE;
    }
-   dst->syncobj = syncobj;
-   dst->kind = TU_SEMAPHORE_SYNCOBJ;
+   *dst = syncobj;
 
    return VK_SUCCESS;
 }
@@ -578,15 +548,7 @@ tu_GetSemaphoreFdKHR(VkDevice _device,
    TU_FROM_HANDLE(tu_device, device, _device);
    TU_FROM_HANDLE(tu_semaphore, sem, pGetFdInfo->semaphore);
    int ret;
-   uint32_t syncobj_handle;
-
-   if (sem->temporary.kind != TU_SEMAPHORE_NONE) {
-      assert(sem->temporary.kind == TU_SEMAPHORE_SYNCOBJ);
-      syncobj_handle = sem->temporary.syncobj;
-   } else {
-      assert(sem->permanent.kind == TU_SEMAPHORE_SYNCOBJ);
-      syncobj_handle = sem->permanent.syncobj;
-   }
+   uint32_t syncobj_handle = sem->temporary ?: sem->permanent;
 
    switch(pGetFdInfo->handleType) {
    case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
@@ -595,8 +557,8 @@ tu_GetSemaphoreFdKHR(VkDevice _device,
    case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
       ret = drmSyncobjExportSyncFile(device->fd, syncobj_handle, pFd);
       if (!ret) {
-         if (sem->temporary.kind != TU_SEMAPHORE_NONE) {
-            tu_semaphore_part_destroy(device, &sem->temporary);
+         if (sem->temporary) {
+            semaphore_set_temporary(device, sem, 0);
          } else {
             drmSyncobjReset(device->fd, &syncobj_handle, 1);
          }
@@ -652,28 +614,16 @@ tu_QueueSubmit(VkQueue _queue,
 
       for (uint32_t i = 0; i < submit->waitSemaphoreCount; i++) {
          TU_FROM_HANDLE(tu_semaphore, sem, submit->pWaitSemaphores[i]);
-
-         struct tu_semaphore_part *part =
-            sem->temporary.kind != TU_SEMAPHORE_NONE ?
-               &sem->temporary : &sem->permanent;
-         if (part->kind != TU_SEMAPHORE_SYNCOBJ)
-            continue;
          in_syncobjs[nr_in_syncobjs++] = (struct drm_msm_gem_submit_syncobj) {
-            .handle = part->syncobj,
+            .handle = sem->temporary ?: sem->permanent,
             .flags = MSM_SUBMIT_SYNCOBJ_RESET,
          };
       }
 
       for (uint32_t i = 0; i < submit->signalSemaphoreCount; i++) {
          TU_FROM_HANDLE(tu_semaphore, sem, submit->pSignalSemaphores[i]);
-
-         struct tu_semaphore_part *part =
-            sem->temporary.kind != TU_SEMAPHORE_NONE ?
-               &sem->temporary : &sem->permanent;
-         if (part->kind != TU_SEMAPHORE_SYNCOBJ)
-            continue;
          out_syncobjs[nr_out_syncobjs++] = (struct drm_msm_gem_submit_syncobj) {
-            .handle = part->syncobj,
+            .handle = sem->temporary ?: sem->permanent,
             .flags = 0,
          };
       }
@@ -744,8 +694,12 @@ tu_QueueSubmit(VkQueue _queue,
                                    strerror(errno));
       }
 
-      tu_semaphores_remove_temp(queue->device, pSubmits[i].pWaitSemaphores,
-                                pSubmits[i].waitSemaphoreCount);
+      /* restore permanent payload on wait */
+      for (uint32_t i = 0; i < submit->waitSemaphoreCount; i++) {
+         TU_FROM_HANDLE(tu_semaphore, sem, submit->pWaitSemaphores[i]);
+         semaphore_set_temporary(queue->device, sem, 0);
+      }
+
       if (last_submit) {
          if (queue->fence >= 0)
             close(queue->fence);
diff --git a/src/freedreno/vulkan/tu_private.h b/src/freedreno/vulkan/tu_private.h
index 6f98ee2fdbb..ea26f757e02 100644
--- a/src/freedreno/vulkan/tu_private.h
+++ b/src/freedreno/vulkan/tu_private.h
@@ -1486,26 +1486,9 @@ struct tu_query_pool
    struct tu_bo bo;
 };
 
-enum tu_semaphore_kind
-{
-   TU_SEMAPHORE_NONE,
-   TU_SEMAPHORE_SYNCOBJ,
-};
-
-struct tu_semaphore_part
-{
-   enum tu_semaphore_kind kind;
-   union {
-      uint32_t syncobj;
-   };
-};
-
-struct tu_semaphore
-{
+struct tu_semaphore {
    struct vk_object_base base;
-
-   struct tu_semaphore_part permanent;
-   struct tu_semaphore_part temporary;
+   uint32_t permanent, temporary;
 };
 
 void