diff --git a/src/compiler/nir/nir_deref.c b/src/compiler/nir/nir_deref.c
index 860a0f84822..1cfea51c9c9 100644
--- a/src/compiler/nir/nir_deref.c
+++ b/src/compiler/nir/nir_deref.c
@@ -230,7 +230,8 @@ nir_deref_instr_has_complex_use(nir_deref_instr *deref,
 
          case nir_intrinsic_deref_atomic:
          case nir_intrinsic_deref_atomic_swap:
-            if (opts & nir_deref_instr_has_complex_use_allow_atomics)
+            if (use_src == &use_intrin->src[0] &&
+                (opts & nir_deref_instr_has_complex_use_allow_atomics))
                continue;
             return true;