diff --git a/src/broadcom/compiler/vir_opt_small_immediates.c b/src/broadcom/compiler/vir_opt_small_immediates.c
index 49773a8f85e..551bc1c8649 100644
--- a/src/broadcom/compiler/vir_opt_small_immediates.c
+++ b/src/broadcom/compiler/vir_opt_small_immediates.c
@@ -32,27 +32,10 @@
 
 static bool debug;
 
-static bool
+static inline bool
 skip_inst(struct qinst *inst)
 {
-        if (inst->qpu.type != V3D_QPU_INSTR_TYPE_ALU)
-                return true;
-
-        if (!vir_is_add(inst))
-                return false;
-
-        switch (inst->qpu.alu.add.op) {
-        case V3D_QPU_A_LDVPMV_IN:
-        case V3D_QPU_A_LDVPMG_IN:
-        case V3D_QPU_A_LDVPMD_IN:
-        case V3D_QPU_A_LDVPMP:
-        case V3D_QPU_A_STVPMV:
-        case V3D_QPU_A_STVPMD:
-        case V3D_QPU_A_STVPMP:
-                return true;
-        default:
-                return false;
-        }
+        return inst->qpu.type != V3D_QPU_INSTR_TYPE_ALU;
 }
 
 bool
@@ -60,6 +43,14 @@ vir_opt_small_immediates(struct v3d_compile *c)
 {
         bool progress = false;
 
+        /* Shader-db shows that small immediates generally lead to higher
+         * instruction counts for geometry stages.
+         */
+        if (c->s->info.stage != MESA_SHADER_FRAGMENT &&
+            c->s->info.stage != MESA_SHADER_COMPUTE) {
+                return progress;
+        }
+
         vir_for_each_inst_inorder(inst, c) {
                 if (skip_inst(inst))
                         continue;