diff --git a/src/modules/module-protocol-pulse/module.c b/src/modules/module-protocol-pulse/module.c
index f87889d5d..57790a6ae 100644
--- a/src/modules/module-protocol-pulse/module.c
+++ b/src/modules/module-protocol-pulse/module.c
@@ -50,8 +50,11 @@ static void on_module_unload(void *obj, void *data, int res, uint32_t id)
 
 void module_schedule_unload(struct module *module)
 {
-	struct impl *impl = module->impl;
-	pw_work_queue_add(impl->work_queue, module, 0, on_module_unload, impl);
+	if (module->unloading)
+		return;
+
+	pw_work_queue_add(module->impl->work_queue, module, 0, on_module_unload, NULL);
+	module->unloading = true;
 }
 
 struct module *module_new(struct impl *impl, const struct module_methods *methods, size_t user_data)
diff --git a/src/modules/module-protocol-pulse/module.h b/src/modules/module-protocol-pulse/module.h
index b5ec49df4..f621edfa5 100644
--- a/src/modules/module-protocol-pulse/module.h
+++ b/src/modules/module-protocol-pulse/module.h
@@ -66,6 +66,7 @@ struct module {
 	struct spa_hook_list listener_list;
 	void *user_data;
 	unsigned int loaded:1;
+	unsigned int unloading:1;
 };
 
 #define module_emit_loaded(m,r) spa_hook_list_call(&m->listener_list, struct module_events, loaded, 0, r)