diff --git a/src/libeis-socket.c b/src/libeis-socket.c
index 5ce918b..71fc2aa 100644
--- a/src/libeis-socket.c
+++ b/src/libeis-socket.c
@@ -165,8 +165,14 @@ eis_setup_backend_socket(struct eis *eis, const char *socketpath)
 			log_error(eis, "Failed to stat socket path %s (%s)", path, strerror(errno));
 			return -errno;
 		}
-	} else if (st.st_mode & (S_IWUSR | S_IWGRP)) {
+	} else if (S_ISSOCK(st.st_mode)) {
 		unlink(path);
+	} else if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode) || S_ISDIR(st.st_mode)) {
+		log_error(eis,
+			  "Socket path %s exists but is not a socket (mode 0%o)",
+			  path,
+			  st.st_mode);
+		return -EEXIST;
 	}
 
 	/* Lockfile succeeded and path is unlinked (if it existed), let's set