diff --git a/src/ply-boot-server.c b/src/ply-boot-server.c
index 55195506..173063f7 100644
--- a/src/ply-boot-server.c
+++ b/src/ply-boot-server.c
@@ -47,7 +47,10 @@ typedef struct
         uid_t              uid;
         pid_t              pid;
 
+        int                reference_count;
+
         uint32_t           credentials_read : 1;
+        uint32_t           disconnected : 1;
 } ply_boot_connection_t;
 
 struct _ply_boot_server
@@ -165,6 +168,7 @@ ply_boot_connection_new (ply_boot_server_t *server,
         connection->fd = fd;
         connection->server = server;
         connection->watch = NULL;
+        connection->reference_count = 1;
 
         return connection;
 }
@@ -179,6 +183,26 @@ ply_boot_connection_free (ply_boot_connection_t *connection)
         free (connection);
 }
 
+static void
+ply_boot_connection_take_reference (ply_boot_connection_t *connection)
+{
+        connection->reference_count++;
+}
+
+static void
+ply_boot_connection_drop_reference (ply_boot_connection_t *connection)
+{
+        if (connection == NULL)
+                return;
+
+        connection->reference_count--;
+
+        assert (connection->reference_count >= 0);
+
+        if (connection->reference_count == 0)
+                ply_boot_connection_free (connection);
+}
+
 bool
 ply_boot_server_listen (ply_boot_server_t *server)
 {
@@ -292,30 +316,43 @@ ply_boot_connection_on_password_answer (ply_boot_connection_t *connection,
 {
         ply_trace ("got password answer");
 
-        ply_boot_connection_send_answer (connection, password);
+        if (!connection->disconnected)
+                ply_boot_connection_send_answer (connection, password);
+
         if (password != NULL)
                 ply_list_append_data (connection->server->cached_passwords,
                                       strdup (password));
+
+        ply_boot_connection_drop_reference (connection);
 }
 
 static void
 ply_boot_connection_on_deactivated (ply_boot_connection_t *connection)
 {
         ply_trace ("deactivated");
-        if (!ply_write (connection->fd,
-                        PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK,
-                        strlen (PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK)))
-                ply_trace ("could not finish writing deactivate reply: %m");
+
+        if (!connection->disconnected) {
+                if (!ply_write (connection->fd,
+                                PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK,
+                                strlen (PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK)))
+                        ply_trace ("could not finish writing deactivate reply: %m");
+        }
+
+        ply_boot_connection_drop_reference (connection);
 }
 
 static void
 ply_boot_connection_on_quit_complete (ply_boot_connection_t *connection)
 {
         ply_trace ("quit complete");
-        if (!ply_write (connection->fd,
-                        PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK,
-                        strlen (PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK)))
-                ply_trace ("could not finish writing quit reply: %m");
+        if (!connection->disconnected) {
+                if (!ply_write (connection->fd,
+                                PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK,
+                                strlen (PLY_BOOT_PROTOCOL_RESPONSE_TYPE_ACK)))
+                        ply_trace ("could not finish writing quit reply: %m");
+        }
+
+        ply_boot_connection_drop_reference (connection);
 }
 
 static void
@@ -323,7 +360,10 @@ ply_boot_connection_on_question_answer (ply_boot_connection_t *connection,
                                         const char            *answer)
 {
         ply_trace ("got question answer: %s", answer);
-        ply_boot_connection_send_answer (connection, answer);
+        if (!connection->disconnected)
+                ply_boot_connection_send_answer (connection, answer);
+
+        ply_boot_connection_drop_reference (connection);
 }
 
 static void
@@ -331,7 +371,10 @@ ply_boot_connection_on_keystroke_answer (ply_boot_connection_t *connection,
                                          const char            *key)
 {
         ply_trace ("got key: %s", key);
-        ply_boot_connection_send_answer (connection, key);
+        if (!connection->disconnected)
+                ply_boot_connection_send_answer (connection, key);
+
+        ply_boot_connection_drop_reference (connection);
 }
 
 static void
@@ -464,6 +507,7 @@ ply_boot_connection_on_request (ply_boot_connection_t *connection)
                                          (ply_trigger_handler_t)
                                          ply_boot_connection_on_deactivated,
                                          connection);
+                ply_boot_connection_take_reference (connection);
 
                 if (server->deactivate_handler != NULL)
                         server->deactivate_handler (server->user_data, deactivate_trigger, server);
@@ -491,6 +535,7 @@ ply_boot_connection_on_request (ply_boot_connection_t *connection)
                                          (ply_trigger_handler_t)
                                          ply_boot_connection_on_quit_complete,
                                          connection);
+                ply_boot_connection_take_reference (connection);
 
                 if (server->quit_handler != NULL)
                         server->quit_handler (server->user_data, retain_splash, quit_trigger, server);
@@ -510,6 +555,7 @@ ply_boot_connection_on_request (ply_boot_connection_t *connection)
                                          (ply_trigger_handler_t)
                                          ply_boot_connection_on_password_answer,
                                          connection);
+                ply_boot_connection_take_reference (connection);
 
                 if (server->ask_for_password_handler != NULL) {
                         server->ask_for_password_handler (server->user_data,
@@ -594,6 +640,7 @@ ply_boot_connection_on_request (ply_boot_connection_t *connection)
                                          (ply_trigger_handler_t)
                                          ply_boot_connection_on_question_answer,
                                          connection);
+                ply_boot_connection_take_reference (connection);
 
                 if (server->ask_question_handler != NULL) {
                         server->ask_question_handler (server->user_data,
@@ -626,6 +673,7 @@ ply_boot_connection_on_request (ply_boot_connection_t *connection)
                                          (ply_trigger_handler_t)
                                          ply_boot_connection_on_keystroke_answer,
                                          connection);
+                ply_boot_connection_take_reference (connection);
 
                 if (server->watch_for_keystroke_handler != NULL) {
                         server->watch_for_keystroke_handler (server->user_data,
@@ -707,13 +755,15 @@ ply_boot_connection_on_hangup (ply_boot_connection_t *connection)
         assert (connection != NULL);
         assert (connection->server != NULL);
 
+        connection->disconnected = true;
+
         server = connection->server;
 
         node = ply_list_find_node (server->connections, connection);
 
         assert (node != NULL);
 
-        ply_boot_connection_free (connection);
+        ply_boot_connection_drop_reference (connection);
         ply_list_remove_node (server->connections, node);
 }