diff --git a/src/libei.c b/src/libei.c
index 0bfaace..150840b 100644
--- a/src/libei.c
+++ b/src/libei.c
@@ -357,6 +357,7 @@ ei_disconnect(struct ei *ei)
 	/* FIXME: unroll the devices here */
 	ei_queue_disconnect_event(ei);
 	ei->state = EI_STATE_DISCONNECTED;
+	source_remove(ei->source);
 	ei->source = source_unref(ei->source);
 }
 
diff --git a/src/util-sources.c b/src/util-sources.c
index 52c2c87..99d746c 100644
--- a/src/util-sources.c
+++ b/src/util-sources.c
@@ -55,6 +55,7 @@ struct source {
 	int fd;
 };
 
+OBJECT_IMPLEMENT_REF(source);
 OBJECT_IMPLEMENT_UNREF(source);
 OBJECT_IMPLEMENT_GETTER(source, fd, int);
 OBJECT_IMPLEMENT_GETTER(source, user_data, void*);
@@ -72,6 +73,8 @@ source_remove(struct source *source)
 		if (source->close_behavior == SOURCE_CLOSE_ON_REMOVE)
 			source->fd = xclose(source->fd);
 	}
+	/* Note: epollfd was the owner of the source, new owner is the removed
+	   list */
 	list_remove(&source->link);
 	list_append(&source->sink->sources_removed, &source->link);
 }
@@ -105,7 +108,7 @@ source_add(struct sink *sink, int sourcefd,
 
 	struct epoll_event e = {
 		.events = EPOLLIN,
-		.data.ptr = source,
+		.data.ptr = source, /* epollfd is the owner of this source */
 	};
 
 	if (epoll_ctl(sink->epollfd, EPOLL_CTL_ADD, sourcefd, &e) < 0) {
@@ -115,7 +118,7 @@ source_add(struct sink *sink, int sourcefd,
 
 	list_append(&sink->sources, &source->link);
 
-	return source;
+	return source_ref(source);
 }
 
 struct source *
diff --git a/src/util-sources.h b/src/util-sources.h
index 03683be..bcd570e 100644
--- a/src/util-sources.h
+++ b/src/util-sources.h
@@ -43,6 +43,9 @@ typedef void (*source_dispatch_t)(struct source *source, void *user_data);
  */
 void source_remove(struct source *source);
 
+struct source *
+source_ref(struct source *source);
+
 /**
  * Unref source. When the last reference is dropped, the source is removed
  * from the sink and resources are released.