diff --git a/src/devices/nm-device.c b/src/devices/nm-device.c
index e0fac76140..43f03739bc 100644
--- a/src/devices/nm-device.c
+++ b/src/devices/nm-device.c
@@ -3885,12 +3885,18 @@ device_link_changed (NMDevice *self)
 	if (priv->up && (!was_up || seen_down)) {
 		/* the link was down and just came up. That happens for example, while changing MTU.
 		 * We must restore IP configuration. */
-		if (!ip_config_merge_and_apply (self, AF_INET, TRUE))
-			_LOGW (LOGD_IP4, "failed applying IP4 config after link comes up again");
+		if (NM_IN_SET (priv->ip_state_4, NM_DEVICE_IP_STATE_CONF,
+		                                 NM_DEVICE_IP_STATE_DONE)) {
+			if (!ip_config_merge_and_apply (self, AF_INET, TRUE))
+				_LOGW (LOGD_IP4, "failed applying IP4 config after link comes up again");
+		}
 
 		priv->linklocal6_dad_counter = 0;
-		if (!ip_config_merge_and_apply (self, AF_INET6, TRUE))
-			_LOGW (LOGD_IP6, "failed applying IP6 config after link comes up again");
+		if (NM_IN_SET (priv->ip_state_6, NM_DEVICE_IP_STATE_CONF,
+		                                 NM_DEVICE_IP_STATE_DONE)) {
+			if (!ip_config_merge_and_apply (self, AF_INET6, TRUE))
+				_LOGW (LOGD_IP6, "failed applying IP6 config after link comes up again");
+		}
 	}
 
 	if (update_unmanaged_specs)