diff --git a/src/devices/nm-device.c b/src/devices/nm-device.c
index d228d6d373..c101185830 100644
--- a/src/devices/nm-device.c
+++ b/src/devices/nm-device.c
@@ -3179,7 +3179,8 @@ ip4_config_merge_and_apply (NMDevice *self,
 
 	composite = nm_ip4_config_new ();
 
-	ensure_con_ipx_config (self);
+	if (commit)
+		ensure_con_ipx_config (self);
 
 	if (priv->dev_ip4_config)
 		nm_ip4_config_merge (composite, priv->dev_ip4_config);
@@ -3756,7 +3757,8 @@ ip6_config_merge_and_apply (NMDevice *self,
 	/* If no config was passed in, create a new one */
 	composite = nm_ip6_config_new ();
 
-	ensure_con_ipx_config (self);
+	if (commit)
+		ensure_con_ipx_config (self);
 
 	/* Merge all the IP configs into the composite config */
 	if (priv->ac_ip6_config)
@@ -6991,7 +6993,6 @@ update_ip_config (NMDevice *self, gboolean initial)
 			g_clear_object (&priv->dev_ip4_config);
 			capture_lease_config (self, priv->ext_ip4_config, &priv->dev_ip4_config, NULL, NULL);
 		}
-		ensure_con_ipx_config (self);
 
 		/* This function was called upon external changes. Remove the configuration
 		 * (adresses,routes) that is no longer present externally from the interal
@@ -7030,8 +7031,6 @@ update_ip_config (NMDevice *self, gboolean initial)
 		linklocal6_just_completed = priv->linklocal6_timeout_id &&
 		                            have_ip6_address (priv->ext_ip6_config, TRUE);
 
-		ensure_con_ipx_config (self);
-
 		/* This function was called upon external changes. Remove the configuration
 		 * (adresses,routes) that is no longer present externally from the interal
 		 * config. This way, we don't readd addresses that were manually removed