From 18fd3e45d8db3428d4840d7cb9ea1980a22ef7de Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 27 Mar 2014 13:49:50 -0500 Subject: [PATCH 1/2] dcb: separate DCB enable/disable and wait for carrier changes (rh #799241) (rh #1081991) Non-git-master versions of lldpad refuse to touch a device that doesn't have a carrier. And when enabling/disabling DCB, the kernel driver will reconfigure itself and may turn carrier off for a few seconds. So we must ensure that before enabling/disabling DCB, the carrier is already on. Next we must ensure that *after* enabling/disabling DCB, the carrier is back on before doing further DCB setup. There's a race condition between enabling/disabling DCB and receiving the carrier event in NetworkManager that has to be handled carefully. Because the carrier may not yet be down after the dcbtool call to enable/disable DCB returns, we need to wait for a couple seconds for the carrier to go down, and then again for it to come back up. Otherwise we might see the still-on carrier, proceed with DCB setup, and the carrier finally goes down halfway through the setup, which will fail the operations with "DCB not enabled, link down, or DCB not supported" errors from lldpad. --- src/devices/nm-device-ethernet.c | 270 +++++++++++++++++++++++++++++-- src/nm-dcb.c | 38 +++-- src/nm-dcb.h | 7 + src/tests/test-dcb.c | 27 ++-- 4 files changed, 300 insertions(+), 42 deletions(-) diff --git a/src/devices/nm-device-ethernet.c b/src/devices/nm-device-ethernet.c index a6279a4cb5..e6f091d62c 100644 --- a/src/devices/nm-device-ethernet.c +++ b/src/devices/nm-device-ethernet.c @@ -86,6 +86,20 @@ typedef struct Supplicant { guint con_timeout_id; } Supplicant; +typedef enum { + DCB_WAIT_UNKNOWN = 0, + /* Ensure carrier is up before enabling DCB */ + DCB_WAIT_CARRIER_PREENABLE_UP, + /* Wait for carrier down when device starts enabling */ + DCB_WAIT_CARRIER_PRECONFIG_DOWN, + /* Wait for carrier up when device has finished enabling */ + DCB_WAIT_CARRIER_PRECONFIG_UP, + /* Wait carrier down when device starts configuring */ + DCB_WAIT_CARRIER_POSTCONFIG_DOWN, + /* Wait carrier up when device has finished configuring */ + DCB_WAIT_CARRIER_POSTCONFIG_UP, +} DcbWait; + typedef struct { guint8 perm_hw_addr[ETH_ALEN]; /* Permanent MAC address */ guint8 initial_hw_addr[ETH_ALEN]; /* Initial MAC address (as seen when NM starts) */ @@ -106,6 +120,11 @@ typedef struct { NMIP4Config *pending_ip4_config; gint32 last_pppoe_time; guint pppoe_wait_id; + + /* DCB */ + DcbWait dcb_wait; + guint dcb_timeout_id; + guint dcb_carrier_id; } NMDeviceEthernetPrivate; enum { @@ -1094,33 +1113,217 @@ pppoe_stage3_ip4_config_start (NMDeviceEthernet *self, NMDeviceStateReason *reas return ret; } +/****************************************************************/ + +static void +dcb_timeout_cleanup (NMDevice *device) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + + if (priv->dcb_timeout_id) { + g_source_remove (priv->dcb_timeout_id); + priv->dcb_timeout_id = 0; + } +} + +static void +dcb_carrier_cleanup (NMDevice *device) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + + if (priv->dcb_carrier_id) { + g_signal_handler_disconnect (device, priv->dcb_carrier_id); + priv->dcb_carrier_id = 0; + } +} + +static void dcb_state (NMDevice *device, gboolean timeout); + +static gboolean +dcb_carrier_timeout (gpointer user_data) +{ + NMDevice *device = NM_DEVICE (user_data); + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + + g_return_val_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG, G_SOURCE_REMOVE); + + priv->dcb_timeout_id = 0; + if (priv->dcb_wait != DCB_WAIT_CARRIER_POSTCONFIG_DOWN) { + nm_log_warn (LOGD_DCB, + "(%s): DCB: timed out waiting for carrier (step %d)", + nm_device_get_iface (device), + priv->dcb_wait); + } + dcb_state (device, TRUE); + return G_SOURCE_REMOVE; +} + +static gboolean +dcb_configure (NMDevice *device) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + NMSettingDcb *s_dcb; + const char *iface = nm_device_get_iface (device); + GError *error = NULL; + + dcb_timeout_cleanup (device); + + s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB); + g_assert (s_dcb); + if (!nm_dcb_setup (iface, s_dcb, &error)) { + nm_log_warn (LOGD_DCB, + "Activation (%s/wired) failed to enable DCB/FCoE: %s", + iface, error->message); + g_clear_error (&error); + return FALSE; + } + + /* Pause again just in case the device takes the carrier down when + * setting specific DCB attributes. + */ + nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (postconfig down)", iface); + priv->dcb_wait = DCB_WAIT_CARRIER_POSTCONFIG_DOWN; + priv->dcb_timeout_id = g_timeout_add_seconds (3, dcb_carrier_timeout, device); + return TRUE; +} + +static gboolean +dcb_enable (NMDevice *device) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + const char *iface = nm_device_get_iface (device); + GError *error = NULL; + + dcb_timeout_cleanup (device); + if (!nm_dcb_enable (iface, TRUE, &error)) { + nm_log_warn (LOGD_DCB, + "Activation (%s/wired) failed to enable DCB/FCoE: %s", + iface, error->message); + g_clear_error (&error); + return FALSE; + } + + /* Pause for 3 seconds after enabling DCB to let the card reconfigure + * itself. Drivers will often re-initialize internal settings which + * takes the carrier down for 2 or more seconds. During this time, + * lldpad will refuse to do anything else with the card since the carrier + * is down. But NM might get the carrier-down signal long after calling + * "dcbtool dcb on", so we have to first wait for the carrier to go down. + */ + nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preconfig down)", iface); + priv->dcb_wait = DCB_WAIT_CARRIER_PRECONFIG_DOWN; + priv->dcb_timeout_id = g_timeout_add_seconds (3, dcb_carrier_timeout, device); + return TRUE; +} + +static void +dcb_state (NMDevice *device, gboolean timeout) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + const char *iface = nm_device_get_iface (device); + gboolean carrier; + + g_return_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG); + + + carrier = nm_platform_link_is_connected (nm_device_get_ifindex (device)); + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() wait %d carrier %d timeout %d", iface, priv->dcb_wait, carrier, timeout); + + switch (priv->dcb_wait) { + case DCB_WAIT_CARRIER_PREENABLE_UP: + if (timeout || carrier) { + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() enabling DCB", iface); + dcb_timeout_cleanup (device); + if (!dcb_enable (device)) { + dcb_carrier_cleanup (device); + nm_device_state_changed (device, + NM_ACT_STAGE_RETURN_FAILURE, + NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED); + } + } + break; + case DCB_WAIT_CARRIER_PRECONFIG_DOWN: + dcb_timeout_cleanup (device); + priv->dcb_wait = DCB_WAIT_CARRIER_PRECONFIG_UP; + + if (!carrier) { + /* Wait for the carrier to come back up */ + nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preconfig up)", iface); + priv->dcb_timeout_id = g_timeout_add_seconds (5, dcb_carrier_timeout, device); + break; + } + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() preconfig down falling through", iface); + /* carrier never went down? fall through */ + case DCB_WAIT_CARRIER_PRECONFIG_UP: + if (timeout || carrier) { + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() preconfig up configuring DCB", iface); + dcb_timeout_cleanup (device); + if (!dcb_configure (device)) { + dcb_carrier_cleanup (device); + nm_device_state_changed (device, + NM_ACT_STAGE_RETURN_FAILURE, + NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED); + } + } + break; + case DCB_WAIT_CARRIER_POSTCONFIG_DOWN: + dcb_timeout_cleanup (device); + priv->dcb_wait = DCB_WAIT_CARRIER_POSTCONFIG_UP; + + if (!carrier) { + /* Wait for the carrier to come back up */ + nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (postconfig up)", iface); + priv->dcb_timeout_id = g_timeout_add_seconds (5, dcb_carrier_timeout, device); + break; + } + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() postconfig down falling through", iface); + /* carrier never went down? fall through */ + case DCB_WAIT_CARRIER_POSTCONFIG_UP: + if (timeout || carrier) { + nm_log_dbg (LOGD_DCB, "(%s): dcb_state() postconfig up starting IP", iface); + dcb_timeout_cleanup (device); + dcb_carrier_cleanup (device); + priv->dcb_wait = DCB_WAIT_UNKNOWN; + nm_device_activate_schedule_stage3_ip_config_start (device); + } + break; + default: + g_assert_not_reached (); + } +} + +static void +dcb_carrier_changed (NMDevice *device, GParamSpec *pspec, gpointer unused) +{ + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); + + g_return_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG); + + if (priv->dcb_timeout_id) { + nm_log_dbg (LOGD_DCB, "(%s): carrier_changed() calling dcb_state()", nm_device_get_iface (device)); + dcb_state (device, FALSE); + } +} + +/****************************************************************/ + static NMActStageReturn act_stage2_config (NMDevice *device, NMDeviceStateReason *reason) { + NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device); NMSettingConnection *s_con; const char *connection_type; NMActStageReturn ret = NM_ACT_STAGE_RETURN_SUCCESS; NMSettingDcb *s_dcb; - GError *error = NULL; g_return_val_if_fail (reason != NULL, NM_ACT_STAGE_RETURN_FAILURE); - /* DCB and FCoE setup */ - s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB); - if (s_dcb) { - if (!nm_dcb_setup (nm_device_get_iface (device), s_dcb, &error)) { - nm_log_warn (LOGD_DEVICE | LOGD_HW, - "Activation (%s/wired) failed to enable DCB/FCoE: %s", - nm_device_get_iface (device), error->message); - g_clear_error (&error); - *reason = NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED; - return NM_ACT_STAGE_RETURN_FAILURE; - } - } - s_con = NM_SETTING_CONNECTION (device_get_setting (device, NM_TYPE_SETTING_CONNECTION)); g_assert (s_con); + dcb_timeout_cleanup (device); + dcb_carrier_cleanup (device); + /* 802.1x has to run before any IP configuration since the 802.1x auth * process opens the port up for normal traffic. */ @@ -1129,8 +1332,36 @@ act_stage2_config (NMDevice *device, NMDeviceStateReason *reason) NMSetting8021x *security; security = (NMSetting8021x *) device_get_setting (device, NM_TYPE_SETTING_802_1X); - if (security) - ret = nm_8021x_stage2_config (NM_DEVICE_ETHERNET (device), reason); + if (security) { + /* FIXME: for now 802.1x is mutually exclusive with DCB */ + return nm_8021x_stage2_config (NM_DEVICE_ETHERNET (device), reason); + } + } + + /* DCB and FCoE setup */ + s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB); + if (s_dcb) { + /* lldpad really really wants the carrier to be up */ + if (nm_platform_link_is_connected (nm_device_get_ifindex (device))) { + if (!dcb_enable (device)) { + *reason = NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED; + return NM_ACT_STAGE_RETURN_FAILURE; + } + } else { + nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preenable up)", + nm_device_get_iface (device)); + priv->dcb_wait = DCB_WAIT_CARRIER_PREENABLE_UP; + priv->dcb_timeout_id = g_timeout_add_seconds (4, dcb_carrier_timeout, device); + } + + /* Watch carrier independently of NMDeviceClass::carrier_changed so + * we get instant notifications of disconnection that aren't deferred. + */ + priv->dcb_carrier_id = g_signal_connect (device, + "notify::" NM_DEVICE_CARRIER, + G_CALLBACK (dcb_carrier_changed), + NULL); + ret = NM_ACT_STAGE_RETURN_POSTPONE; } return ret; @@ -1206,6 +1437,10 @@ deactivate (NMDevice *device) supplicant_interface_release (self); + priv->dcb_wait = DCB_WAIT_UNKNOWN; + dcb_timeout_cleanup (device); + dcb_carrier_cleanup (device); + /* Tear down DCB/FCoE if it was enabled */ s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB); if (s_dcb) { @@ -1409,6 +1644,9 @@ dispose (GObject *object) priv->pppoe_wait_id = 0; } + dcb_timeout_cleanup (NM_DEVICE (self)); + dcb_carrier_cleanup (NM_DEVICE (self)); + G_OBJECT_CLASS (nm_device_ethernet_parent_class)->dispose (object); } diff --git a/src/nm-dcb.c b/src/nm-dcb.c index 5e25cd84e6..428c61e67d 100644 --- a/src/nm-dcb.c +++ b/src/nm-dcb.c @@ -92,6 +92,19 @@ out: return success; } +gboolean +_dcb_enable (const char *iface, + gboolean enable, + DcbFunc run_func, + gpointer user_data, + GError **error) +{ + if (enable) + return do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb on"); + else + return do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb off"); +} + #define SET_FLAGS(f, tag) \ G_STMT_START { \ if (!do_helper (iface, DCBTOOL, run_func, user_data, error, tag " e:%c a:%c w:%c", \ @@ -124,9 +137,6 @@ _dcb_setup (const char *iface, g_assert (s_dcb); - if (!do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb on")) - return FALSE; - /* FCoE */ flags = nm_setting_dcb_get_app_fcoe_flags (s_dcb); SET_APP (flags, s_dcb, fcoe); @@ -229,7 +239,6 @@ _dcb_cleanup (const char *iface, GError **error) { const char *cmds[] = { - "dcb off", "app:fcoe e:0", "app:iscsi e:0", "app:fip e:0", @@ -247,6 +256,9 @@ _dcb_cleanup (const char *iface, iter++; } + if (!_dcb_enable (iface, FALSE, run_func, user_data, success ? error : NULL)) + success = FALSE; + return success; } @@ -354,6 +366,12 @@ run_helper (char **argv, guint which, gpointer user_data, GError **error) return success; } +gboolean +nm_dcb_enable (const char *iface, gboolean enable, GError **error) +{ + return _dcb_enable (iface, enable, run_helper, GUINT_TO_POINTER (DCBTOOL), error); +} + gboolean nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error) { @@ -369,14 +387,8 @@ nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error) gboolean nm_dcb_cleanup (const char *iface, GError **error) { - gboolean success; - - success = _dcb_cleanup (iface, run_helper, GUINT_TO_POINTER (DCBTOOL), error); - if (success) { - /* Only report FCoE errors if DCB cleanup was successful */ - success = _fcoe_cleanup (iface, run_helper, GUINT_TO_POINTER (FCOEADM), success ? error : NULL); - } - - return success; + /* Ignore FCoE cleanup errors */ + _fcoe_cleanup (iface, run_helper, GUINT_TO_POINTER (FCOEADM), NULL); + return _dcb_cleanup (iface, run_helper, GUINT_TO_POINTER (DCBTOOL), error); } diff --git a/src/nm-dcb.h b/src/nm-dcb.h index 3dbd5e6f4f..bfe5cedffb 100644 --- a/src/nm-dcb.h +++ b/src/nm-dcb.h @@ -48,6 +48,7 @@ GQuark nm_dcb_error_quark (void); GType nm_dcb_error_get_type (void); +gboolean nm_dcb_enable (const char *iface, gboolean enable, GError **error); gboolean nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error); gboolean nm_dcb_cleanup (const char *iface, GError **error); @@ -68,6 +69,12 @@ gboolean do_helper (const char *iface, const char *fmt, ...) G_GNUC_PRINTF(6, 7); +gboolean _dcb_enable (const char *iface, + gboolean enable, + DcbFunc run_func, + gpointer user_data, + GError **error); + gboolean _dcb_setup (const char *iface, NMSettingDcb *s_dcb, DcbFunc run_func, diff --git a/src/tests/test-dcb.c b/src/tests/test-dcb.c index 74dcca5de1..aead8f186c 100644 --- a/src/tests/test-dcb.c +++ b/src/tests/test-dcb.c @@ -54,8 +54,7 @@ static void test_dcb_fcoe (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:1 a:1 w:1", + { "dcbtool sc eth0 app:fcoe e:1 a:1 w:1", "dcbtool sc eth0 app:fcoe appcfg:40", "dcbtool sc eth0 app:iscsi e:0 a:0 w:0", "dcbtool sc eth0 app:fip e:0 a:0 w:0", @@ -85,8 +84,7 @@ static void test_dcb_iscsi (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", + { "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", "dcbtool sc eth0 app:iscsi e:1 a:0 w:1", "dcbtool sc eth0 app:iscsi appcfg:08", "dcbtool sc eth0 app:fip e:0 a:0 w:0", @@ -116,8 +114,7 @@ static void test_dcb_fip (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", + { "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", "dcbtool sc eth0 app:iscsi e:0 a:0 w:0", "dcbtool sc eth0 app:fip e:1 a:1 w:0", "dcbtool sc eth0 app:fip appcfg:01", @@ -147,8 +144,7 @@ static void test_dcb_fip_default_prio (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", + { "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", "dcbtool sc eth0 app:iscsi e:0 a:0 w:0", "dcbtool sc eth0 app:fip e:1 a:1 w:0", "dcbtool sc eth0 pfc e:0 a:0 w:0", @@ -177,8 +173,7 @@ static void test_dcb_pfc (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", + { "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", "dcbtool sc eth0 app:iscsi e:0 a:0 w:0", "dcbtool sc eth0 app:fip e:0 a:0 w:0", "dcbtool sc eth0 pfc e:1 a:1 w:1", @@ -216,8 +211,7 @@ static void test_dcb_priority_groups (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb on", - "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", + { "dcbtool sc eth0 app:fcoe e:0 a:0 w:0", "dcbtool sc eth0 app:iscsi e:0 a:0 w:0", "dcbtool sc eth0 app:fip e:0 a:0 w:0", "dcbtool sc eth0 pfc e:0 a:0 w:0", @@ -268,20 +262,27 @@ static void test_dcb_cleanup (void) { static DcbExpected expected = { 0, - { "dcbtool sc eth0 dcb off", + { "fcoeadm -d eth0", "dcbtool sc eth0 app:fcoe e:0", "dcbtool sc eth0 app:iscsi e:0", "dcbtool sc eth0 app:fip e:0", "dcbtool sc eth0 pfc e:0", "dcbtool sc eth0 pg e:0", + "dcbtool sc eth0 dcb off", NULL }, }; GError *error = NULL; gboolean success; + success = _fcoe_cleanup ("eth0", test_dcb_func, &expected, &error); + g_assert_no_error (error); + g_assert (success); + success = _dcb_cleanup ("eth0", test_dcb_func, &expected, &error); g_assert_no_error (error); g_assert (success); + + g_assert_cmpstr (expected.cmds[expected.num], ==, NULL); } static void From 55704170caed97eb26b3853af04cbbf332845966 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 31 Mar 2014 15:06:22 -0500 Subject: [PATCH 2/2] dcb: wait for carrier down/up after disabling FCoE --- src/nm-dcb.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/nm-dcb.c b/src/nm-dcb.c index 428c61e67d..3bdbf7d5ed 100644 --- a/src/nm-dcb.c +++ b/src/nm-dcb.c @@ -384,11 +384,43 @@ nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error) return success; } +static void +carrier_wait (const char *iface, guint secs, gboolean up) +{ + int ifindex, count = secs * 10; + + g_return_if_fail (iface != NULL); + + ifindex = nm_platform_link_get_ifindex (iface); + if (ifindex > 0) { + /* To work around driver quirks and lldpad handling of carrier status, + * we must wait a short period of time to see if the carrier goes + * down, and then wait for the carrier to come back up again. Otherwise + * subsequent lldpad calls may fail with "Device not found, link down + * or DCB not enabled" errors. + */ + nm_log_dbg (LOGD_DCB, "(%s): cleanup waiting for carrier %s", + iface, up ? "up" : "down"); + g_usleep (G_USEC_PER_SEC / 4); + while (nm_platform_link_is_connected (ifindex) != up && count-- > 0) { + g_usleep (G_USEC_PER_SEC / 10); + nm_platform_link_refresh (ifindex); + } + } +} + gboolean nm_dcb_cleanup (const char *iface, GError **error) { /* Ignore FCoE cleanup errors */ _fcoe_cleanup (iface, run_helper, GUINT_TO_POINTER (FCOEADM), NULL); + + /* Must pause a bit to wait for carrier-up since disabling FCoE may + * cause the device to take the link down, making lldpad return errors. + */ + carrier_wait (iface, 2, FALSE); + carrier_wait (iface, 4, TRUE); + return _dcb_cleanup (iface, run_helper, GUINT_TO_POINTER (DCBTOOL), error); }