diff --git a/drivers/usb/gadget/ci_udc.c b/drivers/usb/gadget/ci_udc.c
index 05c01ce5d6..e360f8e2c8 100644
--- a/drivers/usb/gadget/ci_udc.c
+++ b/drivers/usb/gadget/ci_udc.c
@@ -1044,6 +1044,13 @@ int usb_gadget_unregister_driver(struct usb_gadget_driver *driver)
 	free(controller.items_mem);
 	free(controller.epts);
 
+#ifdef CONFIG_DM_USB
+	usb_remove_ehci_gadget(&controller.ctrl);
+#else
+	usb_lowlevel_stop(0);
+	controller.ctrl = NULL;
+#endif
+
 	return 0;
 }
 
diff --git a/drivers/usb/host/usb-uclass.c b/drivers/usb/host/usb-uclass.c
index 4e40f4bc3d..2979212c9f 100644
--- a/drivers/usb/host/usb-uclass.c
+++ b/drivers/usb/host/usb-uclass.c
@@ -405,6 +405,24 @@ int usb_setup_ehci_gadget(struct ehci_ctrl **ctlrp)
 	return 0;
 }
 
+int usb_remove_ehci_gadget(struct ehci_ctrl **ctlrp)
+{
+	struct udevice *dev;
+	int ret;
+
+	/* Find the old device and remove it */
+	ret = uclass_find_device_by_seq(UCLASS_USB, 0, true, &dev);
+	if (ret)
+		return ret;
+	ret = device_remove(dev, DM_REMOVE_NORMAL);
+	if (ret)
+		return ret;
+
+	*ctlrp = NULL;
+
+	return 0;
+}
+
 /* returns 0 if no match, 1 if match */
 static int usb_match_device(const struct usb_device_descriptor *desc,
 			    const struct usb_device_id *id)
diff --git a/include/usb.h b/include/usb.h
index 57a7d8d0ea..279bf5555e 100644
--- a/include/usb.h
+++ b/include/usb.h
@@ -905,6 +905,15 @@ struct ehci_ctrl;
  */
 int usb_setup_ehci_gadget(struct ehci_ctrl **ctlrp);
 
+/**
+ * usb_remove_ehci_gadget() - Remove a gadget USB device
+ *
+ * TODO(sjg@chromium.org): Tidy this up when USB gadgets can use driver model
+ *
+ * This provides a way to tell a controller to remove a USB device
+ */
+int usb_remove_ehci_gadget(struct ehci_ctrl **ctlrp);
+
 /**
  * usb_stor_reset() - Prepare to scan USB storage devices
  *