Fix driver removal upon device removal

The driver itself shouldn't be removed but only the device specific
reference to it.
diff --git a/src/device.c b/src/device.c
index 407611a..ba80220 100644
--- a/src/device.c
+++ b/src/device.c
@@ -436,16 +436,8 @@
 	struct btd_device_driver *driver = driver_data->driver;
 
 	driver->remove(device);
-}
 
-static void driver_free(struct btd_driver_data *driver_data,
-					struct btd_device *device)
-{
-	struct btd_device_driver *driver = driver_data->driver;
-
-	driver->remove(device);
-
-	g_free(driver);
+	device->drivers = g_slist_remove(device->drivers, driver_data);
 	g_free(driver_data);
 }
 
@@ -1134,7 +1126,7 @@
 	if (remove_stored)
 		device_remove_stored(device);
 
-	g_slist_foreach(device->drivers, (GFunc) driver_free, device);
+	g_slist_foreach(device->drivers, (GFunc) driver_remove, device);
 	g_slist_free(device->drivers);
 	device->drivers = NULL;