diff --git a/drivers/usb/serial/console.c b/drivers/usb/serial/console.c
index 72ab3bbf0f2..04007c31d88 100644
--- a/drivers/usb/serial/console.c
+++ b/drivers/usb/serial/console.c
@@ -192,8 +192,9 @@ static int usb_console_setup(struct console *co, char *options)
 		kfree (termios);
 		kfree (tty);
 	}
+	port->console = 1;
 
-	return retval;
+	return 0;
 }
 
 static void usb_console_write(struct console *co, const char *buf, unsigned count)
diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c
index 497e29a700c..5c33e2471be 100644
--- a/drivers/usb/serial/usb-serial.c
+++ b/drivers/usb/serial/usb-serial.c
@@ -264,20 +264,22 @@ static void serial_close(struct tty_struct *tty, struct file * filp)
 	}
 
 	--port->open_count;
-	if (port->open_count == 0) {
+	if (port->open_count == 0)
 		/* only call the device specific close if this 
 		 * port is being closed by the last owner */
 		port->serial->type->close(port, filp);
 
+	if (port->open_count == (port->console? 1 : 0)) {
 		if (port->tty) {
 			if (port->tty->driver_data)
 				port->tty->driver_data = NULL;
 			port->tty = NULL;
 		}
-
-		module_put(port->serial->type->driver.owner);
 	}
 
+	if (port->open_count == 0)
+		module_put(port->serial->type->driver.owner);
+
 	mutex_unlock(&port->mutex);
 	usb_serial_put(port->serial);
 }
diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h
index 488ce128885..ef1e430f7bf 100644
--- a/include/linux/usb/serial.h
+++ b/include/linux/usb/serial.h
@@ -92,6 +92,7 @@ struct usb_serial_port {
 	int			open_count;
 	char			throttled;
 	char			throttle_req;
+	char			console;
 	struct device		dev;
 };
 #define to_usb_serial_port(d) container_of(d, struct usb_serial_port, dev)