diff --git a/vhost-user-backend/CHANGELOG.md b/vhost-user-backend/CHANGELOG.md
index 3ff6065..4c3e5fc 100644
--- a/vhost-user-backend/CHANGELOG.md
+++ b/vhost-user-backend/CHANGELOG.md
@@ -5,6 +5,7 @@
 ### Added
 ### Changed
 - [[#308](https://github.com/rust-vmm/vhost/pull/308)] Replace Eventfd with EventNotifier/EventConsumer.
+- [[#321](https://github.com/rust-vmm/vhost/pull/321)] Don't take ownership of listener in `VhostUserDaemon::start`.
 
 ### Deprecated
 ### Fixed
diff --git a/vhost-user-backend/src/lib.rs b/vhost-user-backend/src/lib.rs
index 435511c..7e1db9e 100644
--- a/vhost-user-backend/src/lib.rs
+++ b/vhost-user-backend/src/lib.rs
@@ -162,9 +162,7 @@ where
     ///
     /// *Note:* A convenience function [VhostUserDaemon::serve] exists that
     /// may be a better option than this for simple use-cases.
-    // TODO: the current implementation has limitations that only one incoming connection will be
-    // handled from the listener. Should it be enhanced to support reconnection?
-    pub fn start(&mut self, listener: Listener) -> Result<()> {
+    pub fn start(&mut self, listener: &mut Listener) -> Result<()> {
         let mut backend_listener = BackendListener::new(listener, self.handler.clone())
             .map_err(Error::CreateBackendListener)?;
         let backend_handler = self.accept(&mut backend_listener)?;
@@ -215,9 +213,9 @@ where
     /// *Note:* See [VhostUserDaemon::start] and [VhostUserDaemon::wait] if you
     /// need more flexibility.
     pub fn serve<P: AsRef<Path>>(&mut self, socket: P) -> Result<()> {
-        let listener = Listener::new(socket, true).map_err(Error::CreateVhostUserListener)?;
+        let mut listener = Listener::new(socket, true).map_err(Error::CreateVhostUserListener)?;
 
-        self.start(listener)?;
+        self.start(&mut listener)?;
         let result = self.wait();
 
         // Regardless of the result, we want to signal worker threads to exit
@@ -279,9 +277,9 @@ mod tests {
                 drop(socket)
             });
 
-            let listener = Listener::new(&path, false).unwrap();
+            let mut listener = Listener::new(&path, false).unwrap();
             barrier.wait();
-            daemon.start(listener).unwrap();
+            daemon.start(&mut listener).unwrap();
             barrier.wait();
             // Above process generates a `HandleRequest(PartialMessage)` error.
             daemon.wait().unwrap_err();
diff --git a/vhost-user-backend/tests/vhost-user-server.rs b/vhost-user-backend/tests/vhost-user-server.rs
index 7c7e381..3c8205a 100644
--- a/vhost-user-backend/tests/vhost-user-server.rs
+++ b/vhost-user-backend/tests/vhost-user-server.rs
@@ -260,9 +260,9 @@ fn vhost_user_server_with_fn<F: FnOnce(Arc<Mutex<MockVhostBackend>>, Arc<Barrier
     let path1 = path.clone();
     let thread = thread::spawn(move || cb(&path1, barrier2));
 
-    let listener = Listener::new(&path, false).unwrap();
+    let mut listener = Listener::new(&path, false).unwrap();
     barrier.wait();
-    daemon.start(listener).unwrap();
+    daemon.start(&mut listener).unwrap();
     barrier.wait();
 
     server_fn(backend, barrier);
diff --git a/vhost/CHANGELOG.md b/vhost/CHANGELOG.md
index 8bc85b5..3eb54b8 100644
--- a/vhost/CHANGELOG.md
+++ b/vhost/CHANGELOG.md
@@ -7,6 +7,8 @@
   `From<UnixListener>` for `vhost_user::Listener`.
 
 ### Changed
+- [[#321](https://github.com/rust-vmm/vhost/pull/321)] Don't take ownership of listener in `BackendListener`.
+
 ### Deprecated
 ### Fixed
 - [[#304]](https://github.com/rust-vmm/vhost/pull/304) Fix building docs.
diff --git a/vhost/src/vhost_user/backend.rs b/vhost/src/vhost_user/backend.rs
index 8463e1a..e0a1d38 100644
--- a/vhost/src/vhost_user/backend.rs
+++ b/vhost/src/vhost_user/backend.rs
@@ -10,16 +10,16 @@ use super::message::*;
 use super::{BackendReqHandler, Result, VhostUserBackendReqHandler};
 
 /// Vhost-user backend side connection listener.
-pub struct BackendListener<S: VhostUserBackendReqHandler> {
-    listener: Listener,
+pub struct BackendListener<'a, S: VhostUserBackendReqHandler> {
+    listener: &'a mut Listener,
     backend: Option<Arc<S>>,
 }
 
 /// Sets up a listener for incoming frontend connections, and handles construction
 /// of a Backend on success.
-impl<S: VhostUserBackendReqHandler> BackendListener<S> {
+impl<'a, S: VhostUserBackendReqHandler> BackendListener<'a, S> {
     /// Create a unix domain socket for incoming frontend connections.
-    pub fn new(listener: Listener, backend: Arc<S>) -> Result<Self> {
+    pub fn new(listener: &'a mut Listener, backend: Arc<S>) -> Result<Self> {
         Ok(BackendListener {
             listener,
             backend: Some(backend),
@@ -55,9 +55,9 @@ mod tests {
     #[test]
     fn test_backend_listener_set_nonblocking() {
         let backend = Arc::new(Mutex::new(DummyBackendReqHandler::new()));
-        let listener =
+        let mut listener =
             Listener::new("/tmp/vhost_user_lib_unit_test_backend_nonblocking", true).unwrap();
-        let backend_listener = BackendListener::new(listener, backend).unwrap();
+        let backend_listener = BackendListener::new(&mut listener, backend).unwrap();
 
         backend_listener.set_nonblocking(true).unwrap();
         backend_listener.set_nonblocking(false).unwrap();
@@ -73,8 +73,8 @@ mod tests {
 
         let path = "/tmp/vhost_user_lib_unit_test_backend_accept";
         let backend = Arc::new(Mutex::new(DummyBackendReqHandler::new()));
-        let listener = Listener::new(path, true).unwrap();
-        let mut backend_listener = BackendListener::new(listener, backend).unwrap();
+        let mut listener = Listener::new(path, true).unwrap();
+        let mut backend_listener = BackendListener::new(&mut listener, backend).unwrap();
 
         backend_listener.set_nonblocking(true).unwrap();
         assert!(backend_listener.accept().unwrap().is_none());
diff --git a/vhost/src/vhost_user/mod.rs b/vhost/src/vhost_user/mod.rs
index c7c1ad3..0ecb686 100644
--- a/vhost/src/vhost_user/mod.rs
+++ b/vhost/src/vhost_user/mod.rs
@@ -293,8 +293,8 @@ mod tests {
         P: AsRef<Path>,
         S: VhostUserBackendReqHandler,
     {
-        let listener = Listener::new(&path, true).unwrap();
-        let mut backend_listener = BackendListener::new(listener, backend).unwrap();
+        let mut listener = Listener::new(&path, true).unwrap();
+        let mut backend_listener = BackendListener::new(&mut listener, backend).unwrap();
         let frontend = Frontend::connect(&path, 1).unwrap();
         (frontend, backend_listener.accept().unwrap().unwrap())
     }