From f6b7e49e3f9ea079f9429707d72ea983f58b89b6 Mon Sep 17 00:00:00 2001
From: Liu Jiang <gerry@linux.alibaba.com>
Date: Wed, 11 Aug 2021 20:06:22 +0800
Subject: [PATCH] handler: send exit event when destructing

VhostUserHandler waits for all working thread to exit in drop(), but
there's no mechanism to notify the working threads to exit. So add
VhostUserHandler::send_exit_event() to notify working threads to exit.

Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
---
 src/event_loop.rs | 59 +++++++++++++++++++++++++++++------------------
 src/handler.rs    | 13 +++++++++--
 2 files changed, 48 insertions(+), 24 deletions(-)

diff --git a/src/event_loop.rs b/src/event_loop.rs
index dcb2227..d8125f8 100644
--- a/src/event_loop.rs
+++ b/src/event_loop.rs
@@ -10,6 +10,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::result;
 
 use vm_memory::bitmap::Bitmap;
+use vmm_sys_util::eventfd::EventFd;
 
 use super::{VhostUserBackend, Vring, GM};
 
@@ -60,6 +61,7 @@ pub struct VringEpollHandler<S: VhostUserBackend<B>, B: Bitmap + 'static> {
     backend: S,
     vrings: Vec<Vring<GM<B>>>,
     thread_id: usize,
+    exit_event_fd: Option<EventFd>,
     exit_event_id: Option<u16>,
 }
 
@@ -72,33 +74,46 @@ impl<S: VhostUserBackend<B>, B: Bitmap + 'static> VringEpollHandler<S, B> {
     ) -> VringEpollResult<Self> {
         let epoll_fd = epoll::create(true).map_err(VringEpollError::EpollCreateFd)?;
         let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
-        let (exit_event_fd, exit_event_id) = match backend.exit_event(thread_id) {
-            Some((exit_event_fd, exit_event_id)) => {
-                (exit_event_fd.as_raw_fd(), Some(exit_event_id))
-            }
-            None => (-1, None),
-        };
-        let handler = VringEpollHandler {
-            epoll_file,
-            backend,
-            vrings,
-            thread_id,
-            exit_event_id,
-        };
 
-        if let Some(exit_event_id) = exit_event_id {
-            epoll::ctl(
-                handler.epoll_file.as_raw_fd(),
-                epoll::ControlOptions::EPOLL_CTL_ADD,
-                exit_event_fd,
-                epoll::Event::new(epoll::Events::EPOLLIN, u64::from(exit_event_id)),
-            )
-            .map_err(VringEpollError::RegisterExitEvent)?;
-        }
+        let handler = match backend.exit_event(thread_id) {
+            Some((exit_event_fd, exit_event_id)) => {
+                epoll::ctl(
+                    epoll_file.as_raw_fd(),
+                    epoll::ControlOptions::EPOLL_CTL_ADD,
+                    exit_event_fd.as_raw_fd(),
+                    epoll::Event::new(epoll::Events::EPOLLIN, u64::from(exit_event_id)),
+                )
+                .map_err(VringEpollError::RegisterExitEvent)?;
+
+                VringEpollHandler {
+                    epoll_file,
+                    backend,
+                    vrings,
+                    thread_id,
+                    exit_event_fd: Some(exit_event_fd),
+                    exit_event_id: Some(exit_event_id),
+                }
+            }
+            None => VringEpollHandler {
+                epoll_file,
+                backend,
+                vrings,
+                thread_id,
+                exit_event_fd: None,
+                exit_event_id: None,
+            },
+        };
 
         Ok(handler)
     }
 
+    /// Send `exit event` to break the event loop.
+    pub fn send_exit_event(&self) {
+        if let Some(eventfd) = self.exit_event_fd.as_ref() {
+            let _ = eventfd.write(1);
+        }
+    }
+
     /// Register an event into the epoll fd.
     ///
     /// When this event is later triggered, the backend implementation of `handle_event` will be
diff --git a/src/handler.rs b/src/handler.rs
index ad51a88..2bdb58d 100644
--- a/src/handler.rs
+++ b/src/handler.rs
@@ -132,11 +132,17 @@ impl<S: VhostUserBackend<B> + Clone, B: Bitmap + Clone + Send + Sync> VhostUserH
     }
 }
 
-impl<S: VhostUserBackend<B> + Clone, B: Bitmap> VhostUserHandler<S, B> {
+impl<S: VhostUserBackend<B>, B: Bitmap> VhostUserHandler<S, B> {
     pub(crate) fn get_epoll_handlers(&self) -> Vec<Arc<VringEpollHandler<S, B>>> {
         self.handlers.clone()
     }
 
+    pub(crate) fn send_exit_event(&self) {
+        for handler in self.handlers.iter() {
+            handler.send_exit_event();
+        }
+    }
+
     fn vmm_va_to_gpa(&self, vmm_va: u64) -> VhostUserHandlerResult<u64> {
         for mapping in self.mappings.iter() {
             if vmm_va >= mapping.vmm_addr && vmm_va < mapping.vmm_addr + mapping.size {
@@ -148,7 +154,7 @@ impl<S: VhostUserBackend<B> + Clone, B: Bitmap> VhostUserHandler<S, B> {
     }
 }
 
-impl<S: VhostUserBackend<B> + Clone, B: NewBitmap + Clone> VhostUserSlaveReqHandlerMut
+impl<S: VhostUserBackend<B>, B: NewBitmap + Clone> VhostUserSlaveReqHandlerMut
     for VhostUserHandler<S, B>
 {
     fn set_owner(&mut self) -> VhostUserResult<()> {
@@ -526,6 +532,9 @@ impl<S: VhostUserBackend<B> + Clone, B: NewBitmap + Clone> VhostUserSlaveReqHand
 
 impl<S: VhostUserBackend<B>, B: Bitmap> Drop for VhostUserHandler<S, B> {
     fn drop(&mut self) {
+        // Signal all working threads to exit.
+        self.send_exit_event();
+
         for thread in self.worker_threads.drain(..) {
             if let Err(e) = thread.join() {
                 error!("Error in vring worker: {:?}", e);