From 8f3462b3e356f56fee33ae4ba833a1fef012c1f0 Mon Sep 17 00:00:00 2001
From: Bo Chen <chen.bo@intel.com>
Date: Tue, 16 Aug 2022 12:03:45 -0700
Subject: [PATCH] tests: live_migration: Run most tests in parallel

Only the ovs-dpdk live-migration tests need to run sequentially as they
use the same ovs-dpdk setup.

This is to reduce our CI time, particularly for the live-migration
and aarch64 jobs.

Signed-off-by: Bo Chen <chen.bo@intel.com>
---
 scripts/run_integration_tests_aarch64.sh      |   9 +-
 .../run_integration_tests_live_migration.sh   |  10 +-
 tests/integration.rs                          | 233 +++++++++---------
 3 files changed, 138 insertions(+), 114 deletions(-)

diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh
index 7394a36e4..36f9dcb93 100755
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -272,7 +272,14 @@ fi
 
 # Run all test cases related to live migration
 if [ $RES -eq 0 ]; then
-    time cargo test $features "live_migration::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    RES=$?
+else
+    exit $RES
+fi
+
+if [ $RES -eq 0 ]; then
+    time cargo test $features "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
     RES=$?
 else
     exit $RES
diff --git a/scripts/run_integration_tests_live_migration.sh b/scripts/run_integration_tests_live_migration.sh
index af3f05ab7..627cd94ec 100755
--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -96,7 +96,15 @@ echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages
 
 export RUST_BACKTRACE=1
-time cargo test $features "live_migration::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+time cargo test $features "live_migration_parallel::$test_filter" -- ${test_binary_args[*]}
 RES=$?
 
+# Run some tests in sequence since the result could be affected by other tests
+# running in parallel.
+if [ $RES -eq 0 ]; then
+    export RUST_BACKTRACE=1
+    time cargo test $features "live_migration_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+    RES=$?
+fi
+
 exit $RES
diff --git a/tests/integration.rs b/tests/integration.rs
index fb8e4ff5d..21db0e148 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -8680,98 +8680,6 @@ mod live_migration {
         handle_child_output(r, &dest_output);
     }
 
-    #[test]
-    fn test_live_migration_basic() {
-        _test_live_migration(false, false, false, false, false)
-    }
-
-    #[test]
-    fn test_live_migration_local() {
-        _test_live_migration(false, false, true, false, false)
-    }
-
-    #[test]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_migration_numa() {
-        _test_live_migration(false, true, false, false, false)
-    }
-
-    #[test]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_migration_numa_local() {
-        _test_live_migration(false, true, true, false, false)
-    }
-
-    #[test]
-    fn test_live_migration_watchdog() {
-        _test_live_migration(false, false, false, true, false)
-    }
-
-    #[test]
-    fn test_live_migration_watchdog_local() {
-        _test_live_migration(false, false, true, true, false)
-    }
-
-    #[test]
-    fn test_live_migration_balloon() {
-        _test_live_migration(false, false, false, false, true)
-    }
-
-    #[test]
-    fn test_live_migration_balloon_local() {
-        _test_live_migration(false, false, true, false, true)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_basic() {
-        _test_live_migration(true, false, false, false, false)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_local() {
-        _test_live_migration(true, false, true, false, false)
-    }
-
-    #[test]
-    #[ignore]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_upgrade_numa() {
-        _test_live_migration(true, true, false, false, false)
-    }
-
-    #[test]
-    #[ignore]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_upgrade_numa_local() {
-        _test_live_migration(true, true, true, false, false)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_watchdog() {
-        _test_live_migration(true, false, false, true, false)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_watchdog_local() {
-        _test_live_migration(true, false, true, true, false)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_balloon() {
-        _test_live_migration(true, false, false, false, true)
-    }
-
-    #[test]
-    #[ignore]
-    fn test_live_upgrade_balloon_local() {
-        _test_live_migration(true, false, true, false, true)
-    }
-
     fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) {
         let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
         let ovs_guest = Guest::new(Box::new(ovs_focal));
@@ -8974,30 +8882,131 @@ mod live_migration {
         cleanup_ovs_dpdk();
     }
 
-    #[test]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_migration_ovs_dpdk() {
-        _test_live_migration_ovs_dpdk(false, false);
+    mod live_migration_parallel {
+        use super::*;
+        #[test]
+        fn test_live_migration_basic() {
+            _test_live_migration(false, false, false, false, false)
+        }
+
+        #[test]
+        fn test_live_migration_local() {
+            _test_live_migration(false, false, true, false, false)
+        }
+
+        #[test]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_migration_numa() {
+            _test_live_migration(false, true, false, false, false)
+        }
+
+        #[test]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_migration_numa_local() {
+            _test_live_migration(false, true, true, false, false)
+        }
+
+        #[test]
+        fn test_live_migration_watchdog() {
+            _test_live_migration(false, false, false, true, false)
+        }
+
+        #[test]
+        fn test_live_migration_watchdog_local() {
+            _test_live_migration(false, false, true, true, false)
+        }
+
+        #[test]
+        fn test_live_migration_balloon() {
+            _test_live_migration(false, false, false, false, true)
+        }
+
+        #[test]
+        fn test_live_migration_balloon_local() {
+            _test_live_migration(false, false, true, false, true)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_basic() {
+            _test_live_migration(true, false, false, false, false)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_local() {
+            _test_live_migration(true, false, true, false, false)
+        }
+
+        #[test]
+        #[ignore]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_upgrade_numa() {
+            _test_live_migration(true, true, false, false, false)
+        }
+
+        #[test]
+        #[ignore]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_upgrade_numa_local() {
+            _test_live_migration(true, true, true, false, false)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_watchdog() {
+            _test_live_migration(true, false, false, true, false)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_watchdog_local() {
+            _test_live_migration(true, false, true, true, false)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_balloon() {
+            _test_live_migration(true, false, false, false, true)
+        }
+
+        #[test]
+        #[ignore]
+        fn test_live_upgrade_balloon_local() {
+            _test_live_migration(true, false, true, false, true)
+        }
     }
 
-    #[test]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_migration_ovs_dpdk_local() {
-        _test_live_migration_ovs_dpdk(false, true);
-    }
+    mod live_migration_sequential {
+        #[cfg(not(feature = "mshv"))]
+        use super::*;
 
-    #[test]
-    #[ignore]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_upgrade_ovs_dpdk() {
-        _test_live_migration_ovs_dpdk(true, false);
-    }
+        // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup
+        #[test]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_migration_ovs_dpdk() {
+            _test_live_migration_ovs_dpdk(false, false);
+        }
 
-    #[test]
-    #[ignore]
-    #[cfg(not(feature = "mshv"))]
-    fn test_live_upgrade_ovs_dpdk_local() {
-        _test_live_migration_ovs_dpdk(true, true);
+        #[test]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_migration_ovs_dpdk_local() {
+            _test_live_migration_ovs_dpdk(false, true);
+        }
+
+        #[test]
+        #[ignore]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_upgrade_ovs_dpdk() {
+            _test_live_migration_ovs_dpdk(true, false);
+        }
+
+        #[test]
+        #[ignore]
+        #[cfg(not(feature = "mshv"))]
+        fn test_live_upgrade_ovs_dpdk_local() {
+            _test_live_migration_ovs_dpdk(true, true);
+        }
     }
 }