314f42f88f
Try to figure out if access to KVM is granted via use of supplemental user groups and warn users that they need to use 'crun' runtime on such systems. TEST=tools/dev_container Change-Id: I7d1ff348898e92ed9a90f0293e15a33eb4403e3f Reviewed-on: https://chromium-review.googlesource.com/c/crosvm/crosvm/+/6573234 Commit-Queue: Dmitry Torokhov <dtor@chromium.org> Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
416 lines
14 KiB
Python
Executable file
416 lines
14 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# Copyright 2021 The ChromiumOS Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
# Usage:
|
|
#
|
|
# To get an interactive shell for development:
|
|
# ./tools/dev_container
|
|
#
|
|
# To run a command in the container, e.g. to run presubmits:
|
|
# ./tools/dev_container ./tools/presubmit
|
|
#
|
|
# The state of the container (including build artifacts) are preserved between
|
|
# calls. To stop the container call:
|
|
# ./tools/dev_container --stop
|
|
#
|
|
# The dev container can also be called with a fresh container for each call that
|
|
# is cleaned up afterwards (e.g. when run by Kokoro):
|
|
#
|
|
# ./tools/dev_container --hermetic CMD
|
|
#
|
|
# There's an alternative container which can be used to test crosvm in crOS tree.
|
|
# It can be launched with:
|
|
# ./tools/dev_container --cros
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
import shutil
|
|
import stat
|
|
from impl.util import (
|
|
add_common_args,
|
|
confirm,
|
|
cros_repo_root,
|
|
CROSVM_ROOT,
|
|
is_cros_repo,
|
|
is_kiwi_repo,
|
|
kiwi_repo_root,
|
|
is_aosp_repo,
|
|
aosp_repo_root,
|
|
)
|
|
from impl.command import (
|
|
chdir,
|
|
cmd,
|
|
quoted,
|
|
)
|
|
from typing import Optional, List
|
|
import getpass
|
|
import sys
|
|
import unittest
|
|
import os
|
|
import zlib
|
|
|
|
DEV_CONTAINER_NAME = (
|
|
f"crosvm_dev_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}"
|
|
)
|
|
CROS_CONTAINER_NAME = (
|
|
f"crosvm_cros_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}"
|
|
)
|
|
|
|
DEV_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_dev"
|
|
CROS_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_cros_cloudbuild"
|
|
DEV_IMAGE_VERSION = (CROSVM_ROOT / "tools/impl/dev_container/version").read_text().strip()
|
|
|
|
CACHE_DIR = os.environ.get("CROSVM_CONTAINER_CACHE", None)
|
|
|
|
COMMON_ARGS = [
|
|
# Share cache dir
|
|
f"--volume {CACHE_DIR}:/cache:rw" if CACHE_DIR else None,
|
|
# Use tmpfs in the container for faster performance.
|
|
"--mount type=tmpfs,destination=/tmp",
|
|
# KVM is required to run a VM for testing.
|
|
"--device /dev/kvm" if Path("/dev/kvm").is_char_device() else None,
|
|
# Enable terminal colors
|
|
f"--env TERM={os.environ.get('TERM', 'xterm-256color')}",
|
|
]
|
|
|
|
DOCKER_ARGS = [
|
|
*COMMON_ARGS,
|
|
]
|
|
|
|
PODMAN_ARGS = [
|
|
*COMMON_ARGS,
|
|
# Allow access to group permissions of the user (e.g. for kvm access).
|
|
"--group-add keep-groups" if os.name == "posix" else None,
|
|
# Increase number of PIDs the container can spawn (we run a lot of test processes in parallel)
|
|
"--pids-limit=4096" if os.name == "posix" else None,
|
|
]
|
|
|
|
# Environment variables to pass through to the container if they are specified.
|
|
ENV_PASSTHROUGH = [
|
|
"NEXTEST_PROFILE",
|
|
"http_proxy",
|
|
"https_proxy",
|
|
]
|
|
|
|
|
|
def machine_is_running(docker: cmd):
|
|
machine_state = docker("machine info").stdout()
|
|
return "MachineState: Running" in machine_state
|
|
|
|
|
|
def container_name(cros: bool):
|
|
if cros:
|
|
return CROS_CONTAINER_NAME
|
|
else:
|
|
return DEV_CONTAINER_NAME
|
|
|
|
|
|
def container_revision(docker: cmd, container_id: str):
|
|
image = docker("container inspect -f {{.Config.Image}}", container_id).stdout()
|
|
parts = image.split(":")
|
|
assert len(parts) == 2, f"Invalid image name {image}"
|
|
return parts[1]
|
|
|
|
|
|
def container_id(docker: cmd, cros: bool):
|
|
return docker(f"ps -a -q -f name={container_name(cros)}").stdout()
|
|
|
|
|
|
def container_is_running(docker: cmd, cros: bool):
|
|
return bool(docker(f"ps -q -f name={container_name(cros)}").stdout())
|
|
|
|
|
|
def delete_container(docker: cmd, cros: bool):
|
|
cid = container_id(docker, cros)
|
|
if cid:
|
|
print(f"Deleting dev-container {cid}.")
|
|
docker("rm -f", cid).fg(quiet=True)
|
|
return True
|
|
return False
|
|
|
|
|
|
def workspace_mount_args(cros: bool):
|
|
"""
|
|
Returns arguments for mounting the crosvm sources to /workspace.
|
|
|
|
In ChromeOS checkouts the crosvm repo uses a symlink or worktree checkout, which links to a
|
|
different folder in the ChromeOS checkout. So we need to mount the whole CrOS checkout.
|
|
"""
|
|
if cros:
|
|
return ["--workdir /home/crosvmdev/chromiumos/src/platform/crosvm"]
|
|
elif is_cros_repo():
|
|
return [
|
|
f"--volume {quoted(cros_repo_root())}:/workspace:rw",
|
|
"--workdir /workspace/src/platform/crosvm",
|
|
]
|
|
elif is_kiwi_repo():
|
|
return [
|
|
f"--volume {quoted(kiwi_repo_root())}:/workspace:rw",
|
|
# We override /scratch because we run out of memory if we use memory to back the
|
|
# `/scratch` mount point.
|
|
f"--volume {quoted(kiwi_repo_root())}/scratch:/scratch/cargo_target:rw",
|
|
"--workdir /workspace/platform/crosvm",
|
|
]
|
|
elif is_aosp_repo():
|
|
return [
|
|
f"--volume {quoted(aosp_repo_root())}:/workspace:rw",
|
|
"--workdir /workspace/external/crosvm",
|
|
]
|
|
else:
|
|
return [
|
|
f"--volume {quoted(CROSVM_ROOT)}:/workspace:rw",
|
|
]
|
|
|
|
|
|
def ensure_container_is_alive(docker: cmd, docker_args: List[Optional[str]], cros: bool):
|
|
cid = container_id(docker, cros)
|
|
if cid and not container_is_running(docker, cros):
|
|
print("Existing container is not running.")
|
|
delete_container(docker, cros)
|
|
elif cid and not cros and container_revision(docker, cid) != DEV_IMAGE_VERSION:
|
|
print(f"New image is available.")
|
|
delete_container(docker, cros)
|
|
|
|
if not container_is_running(docker, cros):
|
|
# Run neverending sleep to keep container alive while we 'docker exec' commands.
|
|
print(f"Starting container...")
|
|
docker(
|
|
f"run --detach --name {container_name(cros)}",
|
|
*docker_args,
|
|
"sleep infinity",
|
|
).fg(quiet=False)
|
|
cid = container_id(docker, cros)
|
|
else:
|
|
cid = container_id(docker, cros)
|
|
print(f"Using existing container ({cid}).")
|
|
return cid
|
|
|
|
|
|
def access_needs_supplemental_group(file_path: str, write=True):
|
|
"""
|
|
Checks if access to a given resource/file is granted through user's supplemental
|
|
group as opposed to ownership or world access.
|
|
|
|
Returns True if and only if access is permitted and is granted through membership
|
|
in a supplemental group. If access is permitted by other means or no access
|
|
granted returns False.
|
|
|
|
This function does not consider superuser case.
|
|
"""
|
|
if not os.access(file_path, os.W_OK if write else os.R_OK, effective_ids=True):
|
|
return False
|
|
|
|
euid = os.geteuid()
|
|
egid = os.getegid()
|
|
|
|
stat_info = os.stat(file_path)
|
|
|
|
if euid == stat_info.st_uid:
|
|
return False # Access granted to owner
|
|
|
|
desired_mode = stat.S_IWGRP if write else stat.S_IRGRP
|
|
if not (stat_info.st_mode & desired_mode):
|
|
return False # Access is not through group ownership
|
|
|
|
if egid == stat_info.st_gid:
|
|
return False # Access granted through the primary group
|
|
|
|
# Assume access is granted through supplemental group.
|
|
return True
|
|
|
|
|
|
def validate_podman(podman: cmd):
|
|
graph_driver_name = podman("info --format={{.Store.GraphDriverName}}").stdout()
|
|
config_file_name = podman("info --format={{.Store.ConfigFile}}").stdout()
|
|
if graph_driver_name == "vfs":
|
|
print("You are using vfs as a storage driver. This will be extremely slow.")
|
|
print("Using the overlay driver is strongly recommended.")
|
|
print("Note: This will delete all existing podman images and containers.")
|
|
if confirm(f"Do you want me to update your config in {config_file_name}?"):
|
|
podman("system reset -f").fg()
|
|
with open(config_file_name, "a") as config_file:
|
|
print("[storage]", file=config_file)
|
|
print('driver = "overlay"', file=config_file)
|
|
|
|
if os.name == "posix":
|
|
username = os.environ["USER"]
|
|
subuids = Path("/etc/subuid").read_text()
|
|
if not username in subuids:
|
|
print("Rootless podman requires subuid's to be set up for your user.")
|
|
usermod = cmd(
|
|
"sudo usermod --add-subuids 900000-965535 --add-subgids 900000-965535", username
|
|
)
|
|
print("I can fix that by running:", usermod)
|
|
if confirm("Ok?"):
|
|
usermod.fg()
|
|
podman("system migrate").fg()
|
|
|
|
if access_needs_supplemental_group("/dev/kvm"):
|
|
runtime = podman("info --format={{.Host.OCIRuntime.Name}}").stdout()
|
|
if runtime != "crun":
|
|
print("On this system access to /dev/kvm relies on supplemental groups.")
|
|
print("This functionality ('keep-groups') is only available with 'crun'.")
|
|
print("Please switch podman to use 'crun' runtime.")
|
|
|
|
|
|
def main(argv: List[str]):
|
|
parser = argparse.ArgumentParser()
|
|
add_common_args(parser)
|
|
parser.add_argument("--stop", action="store_true")
|
|
parser.add_argument("--clean", action="store_true")
|
|
parser.add_argument("--hermetic", action="store_true")
|
|
parser.add_argument("--no-interactive", action="store_true")
|
|
parser.add_argument("--use-docker", action="store_true")
|
|
parser.add_argument("--self-test", action="store_true")
|
|
parser.add_argument("--pull", action="store_true")
|
|
parser.add_argument("--cros", action="store_true")
|
|
parser.add_argument("command", nargs=argparse.REMAINDER)
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
chdir(CROSVM_ROOT)
|
|
|
|
if CACHE_DIR:
|
|
Path(CACHE_DIR).mkdir(exist_ok=True)
|
|
|
|
has_docker = shutil.which("docker") != None
|
|
has_podman = shutil.which("podman") != None
|
|
if not has_podman and not has_docker:
|
|
raise Exception("Please install podman (or docker) to use the dev container.")
|
|
|
|
use_docker = args.use_docker
|
|
if has_docker and not has_podman:
|
|
use_docker = True
|
|
|
|
# cros container only works in docker
|
|
if args.cros:
|
|
use_docker = True
|
|
|
|
if use_docker:
|
|
print(
|
|
"WARNING: Running dev_container with docker may cause root-owned files to be created."
|
|
)
|
|
print("Use podman to prevent this.")
|
|
print()
|
|
docker = cmd("docker")
|
|
docker_args = [
|
|
*DOCKER_ARGS,
|
|
*workspace_mount_args(args.cros),
|
|
]
|
|
else:
|
|
docker = cmd("podman")
|
|
|
|
# On windows, podman uses wsl vm. start the default podman vm for the rest of the script
|
|
# to work properly.
|
|
if os.name == "nt" and not machine_is_running(docker):
|
|
print("Starting podman default machine.")
|
|
docker("machine start").fg(quiet=True)
|
|
docker_args = [
|
|
*PODMAN_ARGS,
|
|
*workspace_mount_args(args.cros),
|
|
]
|
|
validate_podman(docker)
|
|
|
|
if args.cros:
|
|
docker_args.append("--privileged") # cros container requires privileged container
|
|
docker_args.append(CROS_IMAGE_NAME)
|
|
else:
|
|
docker_args.append(DEV_IMAGE_NAME + ":" + DEV_IMAGE_VERSION)
|
|
|
|
# Add environment variables to command line
|
|
exec_args: List[str] = []
|
|
for key in ENV_PASSTHROUGH:
|
|
value = os.environ.get(key)
|
|
if value is not None:
|
|
exec_args.append("--env")
|
|
exec_args.append(f"{key}={quoted(value)}")
|
|
|
|
if args.self_test:
|
|
TestDevContainer.docker = docker
|
|
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDevContainer)
|
|
unittest.TextTestRunner().run(suite)
|
|
return
|
|
|
|
if args.stop:
|
|
if not delete_container(docker, args.cros):
|
|
print(f"container is not running.")
|
|
return
|
|
|
|
if args.clean:
|
|
delete_container(docker, args.cros)
|
|
|
|
if args.pull:
|
|
if args.cros:
|
|
docker("pull", CROS_IMAGE_NAME).fg()
|
|
else:
|
|
docker("pull", f"gcr.io/crosvm-infra/crosvm_dev:{DEV_IMAGE_VERSION}").fg()
|
|
return
|
|
|
|
command = args.command
|
|
|
|
# Default to interactive mode if a tty is present.
|
|
tty_args: List[str] = []
|
|
if sys.stdin.isatty():
|
|
tty_args += ["--tty"]
|
|
if not args.no_interactive:
|
|
tty_args += ["--interactive"]
|
|
|
|
# Start an interactive shell by default
|
|
if args.hermetic:
|
|
# cmd is passed to entrypoint
|
|
quoted_cmd = list(map(quoted, command))
|
|
docker(f"run --rm", *tty_args, *docker_args, *exec_args, *quoted_cmd).fg()
|
|
else:
|
|
# cmd is executed directly
|
|
cid = ensure_container_is_alive(docker, docker_args, args.cros)
|
|
if not command:
|
|
command = ("/bin/bash",)
|
|
quoted_cmd = list(map(quoted, command))
|
|
docker("exec", *tty_args, *exec_args, cid, *quoted_cmd).fg()
|
|
|
|
|
|
class TestDevContainer(unittest.TestCase):
|
|
"""
|
|
Runs live tests using the docker service.
|
|
|
|
Note: This test is not run by health-check since it cannot be run inside the
|
|
container. It is run by infra/recipes/health_check.py before running health checks.
|
|
"""
|
|
|
|
docker: cmd
|
|
docker_args = [
|
|
*workspace_mount_args(cros=False),
|
|
*DOCKER_ARGS,
|
|
]
|
|
|
|
def setUp(self):
|
|
# Start with a stopped container for each test.
|
|
delete_container(self.docker, cros=False)
|
|
|
|
def test_stopped_container(self):
|
|
# Create but do not run a new container.
|
|
self.docker(
|
|
f"create --name {DEV_CONTAINER_NAME}", *self.docker_args, "sleep infinity"
|
|
).stdout()
|
|
self.assertTrue(container_id(self.docker, cros=False))
|
|
self.assertFalse(container_is_running(self.docker, cros=False))
|
|
|
|
def test_container_reuse(self):
|
|
cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False)
|
|
cid2 = ensure_container_is_alive(self.docker, self.docker_args, cros=False)
|
|
self.assertEqual(cid, cid2)
|
|
|
|
def test_handling_of_stopped_container(self):
|
|
cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False)
|
|
self.docker("kill", cid).fg()
|
|
|
|
# Make sure we can get back into a good state and execute commands.
|
|
ensure_container_is_alive(self.docker, self.docker_args, cros=False)
|
|
self.assertTrue(container_is_running(self.docker, cros=False))
|
|
main(["true"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|