diff --git a/.gitignore b/.gitignore index 96ef6c0..4fffb2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /target -Cargo.lock +/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 3068fd1..e6278a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,10 @@ members = [ "winit", "winit-core", "winit-examples", "winit-wayland", + "winit-test", + "winit-test-runner" ] resolver = "2" + +[workspace.dependencies] +winit-test = { path = "winit-test" } diff --git a/ci/build_docker_images.sh b/ci/build_docker_images.sh new file mode 100644 index 0000000..4d1db58 --- /dev/null +++ b/ci/build_docker_images.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -eu + +rx() { + echo >&2 "+ $*" + "$@" +} +build_dockerfile() { + tag="$1" + name="$2" + + rx docker build . -f dockerfiles/"$name" \ + -t ghcr.io/notgull/winit_test:"$tag" +} + +build_dockerfile ubuntu Dockerfile.ubuntu diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 0000000..a38de0f --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# MIT/Apache2 License + +set -eu + +rx() { + echo >&2 "+ $*" + "$@" +} +no_out() { + cmd_noout="$1" + shift + "$cmd_noout" "$@" > /dev/null 2> /dev/null +} +bail() { + echo "[fatal] $*" + exit 1 +} +info() { + echo "[info] $*" +} +bail_if_absent() { + if ! no_out command -v "$1"; then + bail "could not find $1" + fi +} +test_runner() { + bail_if_absent "${CARGO:-cargo}" + rx "${CARGO:-cargo}" run --release -p winit-test-runner -- "$@" +} + +basedir="$(dirname -- "$(dirname -- "$0")")" +cd "$basedir" || exit 1 + +config_path="${1:-basedir/ci/tests_linux.json}" + +# Tell which level of test we're running. +case "${2:-2}" in + 0) info "running level 0 (style) tests"; level=0 ;; + 1) info "running level 1 (function) tests"; level=1 ;; + 2) info "running level 2 (full) tests"; level=2 ;; + *) bail "unknown test level $1" ;; +esac + +# Always run style tests. +#test_runner style --config "$config_path" + +# At level 1 or higher, run functionality tests. +#if [ "$level" -ge 1 ]; then +# test_runner functionality --config "$config_path" +#fi + +# At level 2 or higher, run full tests. +if [ "$level" -ge 2 ]; then + test_runner full --config "$config_path" +fi diff --git a/ci/tests_android.json b/ci/tests_android.json new file mode 100644 index 0000000..8051a91 --- /dev/null +++ b/ci/tests_android.json @@ -0,0 +1,10 @@ +[ + { + "name": "winit-core", + "checks": [ + { + "target": "x86_64-linux-android" + } + ] + } +] \ No newline at end of file diff --git a/ci/tests_linux.json b/ci/tests_linux.json new file mode 100644 index 0000000..c83a27d --- /dev/null +++ b/ci/tests_linux.json @@ -0,0 +1,10 @@ +[ + { + "name": "winit-core", + "checks": [ + { + "target": "x86_64-unknown-linux-gnu" + } + ] + } +] \ No newline at end of file diff --git a/dockerfiles/Dockerfile.ubuntu b/dockerfiles/Dockerfile.ubuntu new file mode 100644 index 0000000..30ec04d --- /dev/null +++ b/dockerfiles/Dockerfile.ubuntu @@ -0,0 +1,18 @@ +# Ubuntu testing container +# Usually tagged as ghcr.io/notgull/winit:ubuntu + +FROM rust as final +ARG DEBIAN_FRONTEND=noninteractive + +RUN \ +apt-get -o Acquire::Retries=10 -qq update && \ +apt-get -o Acquire::Retries=10 -o Dpkg::Use-Pty=0 install -y --no-install-recommends \ + xvfb \ + libx11-dev \ + libxcb1-dev \ + libxkbcommon-dev && \ +rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/* \ + /var/log/* \ + /usr/share/{doc,man} diff --git a/winit-core/Cargo.toml b/winit-core/Cargo.toml index b91ae93..15db065 100644 --- a/winit-core/Cargo.toml +++ b/winit-core/Cargo.toml @@ -9,3 +9,11 @@ publish = false bitflags = "2.4.1" raw-window-handle = "0.6.0" raw-window-handle-05 = { package = "raw-window-handle", version = "0.5.2" } + +[dev-dependencies] +futures-lite = "2.0.0" +winit-test.workspace = true + +[[example]] +name = "winit-core_hello_world" +path = "winit_tests/hello_world/src/lib.rs" diff --git a/winit-core/winit_tests/hello_world/src/lib.rs b/winit-core/winit_tests/hello_world/src/lib.rs new file mode 100644 index 0000000..d67ad3c --- /dev/null +++ b/winit-core/winit_tests/hello_world/src/lib.rs @@ -0,0 +1,14 @@ +// MIT/Apache2 License + +use futures_lite::future::block_on; + +#[allow(clippy::eq_op)] +fn main() { + winit_test::run_tests(|harness| { + block_on(async move { + harness.test("hello world", async { + assert_eq!(1 + 1, 2) + }).await; + }); + }) +} diff --git a/winit-test-runner/Cargo.toml b/winit-test-runner/Cargo.toml new file mode 100644 index 0000000..6f62b3b --- /dev/null +++ b/winit-test-runner/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "winit-test-runner" +version = "0.1.0" +edition = "2021" +authors = ["John Nunley "] +publish = false + +[dependencies] +async-channel = "2.1.1" +async-executor = "1.8.0" +async-fs = "2.1.0" +async-io = "2.2.2" +async-lock = "3.2.0" +async-process = "2.0.1" +blocking = "1.5.1" +clap = "4.4.11" +color-eyre = "0.6.2" +eyre = "0.6.11" +futures-lite = "2.1.0" +human-panic = "1.2.2" +ignore = "0.4.21" +winit-test.workspace = true +once_cell = "1.19.0" +owo-colors = "4.0.0" +pin-project-lite = "0.2.13" +regex = "1.10.2" +serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" +tracing = { version = "0.1.40", default-features = false, features = ["async-await"] } +tracing-subscriber = "0.3.18" +which = "5.0.0" diff --git a/winit-test-runner/README.md b/winit-test-runner/README.md new file mode 100644 index 0000000..a2f5969 --- /dev/null +++ b/winit-test-runner/README.md @@ -0,0 +1,23 @@ +# winit-test-runner + +Test runner for `winit`. + +This binary acts as orchestration for running tests in `winit`. In addition to +what is often used in normal Rust tests (formatting, Clippy, unit tests), it also +runs those tests in cross-compilation environments on other platforms. + +There are different types of tests: + +- **Style tests** make sure that code is formatted and linted properly. + `cargo fmt` and `cargo clippy` are used to inspect Rust code. +- **Functionality tests** run the doctests and unit tests in Rust code. These + often ensure that basic functionality and logic are in working order. +- **Host tests** run the `winit` test suite on the current host. This test suite + fully tests the functionality of `winit` to ensure that it is working properly. + A full CI run with `winit` should be fully bug-free. +- **Cross tests** run the `winit` test suite in Docker containers/virtual + machines in order to ensure `winit` works on all possible hosts. + +## License + +MIT/Apache2 diff --git a/winit-test-runner/src/lib.rs b/winit-test-runner/src/lib.rs new file mode 100644 index 0000000..6949d7b --- /dev/null +++ b/winit-test-runner/src/lib.rs @@ -0,0 +1,4 @@ +// MIT/Apache2 License + +mod runner; +pub use runner::*; diff --git a/winit-test-runner/src/main.rs b/winit-test-runner/src/main.rs new file mode 100644 index 0000000..7b159bc --- /dev/null +++ b/winit-test-runner/src/main.rs @@ -0,0 +1,79 @@ +// MIT/Apache2 License + +mod runner; + +use std::path::Path; +use std::process::exit; + +use color_eyre::{eyre::bail, Result}; +use owo_colors::OwoColorize; + +fn main() { + // Setup error hooks. + tracing_subscriber::fmt::try_init().ok(); + color_eyre::install().ok(); + + // Get CLI matches. + let matches = cli().get_matches(); + + // Run the main function. + if let Err(e) = async_io::block_on(entry(matches)) { + println!("{}{:?}", "encountered a fatal error: ".red().bold(), e); + exit(1); + } +} + +async fn entry(matches: clap::ArgMatches) -> Result<()> { + match matches.subcommand() { + None => bail!("expected a subcommand"), + Some(("style", matches)) => { + let crates = read_config(matches.get_one::("config").unwrap()).await?; + runner::Test::Style.run(crates).await?; + } + Some(("functionality", matches)) => { + let crates = read_config(matches.get_one::("config").unwrap()).await?; + runner::Test::Functionality.run(crates).await?; + } + Some(("full", matches)) => { + let crates = read_config(matches.get_one::("config").unwrap()).await?; + runner::Test::Host.run(crates).await?; + } + Some((subcommand, _matches)) => bail!("unknown subcommand {subcommand}"), + } + + Ok(()) +} + +async fn read_config(path: impl AsRef) -> Result> { + let data = async_fs::read(path).await?; + let crates = serde_json::from_slice(&data)?; + Ok(crates) +} + +fn cli() -> clap::Command { + clap::Command::new("winit-test-runner") + .subcommand( + clap::Command::new("style").arg( + clap::Arg::new("config") + .required(true) + .short('c') + .long("config"), + ), + ) + .subcommand( + clap::Command::new("functionality").arg( + clap::Arg::new("config") + .required(true) + .short('c') + .long("config"), + ), + ) + .subcommand( + clap::Command::new("full").arg( + clap::Arg::new("config") + .required(true) + .short('c') + .long("config"), + ), + ) +} diff --git a/winit-test-runner/src/runner/command.rs b/winit-test-runner/src/runner/command.rs new file mode 100644 index 0000000..a439437 --- /dev/null +++ b/winit-test-runner/src/runner/command.rs @@ -0,0 +1,241 @@ +// MIT/Apache2 License + +use super::environment::{Environment, RunCommand}; +use super::util::spawn; +use super::{Check, Crate}; + +use color_eyre::eyre::{eyre, Result, WrapErr}; +use tracing::Instrument; + +use futures_lite::io::BufReader; +use futures_lite::prelude::*; + +use std::env; +use std::ffi::{OsStr, OsString}; +use std::time::Duration; + +/// A command to run. +pub(crate) struct Command { + /// The command to run. + command: OsString, + + /// Arguments for the command. + args: Vec, + + /// Working directory to use for the command. + pwd: Option, +} + +impl Command { + /// Create a new command. + pub(crate) fn new(command: impl AsRef) -> Self { + Self { + command: command.as_ref().to_os_string(), + args: vec![], + pwd: None, + } + } + + /// Add an argument to this command. + #[inline] + pub(crate) fn arg(&mut self, arg: impl AsRef) -> &mut Self { + self.args.push(arg.as_ref().to_os_string()); + self + } + + /// Add several arguments to this command. + #[inline] + pub(crate) fn args>(&mut self, args: impl IntoIterator) -> &mut Self { + let args = args.into_iter(); + let (lo, _) = args.size_hint(); + + self.args.reserve(lo); + for arg in args { + self.args.push(arg.as_ref().to_os_string()); + } + + self + } + + /// Set the working directory for this command. + #[inline] + #[allow(dead_code)] + pub(crate) fn pwd(&mut self, pwd: impl AsRef) -> &mut Self { + self.pwd = Some(pwd.as_ref().to_os_string()); + self + } + + /// Run this command on a host environment. + pub(crate) fn spawn(&mut self, host: E) -> Result { + let args = self.args.iter().map(|arg| &**arg).collect::>(); + host.run_command(&self.command, args.as_slice(), self.pwd.as_deref()) + } +} + +/// Run a command to completion. +#[inline] +pub async fn run( + name: &str, + mut child: impl RunCommand + Send + 'static, + timeout: Option, +) -> Result<()> { + drop(child.stdin()); + + // Spawn a task to emit stdout to tracing. + let run_stdout = spawn({ + let stdout = child.stdout(); + let span = tracing::trace_span!("stdout", name); + async move { + if let Some(stdout) = stdout { + let mut buffer = String::new(); + let mut stdout = BufReader::new(stdout); + + while stdout.read_line(&mut buffer).await.is_ok() { + if buffer.is_empty() { + break; + } + buffer.pop(); + tracing::trace!("+ {buffer}"); + buffer.clear(); + } + + // Write out any remaining data. + if !buffer.is_empty() { + tracing::trace!("+ {buffer}"); + } + } + } + .instrument(span) + }); + + // Spawn a task to emit stderr to info. + let run_stderr = spawn({ + let stderr = child.stderr(); + let span = tracing::info_span!("stderr", name); + async move { + if let Some(stderr) = stderr { + let mut buffer = String::new(); + let mut stderr = BufReader::new(stderr); + + while stderr.read_line(&mut buffer).await.is_ok() { + if buffer.is_empty() { + break; + } + buffer.pop(); + tracing::debug!("+ {buffer}"); + buffer.clear(); + } + + // Write out any remaining data. + if !buffer.is_empty() { + tracing::debug!("+ {buffer}"); + } + } + } + .instrument(span) + }); + + // Spawn a task to poll the process. + let status = spawn({ + let name = name.to_string(); + let span = tracing::info_span!("status", name); + async move { child.exit().await }.instrument(span) + }); + + // Use a future to time out. + let timeout = async move { + timeout + .map_or_else(async_io::Timer::never, async_io::Timer::after) + .await; + Err(eyre!("child {name} timed out")) + }; + + let result = status + .or(timeout) + .await + .with_context(|| format!("while running command: {name}")); + + // Cancel the other two tasks. + run_stdout.cancel().await; + run_stderr.cancel().await; + + result +} + +/// `rustfmt` +#[inline] +pub fn rustfmt() -> Result { + command_with_env("RUSTFMT", "rustfmt") +} + +/// `rustc` +#[inline] +pub fn rustc() -> Result { + command_with_env("RUSTC", "rustc") +} + +/// `adb` +#[inline] +pub fn adb() -> Result { + command_with_env("ADB", "adb") +} + +/// `cargo` +#[inline] +pub fn cargo() -> Result { + command_with_env("CARGO", "cargo") +} + +/// `docker` +#[inline] +pub fn docker() -> Result { + command_with_env("DOCKER", "docker") +} + +/// `xbuild` +#[inline] +pub fn xbuild() -> Result { + command_with_env("XBUILD", "x") +} + +/// `cargo` for a specific `Crate` and `Check`. +#[inline] +pub fn cargo_for_check(subcommands: &[&str], crate_: &Crate, check: &Check) -> Result { + let mut cargo = cargo()?; + cargo.args(subcommands); + cargo.args(["--package", &crate_.name]); + cargo.args(["--target", &check.target_triple]); + + if check.no_default_features { + cargo.arg("--no-default-features"); + } + if let Some(features) = check.features.as_ref() { + let features = features.join(","); + cargo.args(["--features", &features]); + } + + Ok(cargo) +} + +/// `cargo` for a set of `Crate`s. +#[inline] +pub fn cargo_for_crate<'a>( + subcommands: &'static [&'static str], + crate_: &'a Crate, +) -> impl Iterator> + 'a { + crate_ + .checks + .iter() + .map(|check| cargo_for_check(subcommands, crate_, check)) +} + +/// Get a command based on an environment variable. +#[inline] +fn command_with_env(env_name: &str, alterative: impl AsRef) -> Result { + // Get the command name. + let name = env::var_os(env_name).unwrap_or_else(|| alterative.as_ref().to_os_string()); + + // TODO: Tell if we have it. + + Ok(Command::new(name)) +} diff --git a/winit-test-runner/src/runner/environment/android.rs b/winit-test-runner/src/runner/environment/android.rs new file mode 100644 index 0000000..27d5195 --- /dev/null +++ b/winit-test-runner/src/runner/environment/android.rs @@ -0,0 +1,417 @@ +// MIT/Apache2 License + +//! Run the tests on an Android emulator, using xbuild and adb. +//! +//! Only works on Linux. + +use super::{CurrentHost, Environment, RunCommand}; + +use crate::runner::command::{adb, docker, run, xbuild}; +use crate::runner::util::spawn; + +use async_executor::Task; +use async_lock::OnceCell; +use color_eyre::eyre::{bail, Context, Result}; +use regex::Regex; + +use futures_lite::io::BufReader; +use futures_lite::prelude::*; + +use std::env; +use std::ffi::OsStr; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +const ANDROID_DOCKER_IMAGE: &str = + "us-docker.pkg.dev/android-emulator-268719/images/30-google-x64:30.1.2"; + +const SHORT_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); +const WAIT_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +/// Runs an Android emulator in Docker and communicates with it. +#[derive(Clone)] +pub struct AndroidEnvironment { + /// The cell containing the Docker container running the Android system. + android_runner: Arc>, + + /// Run commands on the current host. + host: Arc, +} + +struct AndroidRunner { + /// The docker ID. + docker_id: String, +} + +impl AndroidEnvironment { + /// Create a new Android runner. + #[inline] + pub fn new(root: PathBuf) -> Self { + Self { + android_runner: Arc::new(OnceCell::new()), + host: Arc::new(CurrentHost::new(root)), + } + } + + #[inline] + async fn setup_android_emulator(&self) -> Result { + // Read the adbkey file in home. + let adbkey = { + let adbkey_path = PathBuf::from(env::var_os("HOME").unwrap()).join(".android/adbkey"); + let mut adbkey_buffer = String::new(); + async_fs::File::open(adbkey_path) + .await? + .read_to_string(&mut adbkey_buffer) + .await?; + adbkey_buffer + }; + + // Start the docker image. + let mut docker_run = docker()? + .arg("run") + .arg("--detach") + .arg("-e") + .arg(format!("ADBKEY={adbkey}")) + .args(["--device", "/dev/kvm"]) + .args(["--publish", "8554:8554/tcp"]) + .args(["--publish", "15555:5555/tcp"]) + .arg(ANDROID_DOCKER_IMAGE) + .spawn(&*self.host)?; + + // Read stdout to get the container ID + let container_id = { + let mut stdout = docker_run.stdout.take().unwrap(); + let stdout_runner = spawn(async move { + let mut buf = String::new(); + stdout.read_to_string(&mut buf).await?; + std::io::Result::Ok(buf) + }); + + run( + "android docker container spawn", + docker_run, + Some(SHORT_COMMAND_TIMEOUT), + ) + .await?; + + let mut container_id = stdout_runner.await?; + if container_id.ends_with('\n') { + container_id.pop(); + } + container_id + }; + + // Wait for Docker to start running. + async_io::Timer::after(Duration::from_millis(100)).await; + + // Initialize ADB connecting to host port 15555 (adb inside the container). + let adb_connect = || async { + run( + "adb connect localhost:15555", + adb()? + .arg("connect") + .arg("localhost:15555") + .spawn(&*self.host)?, + Some(SHORT_COMMAND_TIMEOUT), + ) + .await + }; + for i in 0..5 { + match adb_connect().await { + Ok(()) => break, + Err(err) => { + if i == 5 { + return Err(err); + } else { + tracing::error!("adb connect failed, retrying in two seconds..."); + async_io::Timer::after(Duration::from_secs(2)).await; + continue; + } + } + } + } + + // Wait for the device to come online. + run( + "adb wait-for-device", + adb()?.arg("wait-for-device").spawn(&*self.host)?, + Some(WAIT_TIMEOUT), + ) + .await?; + + // Wait for the boot to complete. + { + let mut retry = 0; + loop { + let mut boot_check = adb()? + .arg("shell") + .arg("getprop") + .arg("sys.boot_completed") + .spawn(&*self.host)?; + + let mut stdout = boot_check.stdout.take().unwrap(); + let runner = spawn(async move { + run( + "adb shell getprop sys.boot_completed", + boot_check, + Some(SHORT_COMMAND_TIMEOUT), + ) + .await + }); + + let mut result = String::new(); + stdout.read_to_string(&mut result).await?; + runner.await?; + + // If the first char is `1`, we are done. + if result.starts_with('1') { + break; + } + + // Otherwise, try again. + retry += 1; + if retry >= 240 { + bail!("failed to get boot status after 240 tries"); + } + + async_io::Timer::after(Duration::from_secs(2)).await; + } + }; + + Ok(AndroidRunner { + docker_id: container_id, + }) + } +} + +impl Environment for AndroidEnvironment { + type Command = AndroidCommand; + + fn cleanup(&self) -> std::pin::Pin> + Send + '_>> { + Box::pin(async move { + if let Some(runner) = self.android_runner.get() { + // Run a process to stop the docker container. + run( + "docker stop", + docker()? + .arg("stop") + .arg(&runner.docker_id) + .spawn(&*self.host) + .context("while spawning docker stop")?, + None, + ) + .await + .context("while running docker stop")?; + + // Clean up the Docker container. + run( + "docker rm", + docker()? + .arg("rm") + .arg(&runner.docker_id) + .spawn(&*self.host) + .context("while spawning docker rm")?, + None, + ) + .await + .context("while spawning docker rm")?; + } + + Ok(()) + }) + } + + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + assert!(pwd.is_none()); + let is_cargo = cmd.to_str().map_or(false, |s| s.ends_with("cargo")); + + // For `cargo test --tests` and `cargo test --doc`, we can't actually run these on Android. + // Just skip them for now. + if is_cargo && args.first().and_then(|arg| arg.to_str()) == Some("test") { + tracing::warn!("cannot run `cargo test` on Android, ignoring"); + return Ok(AndroidCommand::NoOp); + } + + // For `cargo run --example`, we need to run the example itself. + if is_cargo && args.first().and_then(|arg| arg.to_str()) == Some("run") { + let this = self.clone(); + let task = spawn(async move { + // Start up the Android emulator. + SendSyncWrapper { + f: this + .android_runner + .get_or_try_init(|| async { this.setup_android_emulator().await }), + } + .await?; + + // Spawn a child. + let mut xbuild = xbuild()? + .arg("run") + .args(["--device", "adb:localhost:15555"]) + .args(["--arch", "arm64"]) + .args([ + "--manifest-path", + "crates/foundation/winit-reactor/winit_tests/general_tests/Cargo.toml", + ]) + .spawn(&*this.host)?; + + let (line_sender, line_receiver) = async_channel::bounded(1); + + // Take out stdout and stderr and analyze them. + let ls = line_sender.clone(); + let mut stdout = BufReader::new(xbuild.stdout.take().unwrap()); + let stdout = spawn(async move { + loop { + let mut buf = String::new(); + if stdout.read_line(&mut buf).await.is_err() { + break; + } + if buf.is_empty() { + break; + } + + if buf.ends_with('\n') { + buf.pop(); + } + tracing::trace!("xbuild stdout: {buf}"); + + ls.send(buf).await.ok(); + } + }); + + let mut stderr = BufReader::new(xbuild.stderr.take().unwrap()); + let stderr = spawn(async move { + loop { + let mut buf = String::new(); + if stderr.read_line(&mut buf).await.is_err() { + break; + } + if buf.is_empty() { + break; + } + + if buf.ends_with('\n') { + buf.pop(); + } + tracing::trace!("xbuild stderr: {buf}"); + + line_sender.send(buf).await.ok(); + } + }); + + let runner = spawn(async move { run("xbuild", xbuild, None).await }); + let mut reporter = winit_test::reporter::ConsoleReporter::new(); + let dump_finder = Regex::new(r"winit_TEST_DUMP\((.*)\)winit_TEST_DUMP")?; + + let regex_finder = async { + while let Ok(line) = line_receiver.recv().await { + if let Some(mat) = dump_finder.captures(&line) { + if let Some(data) = mat.get(1) { + let mut stop_running = false; + let event: winit_test::TestEvent = + serde_json::from_str(data.as_str())?; + + // Stop running if event is the end event. + if let winit_test::TestEvent::End { .. } = &event { + stop_running = true; + } + + winit_test::reporter::Reporter::report(&mut reporter, event).await; + + if stop_running { + break; + } + } + } + } + + Ok(()) + }; + + let result = regex_finder.await; + + // Cancel tasks. + if let Some(Err(e)) = runner.cancel().await { + return Err(e); + } + stdout.cancel().await; + stderr.cancel().await; + + let code = winit_test::reporter::Reporter::finish(&mut reporter); + if code != 0 { + bail!("received an error from the android runner") + } else { + result + } + }); + + return Ok(AndroidCommand::XbuildRun(task)); + } + + bail!("unable to run Android command: {cmd:?} {args:?}") + } +} + +pub(crate) enum AndroidCommand { + NoOp, + XbuildRun(Task>), +} + +impl RunCommand for AndroidCommand { + #[inline] + fn exit(&mut self) -> std::pin::Pin> + Send + '_>> { + Box::pin(async move { + match self { + Self::NoOp => Ok(()), + Self::XbuildRun(ch) => ch.await, + } + }) + } + + #[inline] + fn stdin(&mut self) -> Option>> { + None + } + + #[inline] + fn stderr(&mut self) -> Option>> { + // Always taken out. + None + } + + #[inline] + fn stdout(&mut self) -> Option>> { + // Always taken out. + None + } +} + +pin_project_lite::pin_project! { + // https://github.com/smol-rs/event-listener-strategy/issues/13 + struct SendSyncWrapper { + #[pin] + f: F + } +} + +unsafe impl Send for SendSyncWrapper {} +unsafe impl Sync for SendSyncWrapper {} + +impl Future for SendSyncWrapper { + type Output = F::Output; + + #[inline] + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + self.project().f.poll(cx) + } +} diff --git a/winit-test-runner/src/runner/environment/choose.rs b/winit-test-runner/src/runner/environment/choose.rs new file mode 100644 index 0000000..ff01668 --- /dev/null +++ b/winit-test-runner/src/runner/environment/choose.rs @@ -0,0 +1,101 @@ +// MIT/Apache2 License + +//! Choose the proper host environment to run inside. + +use super::{DynEnvironment, Environment}; +use crate::runner::util::target_triple; +use crate::runner::Check; + +use color_eyre::eyre::{bail, Result}; +use once_cell::sync::OnceCell; + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, RwLock}; + +static OPEN_ENVIRONMENTS: OnceCell>>> = + OnceCell::new(); + +#[derive(Debug, PartialEq, Eq, Hash)] +struct CheckKey { + target_triple: String, + host_env: Option, +} + +#[allow(clippy::never_loop)] +pub(crate) async fn choose(root: &Path, check: &Check) -> Result> { + // Figure out our current check key. + let key = CheckKey { + target_triple: check.target_triple.clone(), + host_env: check.host_env.clone(), + }; + + // See if we have an environment matching this. + let open_environments = OPEN_ENVIRONMENTS.get_or_init(|| RwLock::new(HashMap::new())); + { + let open_environments = open_environments.read().unwrap(); + if let Some(host) = open_environments.get(&key) { + return Ok(host.clone()); + } + } + + // Otherwise, we need to choose it. + // TODO: Other environments. + let host = Arc::new(loop { + // Get the current target triple. + let host_target = target_triple(root).await?; + + // If the triple is the same as our desired triple, use the current host environment. + if host_target == check.target_triple { + let host = super::host::CurrentHost::new(root.to_path_buf()); + break DynEnvironment::from_environment(host); + } + + // If the host is Linux and the target is Android, use the Android runner. + if host_target.contains("linux") && check.target_triple.contains("android") { + let host = super::android::AndroidEnvironment::new(root.to_path_buf()); + break DynEnvironment::from_environment(host); + } + + // On Windows/Linux hosts, Linux targets that aren't Android can use Docker. + if (host_target.contains("windows") || host_target.contains("linux")) + && check.target_triple.contains("linux") + && !check.target_triple.contains("android") + { + // It will only work for the same architecture, though. + if host_target.split('-').next() == check.target_triple.split('-').next() { + let host = super::docker::DockerEnvironment::start( + root.to_path_buf(), + &check.target_triple, + check.host_env.as_deref(), + ) + .await?; + break DynEnvironment::from_environment(host); + } + } + + bail!( + "cannot find compatible environment for host {host_target} and target {}", + &check.target_triple + ) + }); + + // Insert it into the list. + let mut open_environments = open_environments.write().unwrap(); + open_environments.insert(key, host.clone()); + Ok(host) +} + +#[allow(clippy::await_holding_lock)] +pub(crate) async fn cleanup() -> Result<()> { + let mut open_environments = match OPEN_ENVIRONMENTS.get() { + None => return Ok(()), + Some(oe) => oe.write().unwrap(), + }; + + for (_, host) in open_environments.drain() { + host.cleanup().await?; + } + + Ok(()) +} diff --git a/winit-test-runner/src/runner/environment/docker.rs b/winit-test-runner/src/runner/environment/docker.rs new file mode 100644 index 0000000..33aef39 --- /dev/null +++ b/winit-test-runner/src/runner/environment/docker.rs @@ -0,0 +1,236 @@ +// MIT/Apache2 License + +//! Run tests inside of a Docker container with a configured environment. +//! +//! Useful for Linux/Windows tests on the same architecture. + +use async_executor::Task; +use async_process::Child; +use color_eyre::eyre::{bail, eyre, Result, WrapErr}; + +use futures_lite::io::BufReader; +use futures_lite::prelude::*; + +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::Duration; + +use crate::runner::command::{docker, run}; +use crate::runner::environment::{CurrentHost, Environment, RunCommand}; +use crate::runner::util::spawn; + +const TEST_LISTENER_PATH: &str = "/tmp/winit_test_listener.sock"; + +/// Run commands in a Docker container. +pub(crate) struct DockerEnvironment { + /// Host to run commands on. + host: CurrentHost, + + /// The ID of the Docker container. + docker_id: String, + + /// The running listener task. + listener_task: Mutex>>, +} + +impl DockerEnvironment { + /// Start up the Docker image. + pub(crate) async fn start( + root: PathBuf, + target_triple: &str, + options: Option<&str>, + ) -> Result { + let host = CurrentHost::new(root.clone()); + + let root = root + .to_str() + .ok_or_else(|| eyre!("cannot have root be a non-UTF8 path for Docker environment"))?; + + // Start a Unix command line listener. + let (ready_send, ready_recv) = async_channel::bounded(1); + let listener_task = spawn(async move { + #[cfg(unix)] + { + async_fs::remove_file(TEST_LISTENER_PATH).await.ok(); + if let Err(e) = winit_test::run_unix_listener( + TEST_LISTENER_PATH.as_ref(), + winit_test::reporter::ConsoleReporter::new(), + ready_send, + ) + .await + { + tracing::error!("unable to run Unix listener: {e}"); + } + } + + #[cfg(not(unix))] + todo!("how to run docker sockets outside of Unix?") + }); + ready_recv.recv().await.ok(); + + // Start the docker container. + let mut child = docker()? + .arg("run") + .arg("--detach") + .args(["--volume", &format!("{root}:{root}")]) + .args(["--volume", &format!("{0}:{0}", TEST_LISTENER_PATH)]) + .args([ + "--env", + &format!("winit_TEST_UDS_SOCKET={}", TEST_LISTENER_PATH), + ]) + .args(["--workdir", root]) + .arg(get_target_container(target_triple, options)?) + .args(["sh", "-c", "tail -f /dev/null"]) + .spawn(&host) + .context("while spawning initial docker")?; + + // Read stdout to get the container ID. + let container_id = { + let mut stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + let mut buf = String::new(); + + // Read to end and then wait for finish. + let docker_runner = spawn(async move { child.exit().await }); + let stderr_runner = spawn(async move { + let mut line = String::new(); + let mut stderr = BufReader::new(stderr); + while stderr.read_line(&mut line).await.is_ok() { + line.pop(); + if line.is_empty() { + break; + } + tracing::trace!("docker stderr: {line}"); + } + }); + stdout + .read_to_string(&mut buf) + .await + .context("while reading from Docker daemon")?; + docker_runner + .await + .context("while waiting for docker runner exit")?; + stderr_runner.cancel().await; + + if buf.ends_with('\n') { + buf.pop(); + } + tracing::info!("running container: {buf}"); + + // Buffer should contain the container ID. + buf + }; + + // Wait for a second for the Docker container to start running. + async_io::Timer::after(Duration::from_millis(100)).await; + + Ok(Self { + host, + docker_id: container_id, + listener_task: Mutex::new(Some(listener_task)), + }) + } +} + +impl Environment for DockerEnvironment { + type Command = Child; + + #[inline] + fn cleanup(&self) -> std::pin::Pin> + Send + '_>> { + Box::pin(async move { + // Run a process to stop the docker container. + run( + "docker stop", + docker()? + .arg("stop") + .arg(&self.docker_id) + .spawn(&self.host) + .context("while spawning docker stop")?, + None, + ) + .await + .context("while running docker stop")?; + + // Clean up the Docker container. + run( + "docker rm", + docker()? + .arg("rm") + .arg(&self.docker_id) + .spawn(&self.host) + .context("while spawning docker rm")?, + None, + ) + .await + .context("while spawning docker rm")?; + + // Stop the listener. + let lt = self.listener_task.lock().unwrap().take(); + if let Some(lt) = lt { + lt.cancel().await; + } + + Ok(()) + }) + } + + #[inline] + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + assert!(pwd.is_none()); + let mut sh_command = Path::new(cmd) + .file_name() + .ok_or_else(|| eyre!("no file name for command"))? + .to_str() + .ok_or_else(|| eyre!("cmd was not valid utf-8"))? + .to_string(); + for arg in args { + let arg = arg + .to_str() + .ok_or_else(|| eyre!("arg was not valid utf-8"))?; + + sh_command.push(' '); + sh_command.push_str(arg); + } + + tracing::debug!("docker exec with command: {sh_command}"); + + let child = docker()? + .arg("exec") + .arg(&self.docker_id) + .arg("sh") + .arg("-c") + .arg(sh_command) + .spawn(&self.host) + .context("while spawning docker exec")?; + + Ok(child) + } +} + +fn get_target_container(target_triple: &str, options: Option<&str>) -> Result { + let tag = if target_triple.contains("linux") { + if target_triple.ends_with("gnu") { + // TODO: Fedora, etc etc + "ubuntu" + } else if target_triple.ends_with("musl") { + "alpine" + } else { + bail!("unrecognized linux version {target_triple}") + } + } else { + bail!("no tag for target triple {target_triple}") + }; + + // TODO: Modified images for host options. + if options.is_some() { + bail!("cannot handle options yet"); + } + + Ok(format!("ghcr.io/notgull/winit_test:{tag}")) +} diff --git a/winit-test-runner/src/runner/environment/host.rs b/winit-test-runner/src/runner/environment/host.rs new file mode 100644 index 0000000..82a6b10 --- /dev/null +++ b/winit-test-runner/src/runner/environment/host.rs @@ -0,0 +1,84 @@ +// MIT/Apache2 License + +//! Run the commands on the current host system. + +use super::{Environment, RunCommand}; + +use async_process::{Child, Command, Stdio}; +use color_eyre::eyre::{bail, Result}; +use futures_lite::prelude::*; + +use std::ffi::OsStr; +use std::path::PathBuf; +use std::pin::Pin; + +/// Run commands directly on the current host. +pub(crate) struct CurrentHost { + root: PathBuf, +} + +impl CurrentHost { + pub(crate) fn new(path: PathBuf) -> Self { + Self { root: path } + } +} + +impl Environment for CurrentHost { + type Command = Child; + + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + tracing::debug!("running command {cmd:?} with args {args:?}",); + + let mut command = Command::new(cmd); + command.args(args); + if let Some(pwd) = pwd { + command.current_dir(pwd); + } + let child = command + .current_dir(&self.root) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + Ok(child) + } + + fn cleanup(&self) -> Pin> + Send + '_>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +impl RunCommand for Child { + fn stdin(&mut self) -> Option>> { + let stdin = self.stdin.take()?; + Some(Box::pin(stdin)) + } + + fn stdout(&mut self) -> Option>> { + let stdout = self.stdout.take()?; + Some(Box::pin(stdout)) + } + + fn stderr(&mut self) -> Option>> { + let stderr = self.stderr.take()?; + Some(Box::pin(stderr)) + } + + fn exit(&mut self) -> Pin> + Send + '_>> { + Box::pin(async move { + // Wait for the child to complete. + let status = self.status().await?; + if !status.success() { + bail!("child exited with error code {status:?}"); + } + + Ok(()) + }) + } +} diff --git a/winit-test-runner/src/runner/environment/mod.rs b/winit-test-runner/src/runner/environment/mod.rs new file mode 100644 index 0000000..6ea9a80 --- /dev/null +++ b/winit-test-runner/src/runner/environment/mod.rs @@ -0,0 +1,141 @@ +// MIT/Apache2 License + +//! Host environment to run tests in. +//! +//! This allows the test runner to issue commands to another system. + +mod android; +mod choose; +mod docker; +mod host; + +use color_eyre::Result; +use futures_lite::prelude::*; + +use std::ffi::OsStr; + +use std::pin::Pin; + +pub(crate) use choose::{choose as choose_environment, cleanup as cleanup_hosts}; +pub(crate) use host::CurrentHost; + +/// Host environment to run commands in. +pub(crate) trait Environment { + /// The command to run. + type Command: RunCommand + Send + 'static; + + /// Run a command. + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result; + + /// Clean up the current environment. + fn cleanup(&self) -> Pin> + Send + '_>>; +} + +impl Environment for &E { + type Command = E::Command; + + #[inline] + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + (**self).run_command(cmd, args, pwd) + } + + #[inline] + fn cleanup(&self) -> Pin> + Send + '_>> { + (**self).cleanup() + } +} + +/// A command to run. +pub(crate) trait RunCommand { + /// Get a writer for the standard input. + fn stdin(&mut self) -> Option>>; + /// Get a reader for the standard output. + fn stdout(&mut self) -> Option>>; + /// Get a reader for the standard output. + fn stderr(&mut self) -> Option>>; + + /// Wait for this child to exit. + fn exit(&mut self) -> Pin> + Send + '_>>; +} + +impl RunCommand for Box { + fn stdin(&mut self) -> Option>> { + (**self).stdin() + } + fn stdout(&mut self) -> Option>> { + (**self).stdout() + } + fn stderr(&mut self) -> Option>> { + (**self).stderr() + } + fn exit(&mut self) -> Pin> + Send + '_>> { + (**self).exit() + } +} + +/// A dynamically allocated environment. +pub struct DynEnvironment { + inner: Box< + dyn Environment> + Send + Sync + 'static, + >, +} + +impl DynEnvironment { + /// Create a new `DynEnvironment` from an existing `Environment`. + pub fn from_environment(env: impl Environment + Send + Sync + 'static) -> Self { + struct BoxedRunCommandEnvironment(E); + + impl Environment for BoxedRunCommandEnvironment { + type Command = Box; + + #[inline] + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + let cmd = self.0.run_command(cmd, args, pwd)?; + Ok(Box::new(cmd)) + } + + #[inline] + fn cleanup(&self) -> Pin> + Send + '_>> { + self.0.cleanup() + } + } + + Self { + inner: Box::new(BoxedRunCommandEnvironment(env)), + } + } +} + +impl Environment for DynEnvironment { + type Command = Box; + + #[inline] + fn run_command( + &self, + cmd: &OsStr, + args: &[&OsStr], + pwd: Option<&OsStr>, + ) -> Result { + self.inner.run_command(cmd, args, pwd) + } + + #[inline] + fn cleanup(&self) -> Pin> + Send + '_>> { + self.inner.cleanup() + } +} diff --git a/winit-test-runner/src/runner/functionality.rs b/winit-test-runner/src/runner/functionality.rs new file mode 100644 index 0000000..1f967ef --- /dev/null +++ b/winit-test-runner/src/runner/functionality.rs @@ -0,0 +1,42 @@ +// MIT/Apache2 License + +//! Functionality tests. +//! +//! This just runs "cargo test" in the proper host environment. + +use crate::runner::command::{cargo_for_check, run}; +use crate::runner::environment::choose_environment; +use crate::runner::Crate; + +use color_eyre::eyre::{Result, WrapErr}; + +use std::path::Path; +use std::time::Duration; + +const FUNCTEST_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +/// Run functionality tests. +pub async fn functionality(root: &Path, crates: Vec) -> Result<()> { + for crate_ in crates { + for check in &crate_.checks { + // Choose an environment for this check. + let host = choose_environment(root, check) + .await + .context("while choosing environment")?; + + for mode in ["--tests", "--doc"] { + // Run the cargo command. + let mut command = cargo_for_check(&["test", mode], &crate_, check)?; + run( + "cargo_functionality", + command.spawn(&*host)?, + Some(FUNCTEST_TIMEOUT), + ) + .await + .with_context(|| format!("while running cargo test {mode}"))?; + } + } + } + + Ok(()) +} diff --git a/winit-test-runner/src/runner/mod.rs b/winit-test-runner/src/runner/mod.rs new file mode 100644 index 0000000..525ebdd --- /dev/null +++ b/winit-test-runner/src/runner/mod.rs @@ -0,0 +1,77 @@ +// MIT/Apache2 License + +mod command; +mod environment; +mod functionality; +mod style; +mod tests; +mod util; + +use color_eyre::eyre::eyre; +use serde::{Deserialize, Serialize}; + +use std::path::Path; + +/// A crate to test. +#[derive(Serialize, Deserialize, Debug)] +pub struct Crate { + /// Crate name to test. + pub name: String, + + /// Checks to run. + pub checks: Vec, +} + +/// Check to run for a crate. +#[derive(Serialize, Deserialize, Debug)] +pub struct Check { + /// The target triple to test. + #[serde(rename = "target")] + pub target_triple: String, + + /// Host environment to set up. + pub host_env: Option, + + /// Features to enable. + pub features: Option>, + + /// Turn off default features. + #[serde(default)] + pub no_default_features: bool, + + /// Whether this test should be ignored in the general CI case. + #[serde(default)] + pub niche: bool, +} + +/// Test type to run. +pub enum Test { + /// Run style tests to make sure everything is properly formatted and linted. + Style, + + /// Run functionality tests to make sure unit tests pass. + Functionality, + + /// Run full tests on the current host machine + Host, +} + +impl Test { + /// Run this test. + pub async fn run(self, crates: Vec) -> color_eyre::Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(1) + .ok_or_else(|| eyre!("this cargo package is at an invalid path"))?; + + let result = match self { + Self::Style => util::run(style::style(root, crates)).await, + Self::Functionality => util::run(functionality::functionality(root, crates)).await, + Self::Host => util::run(tests::tests(root, crates)).await, + }; + + environment::cleanup_hosts().await?; + + result + } +} diff --git a/winit-test-runner/src/runner/style.rs b/winit-test-runner/src/runner/style.rs new file mode 100644 index 0000000..828d7dd --- /dev/null +++ b/winit-test-runner/src/runner/style.rs @@ -0,0 +1,121 @@ +// MIT/Apache2 License + +use crate::runner::command::{cargo_for_crate, run, rustfmt}; +use crate::runner::environment::CurrentHost; +use crate::runner::util::spawn; +use crate::runner::Crate; + +use futures_lite::prelude::*; +use futures_lite::stream; + +use async_executor::Task; +use color_eyre::Result; +use tracing::Instrument; + +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +const FMT_TIMEOUT: Duration = Duration::from_secs(30); +const CLIPPY_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +/// Run style tests on this workspace. +pub async fn style(root: &Path, crates: Vec) -> Result<()> { + let root: Arc = root.to_path_buf().into(); + + // Spawn handles for style tasks. + let handles: [Task>; 2] = [ + spawn({ + let root = root.clone(); + let span = tracing::info_span!("rustfmt"); + async move { + rust_fmt(&root).await?; + tracing::info!("rustfmt completed with no errors"); + Ok(()) + } + .instrument(span) + }), + spawn({ + let root = root.clone(); + let span = tracing::info_span!("clippy"); + async move { + rust_clippy(&root, &crates).await?; + tracing::info!("clippy completed with no errors"); + Ok(()) + } + .instrument(span) + }), + ]; + + for handle in handles { + handle.await?; + } + + Ok(()) +} + +/// Run Rust formatting. +async fn rust_fmt(root: &Path) -> Result<()> { + // Get all of the Rust files. + let mut rust_files = files_with_extensions(root, "rs"); + let host = CurrentHost::new(root.to_path_buf()); + + // Create a command to run rustfmt. + let mut fmt = rustfmt()?; + fmt.args(["--edition", "2021", "--check"]); + while let Some(rust_file) = rust_files.next().await { + fmt.arg(rust_file?); + } + run("rustfmt", fmt.spawn(host)?, Some(FMT_TIMEOUT)).await?; + + Ok(()) +} + +/// Run clippy. +async fn rust_clippy(root: &Path, crates: &[Crate]) -> Result<()> { + let command_runner = stream::iter( + crates + .iter() + .flat_map(|crate_| cargo_for_crate(&["clippy"], crate_)), + ) + .then({ + move |command| async move { + let host = CurrentHost::new(root.to_path_buf()); + match command { + Ok(mut command) => run("clippy", command.spawn(host)?, Some(CLIPPY_TIMEOUT)).await, + Err(err) => Err(err), + } + } + }); + futures_lite::pin!(command_runner); + command_runner.try_for_each(|result| result).await?; + + Ok(()) +} + +/// Get all of the files in this namespace with this extension. +fn files_with_extensions( + root: &Path, + ext: impl AsRef, +) -> impl Stream> + 'static { + let root = root.to_path_buf(); + let ext = ext.as_ref().to_os_string(); + + let walker = blocking::unblock(move || ignore::WalkBuilder::new(root).build()); + + stream::once_future(walker) + .flat_map(|walker| blocking::Unblock::with_capacity(16, walker)) + .filter(move |entry| { + if let Ok(entry) = entry { + entry.file_type().map(|f| f.is_file()) == Some(true) + && entry.path().extension() == Some(&*ext) + } else { + true + } + }) + .map(|result| match result { + Ok(entry) => Ok(entry.into_path()), + Err(err) => Err(err.into()), + }) +} diff --git a/winit-test-runner/src/runner/tests.rs b/winit-test-runner/src/runner/tests.rs new file mode 100644 index 0000000..d569a23 --- /dev/null +++ b/winit-test-runner/src/runner/tests.rs @@ -0,0 +1,129 @@ +// MIT/Apache2 License + +//! Run the test suite found in `winit-test`. + +use crate::runner::command::{cargo, cargo_for_check, run}; +use crate::runner::environment::{choose_environment, CurrentHost, RunCommand}; +use crate::runner::util::spawn; + +use async_lock::OnceCell; +use color_eyre::eyre::{eyre, Context, Result}; +use futures_lite::prelude::*; +use serde::Deserialize; +use std::path::{Path, PathBuf}; + +use crate::runner::Crate; + +pub(crate) async fn tests(root: &Path, crates: Vec) -> Result<()> { + for crate_ in crates { + // Get the root directory for this crate. + let crate_root = crate_manifest_root(root, &crate_.name).await?; + + // There should be a folder named "winit-tests" here. If there isn't, there are no tests. + let winit_tests = crate_root.join("winit_tests"); + if async_fs::metadata(&winit_tests).await.is_err() { + tracing::warn!("crate `{}` has no winit_tests", &crate_.name); + continue; + } + + // List the examples here in winit tests. + let mut touched = false; + let running_tests = async_fs::read_dir(winit_tests) + .await? + .inspect(|_| { + touched = true; + }) + .then({ + let crate_ = &crate_; + move |example| async move { + let example = example?.path(); + let name = example + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + eyre!("encountered invalid winit_test example: {example:?}") + })?; + + for check in &crate_.checks { + let env = choose_environment(root, check).await?; + let example_name = format!("{}_{}", crate_.name, name); + + let mut cmd = + cargo_for_check(&["run", "--example", &example_name], crate_, check)?; + run(&format!("cargo_test_{name}"), cmd.spawn(&*env)?, None) + .await + .with_context(|| format!("while running winit test {name}"))?; + } + + Ok(()) + } + }); + + futures_lite::pin!(running_tests); + running_tests + .try_for_each(|r: color_eyre::eyre::Result<()>| r) + .await?; + + if !touched { + tracing::warn!("no winit tests run for {}", &crate_.name); + } + } + + Ok(()) +} + +/// Get the root of a specific crate name. +async fn crate_manifest_root(root: &Path, name: &str) -> Result<&'static Path> { + let metadata = cargo_metadata(root).await?; + + for pack in &metadata.packages { + if pack.name == name { + return pack + .manifest_path + .parent() + .ok_or_else(|| eyre!("manifest path should never be root")); + } + } + + Err(eyre!("unable to find package {name}")) +} + +/// Get the output of `cargo metadata`. +async fn cargo_metadata(root: &Path) -> Result<&'static CargoMetadata> { + static CARGO_METADATA: OnceCell = OnceCell::new(); + + CARGO_METADATA + .get_or_try_init(|| async { + // Launch the child. + let host = CurrentHost::new(root.to_path_buf()); + let mut child = cargo()?.arg("metadata").spawn(host)?; + + // Run the process. + let mut stdout = child.stdout().unwrap(); + let stdout = spawn(async move { + let mut buf = Vec::new(); + stdout.read_to_end(&mut buf).await?; + std::io::Result::Ok(buf) + }); + run("cargo metadata", child, None).await?; + + // Finish reading stdout. + let package_metadata = stdout.await?; + + // Parse stdout. + let meta = serde_json::from_slice(&package_metadata)?; + Ok(meta) + }) + .await +} + +#[derive(Deserialize)] +struct CargoMetadata { + packages: Vec, +} + +#[derive(Deserialize)] +struct MetadataPackage { + name: String, + manifest_path: PathBuf, +} diff --git a/winit-test-runner/src/runner/util.rs b/winit-test-runner/src/runner/util.rs new file mode 100644 index 0000000..2456ca8 --- /dev/null +++ b/winit-test-runner/src/runner/util.rs @@ -0,0 +1,80 @@ +// MIT/Apache2 License + +use crate::runner::command::rustc; +use crate::runner::environment::{CurrentHost, RunCommand}; +use async_executor::{Executor, Task}; +use color_eyre::eyre::{eyre, Result}; +use once_cell::sync::OnceCell; + +use futures_lite::io::BufReader; +use futures_lite::prelude::*; + +use std::future::{pending, Future}; +use std::path::Path; +use std::thread; + +/// Get the target triple for the host by calling into `rustc`. +pub(crate) async fn target_triple(root: &Path) -> Result { + let mut rustc_call = rustc()?.arg("-vV").spawn(CurrentHost::new(root.into()))?; + let mut stdout = { + let stdout = rustc_call + .stdout() + .ok_or_else(|| eyre!("no stdout for rustc call"))?; + BufReader::new(stdout) + }; + + // In the background, run the rustc process. + let rustc_runner = spawn(async move { rustc_call.exit().await }); + + // Read lines from stdout. + let mut line = String::new(); + while stdout.read_line(&mut line).await.is_ok() { + // If the line is empty, break out. + if line.is_empty() { + break; + } + + // If the line starts with "host: ", get the part after. + if let Some(target) = line.strip_prefix("host: ") { + // This is the target. Clean up the rustc process before stopping. + drop(stdout); + rustc_runner.await?; + return Ok(target.trim().to_string()); + } + + line.clear(); + } + + Err(eyre!("unable to find 'host:' line in rustc output")) +} + +/// Spawn a future onto the global executor. +pub(crate) fn spawn(f: F) -> Task +where + F::Output: Send + 'static, +{ + executor().spawn(f) +} + +/// Run the executor alongside this future. +pub(crate) async fn run(f: F) -> F::Output { + executor().run(f).await +} + +fn executor() -> &'static Executor<'static> { + static EXECUTOR: OnceCell> = OnceCell::new(); + + EXECUTOR.get_or_init(|| { + // Only use two executor threads. + for i in 0..2 { + thread::Builder::new() + .name(format!("winit-test-runner-{i}")) + .spawn(|| { + async_io::block_on(executor().run(pending::<()>())); + }) + .expect("failed to spawn runner thread"); + } + + Executor::new() + }) +} diff --git a/winit-test/Cargo.toml b/winit-test/Cargo.toml new file mode 100644 index 0000000..d97dd24 --- /dev/null +++ b/winit-test/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "winit-test" +version = "0.1.0" +edition = "2021" +authors = ["John Nunley "] +publish = false + +[dependencies] +async-channel = "2.1.1" +async-executor = "1.8.0" +async-io = "2.2.2" +async-lock = "3.2.0" +async-net = "2.0.0" +blocking = "1.5.1" +color-eyre = "0.6.2" +futures-lite = "2.1.0" +human-panic = "1.2.2" +owo-colors = "4.0.0" +serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" +tracing = { version = "0.1.40", default-features = false } +tracing-subscriber = "0.3.18" +web-time = "0.2.3" diff --git a/winit-test/README.md b/winit-test/README.md new file mode 100644 index 0000000..ea1e157 --- /dev/null +++ b/winit-test/README.md @@ -0,0 +1,12 @@ +# winit-test + +A testing framework for use in `winit`. + +Requirements for this crate that are not fulfilled by Rust's default testing framework: + +- Can be used remotely. +- Can be easily compiled into arbitrary binaries. + +## License + +MIT/Apache2 diff --git a/winit-test/src/lib.rs b/winit-test/src/lib.rs new file mode 100644 index 0000000..f4beff2 --- /dev/null +++ b/winit-test/src/lib.rs @@ -0,0 +1,341 @@ +// MIT/Apache2 License + +//! A testing framework designed to be used internally in `winit`. + +pub mod reporter; + +use async_channel::Sender; +use async_lock::Mutex; +use futures_lite::{future, prelude::*}; +use owo_colors::OwoColorize; +use reporter::Reporter; +use serde::{Deserialize, Serialize}; +use web_time::Duration; + +use std::borrow::Cow; +use std::env; +use std::future::Future; +use std::io; +use std::panic; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const DEFAULT_TCP_CONNECT_TIMEOUT: u64 = 15; + +/// The event of a test. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TestEvent { + /// There are no more tests. + End { + /// The total number of tests. + count: usize, + }, + + /// Begin a test group. + BeginGroup { + /// Name of the test group. + name: Cow<'static, str>, + + /// Number of tests. + count: usize, + }, + + /// End a test group. + EndGroup(Cow<'static, str>), + + /// The result of a test. + Result(TestResult), +} + +/// The result of the test. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TestResult { + /// The name of the test. + pub name: Cow<'static, str>, + + /// The status of the test. + pub status: TestStatus, + + /// Description of the test failure. + pub failure: Cow<'static, str>, +} + +/// The status of the test. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TestStatus { + /// The test succeeded. + Success, + + /// The test failed. + Failed, + + /// The test was ignored. + Ignored, +} + +/// The test harness. +pub struct TestHarness { + reporter: Mutex>, + count: AtomicUsize, +} + +impl TestHarness { + /// Run with a test group. + pub async fn group(&self, name: impl Into, count: usize, f: impl Future) { + let name = name.into(); + self.reporter + .lock() + .await + .report(TestEvent::BeginGroup { + name: name.clone().into(), + count, + }) + .await; + f.await; + self.reporter + .lock() + .await + .report(TestEvent::EndGroup(name.into())) + .await; + } + + /// Run a test. + pub async fn test(&self, name: impl Into, f: impl Future) { + let name = name.into(); + self.count.fetch_add(1, Ordering::Relaxed); + + let result = { panic::AssertUnwindSafe(f).catch_unwind().await }; + + match result { + Ok(()) => { + self.reporter + .lock() + .await + .report(TestEvent::Result(TestResult { + name: name.into(), + status: TestStatus::Success, + failure: "".into(), + })) + .await; + } + + Err(err) => { + let failure: Cow<'static, str> = if let Some(e) = err.downcast_ref::<&'static str>() + { + (*e).into() + } else if let Ok(e) = err.downcast::() { + (*e).into() + } else { + "".into() + }; + + self.reporter + .lock() + .await + .report(TestEvent::Result(TestResult { + name: name.into(), + status: TestStatus::Failed, + failure, + })) + .await; + } + } + } +} + +/// Run tests with a harness. +#[allow(clippy::never_loop)] +pub fn run_tests(f: impl FnOnce(&TestHarness) -> T) -> T { + // Set up hooks. + tracing_subscriber::fmt::try_init().ok(); + color_eyre::install().ok(); + + // Figure out which reporter we're using. + let reporter: Box = + if let Ok(address) = env::var("winit_TEST_TCP_ADDRESS") { + Box::new( + future::block_on(reporter::StreamReporter::connect( + async_net::TcpStream::connect(address), + Duration::from_secs( + env::var("winit_TEST_TCP_TIMEOUT") + .ok() + .and_then(|timeout| timeout.parse::().ok()) + .unwrap_or(DEFAULT_TCP_CONNECT_TIMEOUT), + ), + )) + .expect("failed to connect to TCP port"), + ) + } else { + loop { + #[cfg(unix)] + { + if let Ok(path) = env::var("winit_TEST_UDS_SOCKET") { + break Box::new( + future::block_on(reporter::StreamReporter::connect( + async_net::unix::UnixStream::connect(path), + Duration::from_secs( + env::var("winit_TEST_UDS_TIMEOUT") + .ok() + .and_then(|timeout| timeout.parse::().ok()) + .unwrap_or(DEFAULT_TCP_CONNECT_TIMEOUT), + ), + )) + .expect("failed to connect to Unix socket"), + ); + } + } + + if cfg!(target_os = "android") { + break Box::new(reporter::DumpReporter::new()); + } + + // By default, use the console reporter. + break Box::new(reporter::ConsoleReporter::new()); + } + }; + + // Create our test harness. + let harness = TestHarness { + reporter: Mutex::new(reporter), + count: AtomicUsize::new(0), + }; + + // Run the tests. + let value = f(&harness); + + // Count tests. + let TestHarness { reporter, count } = harness; + let mut reporter = reporter.into_inner(); + future::block_on(reporter.report(TestEvent::End { + count: count.into_inner(), + })); + + // Finish with an exit code. + let code = reporter.finish(); + std::process::exit(code); + + value +} + +/// Drive a TCP listener at the specified port. +pub async fn run_tcp_listener( + port: u16, + reporter: impl Reporter + Send + 'static, + once_ready: Sender<()>, +) -> io::Result<()> { + // Bind to a listening port. + let listener = + async_net::TcpListener::bind((async_net::IpAddr::from([0u8, 0, 0, 0]), port)).await?; + + // Wait for the client to connect. + println!( + "{} {:?}{}", + "listening at".white().italic(), + listener.local_addr()?.cyan().bold(), + ", waiting for connection...".white().italic() + ); + once_ready.send(()).await.ok(); + let (socket, addr) = async { listener.accept().await } + .or(async { + // Five-minute timeout. + async_io::Timer::after(Duration::from_secs(60 * 5)).await; + Err(io::ErrorKind::TimedOut.into()) + }) + .await?; + drop(listener); + + println!( + "{}{:?}", + "got connection at address ".white().italic(), + addr.cyan().bold() + ); + + run_over_stream(socket, reporter).await +} + +/// Drive a Unix listener at the specified path. +#[cfg(unix)] +pub async fn run_unix_listener( + path: &std::path::Path, + reporter: impl Reporter + Send + 'static, + once_ready: Sender<()>, +) -> io::Result<()> { + use async_net::unix::UnixListener; + + // Bind to a listening port. + let listener = UnixListener::bind(path)?; + + // Wait for the client to connect. + println!( + "{} {:?}{}", + "listening at".white().italic(), + listener.local_addr()?.cyan().bold(), + ", waiting for connection...".white().italic() + ); + once_ready.send(()).await.ok(); + let (socket, addr) = async { listener.accept().await } + .or(async { + // Five-minute timeout. + async_io::Timer::after(Duration::from_secs(60 * 5)).await; + Err(io::ErrorKind::TimedOut.into()) + }) + .await?; + drop(listener); + + println!( + "{}{:?}", + "got connection at address ".white().italic(), + addr.cyan().bold() + ); + + run_over_stream(socket, reporter).await +} + +#[inline] +async fn run_over_stream( + mut socket: impl futures_lite::AsyncRead + Send + Unpin, + reporter: impl Reporter + Send + 'static, +) -> io::Result<()> { + // Start reading from the socket. + let mut buf = Vec::with_capacity(4096); + let reporter = Mutex::new(reporter); + let ex = async_executor::Executor::new(); + let mut handles = vec![]; + + ex.run({ + let ex = &ex; + let reporter = &reporter; + async move { + loop { + let mut bytes_to_read = [0u8; 8]; + + // Read number of bytes to read from the stream. + match socket.read_exact(&mut bytes_to_read).await { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e), + } + + // Read the remaining bytes in this packet. + buf.resize(u64::from_le_bytes(bytes_to_read) as usize, 0); + socket.read_exact(&mut buf).await?; + + // Parse to JSON. + let event: TestEvent = serde_json::from_slice(&buf).expect("failed to parse JSON"); + + // Spawn a task to write the event to the reporter. + handles.push(ex.spawn(async move { + let mut reporter = reporter.lock().await; + reporter.report(event).await; + })); + } + + // Wait for all of the tasks to finish. + for handle in handles { + handle.await; + } + + Ok(()) + } + }) + .await +} diff --git a/winit-test/src/reporter/console.rs b/winit-test/src/reporter/console.rs new file mode 100644 index 0000000..b91e6fe --- /dev/null +++ b/winit-test/src/reporter/console.rs @@ -0,0 +1,138 @@ +// MIT/Apache2 + +use super::Reporter; +use crate::{TestEvent, TestResult, TestStatus}; + +use futures_lite::prelude::*; +use owo_colors::OwoColorize; + +use std::borrow::Cow; +use std::fmt::{self, Write as _}; +use std::future::Future; +use std::io::{self, prelude::*}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +/// Report tests to the output console. +pub struct ConsoleReporter(Arc>); + +struct Inner { + /// The current exit code. + exit_code: i32, + + /// Current indentation. + indent: usize, + + /// Look for test failures. + failures: Vec<(Cow<'static, str>, Cow<'static, str>)>, +} + +impl Default for ConsoleReporter { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl ConsoleReporter { + /// Create a new `ConsoleReporter`. + #[inline] + pub fn new() -> Self { + Self(Arc::new(Mutex::new(Inner { + exit_code: 0, + indent: 0, + failures: vec![], + }))) + } +} + +impl Reporter for ConsoleReporter { + #[inline] + fn report(&mut self, test: TestEvent) -> Pin + Send + 'static>> { + let this = self.0.clone(); + blocking::unblock(move || { + let mut this = this.lock().unwrap(); + let mut cout = io::stdout().lock(); + + match test { + TestEvent::End { count: _ } => { + if this.exit_code == 0 { + writeln!(cout, "{}{}", "test result: ".white(), "ok".green()).unwrap(); + } else { + writeln!(cout, "{}{}", "test result: ".white(), "FAILED".red()).unwrap(); + } + } + TestEvent::BeginGroup { name, count } => { + writeln!( + cout, + "{}{}{}{}{}{}", + Indent(this.indent), + "running test group '".white().italic(), + name.cyan().bold(), + "' with ".white().italic(), + count.cyan().bold(), + " tests...".white().italic() + ) + .unwrap(); + + this.indent += 1; + } + TestEvent::EndGroup(_name) => { + this.indent -= 1; + } + TestEvent::Result(TestResult { + name, + status, + failure, + }) => { + write!( + cout, + "{}{}{}{}", + Indent(this.indent), + "test ".white(), + name.bold().white(), + "... ".white() + ) + .unwrap(); + + match status { + TestStatus::Failed => { + this.failures.push((name, failure)); + this.exit_code = 1; + writeln!(cout, "{}", "ok".green().bold()).unwrap(); + } + + TestStatus::Ignored => { + writeln!(cout, "{}", "ignored".yellow().bold()).unwrap(); + } + + TestStatus::Success => { + writeln!(cout, "{}", "ok".green().bold()).unwrap(); + } + } + } + } + }) + .boxed() + } + + #[inline] + fn finish(&mut self) -> i32 { + self.0.lock().unwrap().exit_code + } +} + +const SPACES_PER_INDENT: usize = 2; + +struct Indent(usize); + +impl fmt::Display for Indent { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let spaces = self.0 * SPACES_PER_INDENT; + for _ in 0..spaces { + f.write_char(' ')?; + } + Ok(()) + } +} diff --git a/winit-test/src/reporter/dump.rs b/winit-test/src/reporter/dump.rs new file mode 100644 index 0000000..7004328 --- /dev/null +++ b/winit-test/src/reporter/dump.rs @@ -0,0 +1,36 @@ +// MIT/Apache2 License + +//! Dump JSON output to the console. +//! +//! This is needed for transmitting details for the Android implementation, +//! unfortunately. + +use super::Reporter; + +/// Report tests to the output console in JSON form. +pub struct DumpReporter; + +impl DumpReporter { + #[inline] + pub fn new() -> Self { + Self + } +} + +impl Reporter for DumpReporter { + #[inline] + fn report( + &mut self, + test: crate::TestEvent, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(blocking::unblock(move || { + let data = serde_json::to_string(&test).unwrap(); + println!("winit_TEST_DUMP({data})winit_TEST_DUMP"); + })) + } + + #[inline] + fn finish(&mut self) -> i32 { + 0 + } +} diff --git a/winit-test/src/reporter/mod.rs b/winit-test/src/reporter/mod.rs new file mode 100644 index 0000000..88feefb --- /dev/null +++ b/winit-test/src/reporter/mod.rs @@ -0,0 +1,25 @@ +// MIT/Apache2 License + +//! The trait for reporting error results. + +use super::TestEvent; + +use std::future::Future; +use std::pin::Pin; + +mod console; +mod dump; +mod writer; + +pub use console::ConsoleReporter; +pub use dump::DumpReporter; +pub use writer::StreamReporter; + +/// Something that receives test results. +pub trait Reporter { + /// Report a test event. + fn report(&mut self, test: TestEvent) -> Pin + Send + '_>>; + + /// Finish our report, returning an exit code. + fn finish(&mut self) -> i32; +} diff --git a/winit-test/src/reporter/writer.rs b/winit-test/src/reporter/writer.rs new file mode 100644 index 0000000..c73e0eb --- /dev/null +++ b/winit-test/src/reporter/writer.rs @@ -0,0 +1,71 @@ +// MIT/Apache2 License + +/// Implements the `Reporter` trait around something that can asynchronously write. +use super::Reporter; +use crate::{TestEvent, TestResult, TestStatus}; + +use futures_lite::prelude::*; + +use std::io; +use std::time::Duration; + +/// A reporter that runs over a stream. +pub struct StreamReporter { + /// Current exit code. + exit_code: i32, + + /// TCP stream to send data over. + socket: S, +} + +impl StreamReporter { + /// Connect to the given address. + #[inline] + pub async fn connect( + stream: impl Future>, + timeout: Duration, + ) -> io::Result { + let timeout = async move { + async_io::Timer::after(timeout).await; + Err(io::ErrorKind::TimedOut.into()) + }; + + let socket = stream.or(timeout).await?; + Ok(Self { + socket, + exit_code: 0, + }) + } +} + +impl Reporter for StreamReporter { + fn report( + &mut self, + test: TestEvent, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(async move { + if let TestEvent::Result(TestResult { + status: TestStatus::Failed, + .. + }) = &test + { + self.exit_code = 1; + } + + // The format is the JSON bytes preceded by the number of bytes expected. + let mut bytes = serde_json::to_vec(&test).expect("failed to serialize TestEvent"); + let count = bytes.len() as u64; + bytes.splice(0..0, count.to_le_bytes()); + + // Write these bytes to the stream. + self.socket + .write_all(&bytes) + .await + .expect("failed to write to other end of stream"); + }) + } + + fn finish(&mut self) -> i32 { + 0 + } +}