Merge nucleic/sleek-thistle-egret-fyej into dev
This commit is contained in:
@@ -0,0 +1,374 @@
|
||||
//! Comparison types for test results.
|
||||
|
||||
use std::{io::Read, path::PathBuf, process::ExitStatus};
|
||||
|
||||
/// Comparison of durations between oracle and test runs.
|
||||
#[derive(Default)]
|
||||
pub struct DurationComparison {
|
||||
/// Duration of the oracle run.
|
||||
pub oracle: std::time::Duration,
|
||||
/// Duration of the test run.
|
||||
pub test: std::time::Duration,
|
||||
}
|
||||
|
||||
/// Comparison of exit statuses.
|
||||
pub enum ExitStatusComparison {
|
||||
/// Exit status was ignored.
|
||||
Ignored,
|
||||
/// Exit statuses match.
|
||||
Same(ExitStatus),
|
||||
/// Exit statuses differ.
|
||||
TestDiffers {
|
||||
/// Exit status from the test shell.
|
||||
test_exit_status: ExitStatus,
|
||||
/// Exit status from the oracle shell.
|
||||
oracle_exit_status: ExitStatus,
|
||||
},
|
||||
}
|
||||
|
||||
impl ExitStatusComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::TestDiffers {
|
||||
test_exit_status: _,
|
||||
oracle_exit_status: _
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Comparison of string outputs (stdout/stderr).
|
||||
pub enum StringComparison {
|
||||
/// Output was ignored.
|
||||
Ignored {
|
||||
/// Output from the test shell.
|
||||
test_string: String,
|
||||
/// Output from the oracle shell.
|
||||
oracle_string: String,
|
||||
},
|
||||
/// Outputs match.
|
||||
Same(String),
|
||||
/// Outputs differ.
|
||||
TestDiffers {
|
||||
/// Output from the test shell.
|
||||
test_string: String,
|
||||
/// Output from the oracle shell.
|
||||
oracle_string: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl StringComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::TestDiffers {
|
||||
test_string: _,
|
||||
oracle_string: _
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A single entry in a directory comparison.
|
||||
pub enum DirComparisonEntry {
|
||||
/// File exists only in the left (oracle) directory.
|
||||
LeftOnly(PathBuf),
|
||||
/// File exists only in the right (test) directory.
|
||||
RightOnly(PathBuf),
|
||||
/// Files differ between directories.
|
||||
Different(PathBuf, String, PathBuf, String),
|
||||
}
|
||||
|
||||
/// Comparison of directory contents.
|
||||
pub enum DirComparison {
|
||||
/// Directory comparison was ignored.
|
||||
Ignored,
|
||||
/// Directory contents match.
|
||||
Same,
|
||||
/// Directory contents differ.
|
||||
TestDiffers(Vec<DirComparisonEntry>),
|
||||
}
|
||||
|
||||
impl DirComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(self, Self::TestDiffers(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// Full comparison between oracle and test shell runs.
|
||||
pub struct OracleComparison {
|
||||
/// Comparison of exit statuses.
|
||||
pub exit_status: ExitStatusComparison,
|
||||
/// Comparison of stdout.
|
||||
pub stdout: StringComparison,
|
||||
/// Comparison of stderr.
|
||||
pub stderr: StringComparison,
|
||||
/// Comparison of temporary directory contents.
|
||||
pub temp_dir: DirComparison,
|
||||
/// Comparison of durations.
|
||||
pub duration: DurationComparison,
|
||||
}
|
||||
|
||||
impl OracleComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
self.exit_status.is_failure()
|
||||
|| self.stdout.is_failure()
|
||||
|| self.stderr.is_failure()
|
||||
|| self.temp_dir.is_failure()
|
||||
}
|
||||
|
||||
/// Creates an ignored comparison (all fields ignored).
|
||||
pub fn ignored() -> Self {
|
||||
Self {
|
||||
exit_status: ExitStatusComparison::Ignored,
|
||||
stdout: StringComparison::Ignored {
|
||||
test_string: String::new(),
|
||||
oracle_string: String::new(),
|
||||
},
|
||||
stderr: StringComparison::Ignored {
|
||||
test_string: String::new(),
|
||||
oracle_string: String::new(),
|
||||
},
|
||||
temp_dir: DirComparison::Ignored,
|
||||
duration: DurationComparison::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Comparison of a single expectation.
|
||||
#[derive(Debug)]
|
||||
pub enum SingleExpectationComparison {
|
||||
/// Expectation was not specified (ignored).
|
||||
NotSpecified,
|
||||
/// Actual matches expected.
|
||||
Matches,
|
||||
/// Actual differs from expected.
|
||||
Differs {
|
||||
/// The expected value.
|
||||
expected: String,
|
||||
/// The actual value.
|
||||
actual: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl SingleExpectationComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(self, Self::Differs { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// Comparison against inline expectations.
|
||||
#[derive(Debug)]
|
||||
pub struct ExpectationComparison {
|
||||
/// Comparison of exit code.
|
||||
pub exit_code: SingleExpectationComparison,
|
||||
/// Comparison of stdout.
|
||||
pub stdout: SingleExpectationComparison,
|
||||
/// Comparison of stderr.
|
||||
pub stderr: SingleExpectationComparison,
|
||||
/// Whether snapshot comparison was used.
|
||||
pub snapshot_used: bool,
|
||||
/// Snapshot comparison result (if used).
|
||||
pub snapshot_result: Option<SnapshotResult>,
|
||||
}
|
||||
|
||||
impl ExpectationComparison {
|
||||
/// Creates an empty expectation comparison (all not specified).
|
||||
pub const fn not_specified() -> Self {
|
||||
Self {
|
||||
exit_code: SingleExpectationComparison::NotSpecified,
|
||||
stdout: SingleExpectationComparison::NotSpecified,
|
||||
stderr: SingleExpectationComparison::NotSpecified,
|
||||
snapshot_used: false,
|
||||
snapshot_result: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub fn is_failure(&self) -> bool {
|
||||
self.exit_code.is_failure()
|
||||
|| self.stdout.is_failure()
|
||||
|| self.stderr.is_failure()
|
||||
|| self
|
||||
.snapshot_result
|
||||
.as_ref()
|
||||
.is_some_and(|r| r.is_failure())
|
||||
}
|
||||
|
||||
/// Returns whether any expectations were checked.
|
||||
pub const fn has_any_checks(&self) -> bool {
|
||||
!matches!(self.exit_code, SingleExpectationComparison::NotSpecified)
|
||||
|| !matches!(self.stdout, SingleExpectationComparison::NotSpecified)
|
||||
|| !matches!(self.stderr, SingleExpectationComparison::NotSpecified)
|
||||
|| self.snapshot_used
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a snapshot comparison.
|
||||
#[derive(Debug)]
|
||||
pub enum SnapshotResult {
|
||||
/// Snapshot matches.
|
||||
Matches,
|
||||
/// Snapshot differs (new snapshot created or update needed).
|
||||
Differs {
|
||||
/// Description of the difference.
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl SnapshotResult {
|
||||
/// Returns whether this result indicates a failure.
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(self, Self::Differs { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// Combined test comparison result.
|
||||
pub struct TestComparison {
|
||||
/// Oracle comparison (if oracle mode was used).
|
||||
pub oracle: Option<OracleComparison>,
|
||||
/// Expectation comparison (if expectations were defined).
|
||||
pub expectation: ExpectationComparison,
|
||||
/// Duration of the test run.
|
||||
pub duration: std::time::Duration,
|
||||
}
|
||||
|
||||
impl TestComparison {
|
||||
/// Returns whether this comparison indicates a failure.
|
||||
pub fn is_failure(&self) -> bool {
|
||||
self.oracle.as_ref().is_some_and(|o| o.is_failure()) || self.expectation.is_failure()
|
||||
}
|
||||
|
||||
/// Creates a skipped comparison.
|
||||
pub fn skipped() -> Self {
|
||||
Self {
|
||||
oracle: None,
|
||||
expectation: ExpectationComparison::not_specified(),
|
||||
duration: std::time::Duration::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compares two strings, optionally ignoring whitespace.
|
||||
pub fn output_matches(oracle: &str, test: &str, ignore_whitespace: bool) -> bool {
|
||||
if ignore_whitespace {
|
||||
let whitespace_re = regex::Regex::new(r"\s+").unwrap();
|
||||
|
||||
let cleaned_oracle = whitespace_re.replace_all(oracle, " ").to_string();
|
||||
let cleaned_test = whitespace_re.replace_all(test, " ").to_string();
|
||||
|
||||
cleaned_oracle == cleaned_test
|
||||
} else {
|
||||
oracle == test
|
||||
}
|
||||
}
|
||||
|
||||
/// Compares directory contents between oracle and test.
|
||||
pub fn diff_dirs(
|
||||
oracle_path: &std::path::Path,
|
||||
test_path: &std::path::Path,
|
||||
) -> anyhow::Result<DirComparison> {
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
|
||||
fn get_dir_entries(
|
||||
dir_path: &std::path::Path,
|
||||
) -> anyhow::Result<HashMap<String, fs::FileType>> {
|
||||
let mut entries = HashMap::new();
|
||||
for entry in fs::read_dir(dir_path)? {
|
||||
let entry = entry?;
|
||||
let file_type = entry.file_type()?;
|
||||
let filename = entry.file_name().to_string_lossy().to_string();
|
||||
|
||||
// Ignore raw coverage profile data files.
|
||||
if filename.ends_with(".profraw") {
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.insert(filename, file_type);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
let mut entries = vec![];
|
||||
|
||||
let oracle_entries = get_dir_entries(oracle_path)?;
|
||||
let test_entries = get_dir_entries(test_path)?;
|
||||
|
||||
// Look through all the files in the oracle directory
|
||||
for (filename, file_type) in &oracle_entries {
|
||||
if !test_entries.contains_key(filename) {
|
||||
for left_only_file in walkdir::WalkDir::new(oracle_path.join(filename)) {
|
||||
let entry = left_only_file?;
|
||||
let left_only_path = entry.path();
|
||||
entries.push(DirComparisonEntry::LeftOnly(left_only_path.to_owned()));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
let oracle_file_path = oracle_path.join(filename);
|
||||
let test_file_path = test_path.join(filename);
|
||||
|
||||
if file_type.is_file() {
|
||||
let mut oracle_file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(&oracle_file_path)?;
|
||||
let mut oracle_bytes = vec![];
|
||||
oracle_file.read_to_end(&mut oracle_bytes)?;
|
||||
|
||||
let mut test_file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(&test_file_path)?;
|
||||
let mut test_bytes = vec![];
|
||||
test_file.read_to_end(&mut test_bytes)?;
|
||||
|
||||
if oracle_bytes != test_bytes {
|
||||
let oracle_display_text = String::from_utf8_lossy(&oracle_bytes);
|
||||
let test_display_text = String::from_utf8_lossy(&test_bytes);
|
||||
|
||||
entries.push(DirComparisonEntry::Different(
|
||||
oracle_file_path,
|
||||
oracle_display_text.to_string(),
|
||||
test_file_path,
|
||||
test_display_text.to_string(),
|
||||
));
|
||||
}
|
||||
} else if file_type.is_dir() {
|
||||
let subdir_comparison =
|
||||
diff_dirs(oracle_file_path.as_path(), test_file_path.as_path())?;
|
||||
if let DirComparison::TestDiffers(subdir_entries) = subdir_comparison {
|
||||
entries.extend(subdir_entries);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (filename, file_type) in &test_entries {
|
||||
if oracle_entries.contains_key(filename) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if file_type.is_dir() {
|
||||
for right_only_file in walkdir::WalkDir::new(test_path.join(filename)) {
|
||||
let entry = right_only_file?;
|
||||
let right_only_path = entry.path();
|
||||
entries.push(DirComparisonEntry::RightOnly(right_only_path.to_owned()));
|
||||
}
|
||||
} else {
|
||||
entries.push(DirComparisonEntry::RightOnly(test_path.join(filename)));
|
||||
}
|
||||
}
|
||||
|
||||
if entries.is_empty() {
|
||||
Ok(DirComparison::Same)
|
||||
} else {
|
||||
Ok(DirComparison::TestDiffers(entries))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
//! Configuration types for the test harness.
|
||||
|
||||
use clap::Parser;
|
||||
use std::{collections::HashSet, ffi::OsString, path::PathBuf};
|
||||
|
||||
/// Which shell to use for a test.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum WhichShell {
|
||||
/// The shell under test (brush).
|
||||
ShellUnderTest(PathBuf),
|
||||
/// A named shell (e.g., bash, sh).
|
||||
NamedShell(PathBuf),
|
||||
}
|
||||
|
||||
/// Configuration for a shell.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ShellConfig {
|
||||
/// Which shell this is.
|
||||
pub which: WhichShell,
|
||||
/// Default arguments to pass to this shell.
|
||||
pub default_args: Vec<String>,
|
||||
/// Default PATH variable for this shell.
|
||||
pub default_path_var: Option<String>,
|
||||
/// Optional launcher command to prepend (e.g., `["wasmtime", "run", "--"]` for wasm
|
||||
/// targets). The first element is the program to execute; the rest are leading arguments
|
||||
/// inserted before the shell binary path.
|
||||
pub launcher: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl ShellConfig {
|
||||
/// Computes the PATH variable to use for tests.
|
||||
pub fn compute_test_path_var(&self) -> OsString {
|
||||
let mut dirs = vec![];
|
||||
|
||||
// Start with any default we were provided.
|
||||
if let Some(default_path_var) = &self.default_path_var {
|
||||
dirs.extend(std::env::split_paths(default_path_var));
|
||||
}
|
||||
|
||||
// Add hard-coded paths that will work on *most* Unix-like systems.
|
||||
dirs.extend([
|
||||
"/usr/local/sbin".into(),
|
||||
"/usr/local/bin".into(),
|
||||
"/usr/sbin".into(),
|
||||
"/usr/bin".into(),
|
||||
"/sbin".into(),
|
||||
"/bin".into(),
|
||||
]);
|
||||
|
||||
// Handle systems that store their standard POSIX binaries elsewhere.
|
||||
// For example, NixOS has an interesting set of paths that must be consulted.
|
||||
if let Some(host_path) = std::env::var_os("PATH") {
|
||||
for path in std::env::split_paths(&host_path) {
|
||||
if !dirs.contains(&path) && path.join("sh").is_file() {
|
||||
dirs.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::env::join_paths(dirs).unwrap_or_else(|_| PathBuf::from("").into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for the oracle shell (e.g., bash).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OracleConfig {
|
||||
/// Name of this oracle configuration (e.g., "bash", "sh").
|
||||
pub name: String,
|
||||
/// Shell configuration for the oracle.
|
||||
pub shell: ShellConfig,
|
||||
/// Version string of the oracle.
|
||||
pub version_str: Option<String>,
|
||||
}
|
||||
|
||||
/// The mode in which to run tests.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum TestMode {
|
||||
/// Compare test shell output against an oracle shell.
|
||||
#[default]
|
||||
Oracle,
|
||||
/// Validate against inline expectations or snapshots only.
|
||||
Expectation,
|
||||
/// Both oracle comparison and expectation validation.
|
||||
Hybrid,
|
||||
}
|
||||
|
||||
/// Configuration for the test runner.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RunnerConfig {
|
||||
/// The test mode to use.
|
||||
pub mode: TestMode,
|
||||
/// Configuration for the oracle shell (if using oracle mode).
|
||||
pub oracle: Option<OracleConfig>,
|
||||
/// Configuration for the test shell (brush).
|
||||
pub test_shell: ShellConfig,
|
||||
/// Directory containing test case YAML files.
|
||||
pub test_cases_dir: PathBuf,
|
||||
/// Directory for storing snapshots (relative to test case YAML files).
|
||||
pub snapshot_dir_name: String,
|
||||
/// Host OS ID (for filtering incompatible tests).
|
||||
pub host_os_id: Option<String>,
|
||||
/// Active runtime platform tags (e.g., "wasi", "wasm"). Tests that
|
||||
/// declare any of these in `incompatible_platforms` will be skipped.
|
||||
pub platform_tags: HashSet<String>,
|
||||
}
|
||||
|
||||
impl RunnerConfig {
|
||||
/// Creates a new runner config with minimal safe defaults.
|
||||
///
|
||||
/// N.B. Callers typically override `test_shell` via
|
||||
/// `TestOptions::create_test_shell_config()`, which adds
|
||||
/// platform-appropriate flags like `--input-backend=basic`.
|
||||
pub fn new(test_shell_path: PathBuf, test_cases_dir: PathBuf) -> Self {
|
||||
Self {
|
||||
mode: TestMode::Expectation,
|
||||
oracle: None,
|
||||
test_shell: ShellConfig {
|
||||
which: WhichShell::ShellUnderTest(test_shell_path),
|
||||
default_args: vec![
|
||||
"--norc".into(),
|
||||
"--noprofile".into(),
|
||||
"--no-config".into(),
|
||||
"--disable-bracketed-paste".into(),
|
||||
"--disable-color".into(),
|
||||
],
|
||||
default_path_var: None,
|
||||
launcher: None,
|
||||
},
|
||||
test_cases_dir,
|
||||
snapshot_dir_name: String::from("snaps"),
|
||||
host_os_id: crate::util::get_host_os_id(),
|
||||
platform_tags: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the active runtime platform tags.
|
||||
#[must_use]
|
||||
pub fn with_platform_tags(mut self, tags: HashSet<String>) -> Self {
|
||||
self.platform_tags = tags;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the oracle configuration, enabling oracle comparison mode.
|
||||
#[must_use]
|
||||
pub fn with_oracle(mut self, oracle: OracleConfig) -> Self {
|
||||
self.oracle = Some(oracle);
|
||||
self.mode = TestMode::Oracle;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the test mode.
|
||||
#[must_use]
|
||||
pub const fn with_mode(mut self, mode: TestMode) -> Self {
|
||||
self.mode = mode;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the snapshot directory name.
|
||||
#[must_use]
|
||||
pub fn with_snapshot_dir_name(mut self, name: impl Into<String>) -> Self {
|
||||
self.snapshot_dir_name = name.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the default PATH variable for the test shell.
|
||||
#[must_use]
|
||||
pub fn with_test_path_var(mut self, path_var: Option<String>) -> Self {
|
||||
self.test_shell.default_path_var = path_var;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Output format for test results.
|
||||
#[derive(Clone, Copy, Default, clap::ValueEnum, Debug)]
|
||||
pub enum OutputFormat {
|
||||
/// Human-readable colored output.
|
||||
#[default]
|
||||
Pretty,
|
||||
/// `JUnit` XML format.
|
||||
Junit,
|
||||
/// Minimal output.
|
||||
Terse,
|
||||
}
|
||||
|
||||
/// Command-line options for the test harness.
|
||||
#[derive(Clone, Parser, Debug)]
|
||||
#[clap(version, about, disable_help_flag = true, disable_version_flag = true)]
|
||||
pub struct TestOptions {
|
||||
/// Display usage information.
|
||||
#[clap(long = "help", action = clap::ArgAction::HelpLong)]
|
||||
pub help: Option<bool>,
|
||||
|
||||
/// Output format for test results.
|
||||
#[clap(long = "format", default_value = "pretty")]
|
||||
pub format: OutputFormat,
|
||||
|
||||
/// Display full details on known failures.
|
||||
#[clap(long = "known-failure-details")]
|
||||
pub display_known_failure_details: bool,
|
||||
|
||||
/// Display details regarding successful test cases.
|
||||
#[clap(short = 'v', long = "verbose", env = "BRUSH_VERBOSE")]
|
||||
pub verbose: bool,
|
||||
|
||||
/// Enable a specific configuration.
|
||||
#[clap(long = "enable-config")]
|
||||
pub enabled_configs: Vec<String>,
|
||||
|
||||
/// List available tests without running them.
|
||||
#[clap(long = "list")]
|
||||
pub list_tests_only: bool,
|
||||
|
||||
/// Exactly match filters (not just substring match).
|
||||
#[clap(long = "exact")]
|
||||
pub exact_match: bool,
|
||||
|
||||
/// Optionally specify a non-default path for bash.
|
||||
#[clap(long = "bash-path", default_value = "bash", env = "BASH_PATH")]
|
||||
pub bash_path: PathBuf,
|
||||
|
||||
/// Optionally specify a non-default path for brush.
|
||||
#[clap(long = "brush-path", default_value = "", env = "BRUSH_PATH")]
|
||||
pub brush_path: String,
|
||||
|
||||
/// Optionally specify additional arguments for brush.
|
||||
#[clap(long = "brush-args", default_value = "", env = "BRUSH_ARGS")]
|
||||
pub brush_args: String,
|
||||
|
||||
/// Optionally specify a launcher command to prepend when invoking brush
|
||||
/// (e.g., "wasmtime run --" to execute a wasm build under wasmtime).
|
||||
/// The string is split on whitespace; the first token becomes the program
|
||||
/// to execute and the remainder are passed as leading arguments before
|
||||
/// the brush binary path.
|
||||
#[clap(long = "brush-launcher", default_value = "", env = "BRUSH_LAUNCHER")]
|
||||
pub brush_launcher: String,
|
||||
|
||||
/// Runtime platform tags (e.g., "wasi", "wasm") describing the
|
||||
/// environment in which brush is being executed. Test cases that
|
||||
/// declare any of these tags in `incompatible_platforms` will be
|
||||
/// skipped. May be specified multiple times on the CLI or as a
|
||||
/// space-separated value in the environment variable.
|
||||
#[clap(
|
||||
long = "brush-platform-tags",
|
||||
value_delimiter = ' ',
|
||||
env = "BRUSH_PLATFORM_TAGS"
|
||||
)]
|
||||
pub brush_platform_tags: Vec<String>,
|
||||
|
||||
/// Optionally specify path to test cases.
|
||||
#[clap(long = "test-cases-path", env = "BRUSH_TEST_CASES")]
|
||||
pub test_cases_path: Option<PathBuf>,
|
||||
|
||||
/// Optionally specify PATH variable to use in shells.
|
||||
#[clap(long = "test-path-var", env = "BRUSH_TEST_PATH_VAR")]
|
||||
pub test_path_var: Option<String>,
|
||||
|
||||
/// Show output from test cases (for compatibility only, has no effect).
|
||||
#[clap(long = "show-output")]
|
||||
pub show_output: bool,
|
||||
|
||||
/// Capture output? (for compatibility only, has no effect).
|
||||
#[clap(long = "nocapture")]
|
||||
pub no_capture: bool,
|
||||
|
||||
/// Colorize output? (for compatibility only, has no effect).
|
||||
#[clap(long = "color", default_value_t = clap::ColorChoice::Auto)]
|
||||
pub color: clap::ColorChoice,
|
||||
|
||||
/// Run skipped tests only.
|
||||
#[clap(long = "ignored")]
|
||||
pub skipped_tests_only: bool,
|
||||
|
||||
/// Unstable flags (for compatibility only, has no effect).
|
||||
#[clap(short = 'Z')]
|
||||
pub unstable_flag: Vec<String>,
|
||||
|
||||
/// Patterns for tests to be excluded.
|
||||
#[clap(long = "skip")]
|
||||
pub exclude_filters: Vec<String>,
|
||||
|
||||
/// Patterns for tests to be included.
|
||||
pub include_filters: Vec<String>,
|
||||
}
|
||||
|
||||
impl TestOptions {
|
||||
/// Returns the configured platform tags as a set.
|
||||
pub fn platform_tags(&self) -> HashSet<String> {
|
||||
self.brush_platform_tags.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Builds the default `ShellConfig` for the shell under test based on
|
||||
/// the common options (path, launcher, platform tags, extra args).
|
||||
///
|
||||
/// Resolves the launcher binary to an absolute path (if one is
|
||||
/// configured) because the test harness clears env vars — including
|
||||
/// `PATH` — before spawning child processes.
|
||||
pub fn create_test_shell_config(&self) -> anyhow::Result<ShellConfig> {
|
||||
let mut default_args: Vec<String> = vec![
|
||||
"--norc".into(),
|
||||
"--noprofile".into(),
|
||||
"--no-config".into(),
|
||||
"--disable-bracketed-paste".into(),
|
||||
"--disable-color".into(),
|
||||
];
|
||||
|
||||
// Use the basic input backend for native builds. WASI builds are
|
||||
// compiled with `--features minimal` which doesn't include the basic
|
||||
// backend, so passing this flag would cause a startup error. Omitting
|
||||
// it lets brush pick its own default (Minimal on wasm targets).
|
||||
if !self.platform_tags().contains("wasi") {
|
||||
default_args.push("--input-backend=basic".into());
|
||||
}
|
||||
|
||||
// Append any additional brush args specified by the caller.
|
||||
self.brush_args.split_whitespace().for_each(|arg| {
|
||||
default_args.push(arg.into());
|
||||
});
|
||||
|
||||
let launcher = if self.brush_launcher.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let mut tokens: Vec<String> = self
|
||||
.brush_launcher
|
||||
.split_whitespace()
|
||||
.map(Into::into)
|
||||
.collect();
|
||||
crate::util::resolve_launcher_path(&mut tokens)?;
|
||||
Some(tokens)
|
||||
};
|
||||
|
||||
Ok(ShellConfig {
|
||||
which: WhichShell::ShellUnderTest(PathBuf::from(&self.brush_path)),
|
||||
default_args,
|
||||
default_path_var: self.test_path_var.clone(),
|
||||
launcher,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the given config name should be enabled.
|
||||
pub fn should_enable_config(&self, config: &str, default_configs: &[&str]) -> bool {
|
||||
let enabled_configs = if self.enabled_configs.is_empty() {
|
||||
default_configs.iter().map(|s| String::from(*s)).collect()
|
||||
} else {
|
||||
self.enabled_configs.clone()
|
||||
};
|
||||
|
||||
enabled_configs.contains(&config.to_string())
|
||||
}
|
||||
|
||||
/// Returns whether a test should run based on include/exclude filters.
|
||||
pub fn should_run_test(&self, qualified_name: &str) -> bool {
|
||||
if self.include_filters.is_empty() && self.exclude_filters.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If any include filters were given, then we are in opt-in mode.
|
||||
if !self.include_filters.is_empty()
|
||||
&& !self.test_matches_filters(qualified_name, &self.include_filters)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// In all cases, exclude filters may be used to exclude tests.
|
||||
if !self.exclude_filters.is_empty()
|
||||
&& self.test_matches_filters(qualified_name, &self.exclude_filters)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn test_matches_filters(&self, qualified_test_name: &str, filters: &[String]) -> bool {
|
||||
if self.exact_match {
|
||||
filters.iter().any(|f| f == qualified_test_name)
|
||||
} else {
|
||||
filters
|
||||
.iter()
|
||||
.any(|filter| qualified_test_name.contains(filter))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Execution logic for running shell commands.
|
||||
|
||||
use crate::config::{ShellConfig, WhichShell};
|
||||
use crate::testcase::{ShellInvocation, TestCase, TestCaseSet, TestFile};
|
||||
use anyhow::{Context, Result};
|
||||
use assert_fs::fixture::{FileWriteStr, PathChild};
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::{fs::PermissionsExt, process::CommandExt, process::ExitStatusExt};
|
||||
use std::{path::PathBuf, process::ExitStatus};
|
||||
|
||||
/// Default timeout for test commands in seconds.
|
||||
pub const DEFAULT_TIMEOUT_IN_SECONDS: u64 = 15;
|
||||
|
||||
/// Result of running a shell command.
|
||||
#[derive(Debug)]
|
||||
pub struct RunResult {
|
||||
/// Exit status of the command.
|
||||
pub exit_status: ExitStatus,
|
||||
/// Standard output.
|
||||
pub stdout: String,
|
||||
/// Standard error.
|
||||
pub stderr: String,
|
||||
/// Duration of the command.
|
||||
pub duration: std::time::Duration,
|
||||
}
|
||||
|
||||
impl TestCase {
|
||||
/// Runs this test case with the given shell configuration.
|
||||
pub async fn run_shell(
|
||||
&self,
|
||||
shell_config: &ShellConfig,
|
||||
working_dir: &assert_fs::TempDir,
|
||||
) -> Result<RunResult> {
|
||||
let test_cmd = self.create_command_for_shell(shell_config, working_dir);
|
||||
|
||||
let result = if self.pty {
|
||||
self.run_command_with_pty(test_cmd).await?
|
||||
} else {
|
||||
self.run_command_with_stdin(test_cmd).await?
|
||||
};
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Creates the test files in the given temporary directory.
|
||||
pub fn create_test_files_in(
|
||||
&self,
|
||||
temp_dir: &assert_fs::TempDir,
|
||||
test_case_set: &TestCaseSet,
|
||||
) -> Result<()> {
|
||||
for test_file in test_case_set
|
||||
.common_test_files
|
||||
.iter()
|
||||
.chain(self.test_files.iter())
|
||||
{
|
||||
Self::create_test_file(temp_dir, test_file, &test_case_set.source_dir)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_test_file(
|
||||
temp_dir: &assert_fs::TempDir,
|
||||
test_file: &TestFile,
|
||||
source_dir: &std::path::Path,
|
||||
) -> Result<()> {
|
||||
let test_file_path = temp_dir.child(test_file.path.as_path());
|
||||
|
||||
if let Some(source_path) = &test_file.source_path {
|
||||
if !test_file.contents.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"test file {} has both contents and source_path",
|
||||
test_file_path.to_string_lossy()
|
||||
));
|
||||
}
|
||||
|
||||
if source_path.is_absolute() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"source_path {} is not a relative path",
|
||||
source_path.to_string_lossy()
|
||||
));
|
||||
}
|
||||
|
||||
let abs_source_path = source_dir.join(source_path);
|
||||
|
||||
let source_contents = std::fs::read_to_string(&abs_source_path)
|
||||
.with_context(|| format!("reading {}", abs_source_path.to_string_lossy()))?;
|
||||
|
||||
test_file_path.write_str(source_contents.as_str())?;
|
||||
} else {
|
||||
test_file_path.write_str(test_file.contents.as_str())?;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
if test_file.executable {
|
||||
// chmod u+x
|
||||
let mut perms = test_file_path.metadata()?.permissions();
|
||||
perms.set_mode(perms.mode() | 0o100);
|
||||
std::fs::set_permissions(test_file_path, perms)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Constructs a `Command` to invoke the given shell binary, optionally
|
||||
/// prepending a launcher (e.g., `["wasmtime", "run", "--"]`). When a
|
||||
/// launcher is provided, the first element becomes the program to execute
|
||||
/// and the rest are passed as leading arguments before the shell binary path.
|
||||
fn new_shell_command(
|
||||
shell_path: &std::path::Path,
|
||||
launcher: Option<&[String]>,
|
||||
) -> std::process::Command {
|
||||
if let Some([program, leading_args @ ..]) = launcher {
|
||||
let mut cmd = std::process::Command::new(program);
|
||||
cmd.args(leading_args);
|
||||
cmd.arg(shell_path);
|
||||
cmd
|
||||
} else {
|
||||
std::process::Command::new(shell_path)
|
||||
}
|
||||
}
|
||||
|
||||
fn create_command_for_shell(
|
||||
&self,
|
||||
shell_config: &ShellConfig,
|
||||
working_dir: &assert_fs::TempDir,
|
||||
) -> std::process::Command {
|
||||
let (mut test_cmd, coverage_target_dir) = match self.invocation {
|
||||
ShellInvocation::ExecShellBinary => match &shell_config.which {
|
||||
WhichShell::ShellUnderTest(name) => {
|
||||
let cli_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let default_target_dir = || cli_dir.parent().unwrap().join("target");
|
||||
let target_dir = std::env::var("CARGO_TARGET_DIR")
|
||||
.ok()
|
||||
.map_or_else(default_target_dir, PathBuf::from);
|
||||
(
|
||||
Self::new_shell_command(name, shell_config.launcher.as_deref()),
|
||||
Some(target_dir),
|
||||
)
|
||||
}
|
||||
// Launcher only applies to the shell under test; the oracle is invoked directly.
|
||||
WhichShell::NamedShell(name) => (Self::new_shell_command(name, None), None),
|
||||
},
|
||||
ShellInvocation::ExecScript(_) => unimplemented!("exec script test"),
|
||||
};
|
||||
|
||||
if matches!(shell_config.which, WhichShell::ShellUnderTest(_)) {
|
||||
for arg in &self.additional_test_args {
|
||||
test_cmd.arg(arg);
|
||||
}
|
||||
}
|
||||
|
||||
for arg in &shell_config.default_args {
|
||||
if !self.removed_default_args.contains(arg) {
|
||||
test_cmd.arg(arg);
|
||||
}
|
||||
}
|
||||
|
||||
// Clear all environment vars for consistency.
|
||||
test_cmd.args(&self.args).env_clear();
|
||||
|
||||
// Set locale to C for consistent behavior across systems.
|
||||
test_cmd.env("LC_ALL", "C");
|
||||
// Hard-code a well known prompt for PS1.
|
||||
test_cmd.env("PS1", "test$ ");
|
||||
// Try to get decent backtraces when problems get hit.
|
||||
test_cmd.env("RUST_BACKTRACE", "1");
|
||||
// Compute a PATH that contains what we need.
|
||||
test_cmd.env("PATH", shell_config.compute_test_path_var());
|
||||
|
||||
// Set up any env vars needed for collecting coverage data.
|
||||
if let Some(coverage_target_dir) = &coverage_target_dir {
|
||||
test_cmd.env("CARGO_LLVM_COV_TARGET_DIR", coverage_target_dir);
|
||||
test_cmd.env(
|
||||
"LLVM_PROFILE_FILE",
|
||||
coverage_target_dir.join("brush-%p-%40m.profraw"),
|
||||
);
|
||||
}
|
||||
|
||||
for (k, v) in &self.env {
|
||||
test_cmd.env(k, v);
|
||||
}
|
||||
|
||||
if let Some(home_dir) = &self.home_dir {
|
||||
let abs_home_dir = if home_dir.is_relative() {
|
||||
working_dir.join(home_dir)
|
||||
} else {
|
||||
home_dir.to_owned()
|
||||
};
|
||||
|
||||
test_cmd.env("HOME", abs_home_dir.to_string_lossy().to_string());
|
||||
}
|
||||
|
||||
test_cmd.current_dir(working_dir.to_string_lossy().to_string());
|
||||
|
||||
test_cmd
|
||||
}
|
||||
|
||||
#[expect(clippy::unused_async)]
|
||||
#[cfg(not(unix))]
|
||||
async fn run_command_with_pty(&self, _cmd: std::process::Command) -> Result<RunResult> {
|
||||
Err(anyhow::anyhow!("pty test not supported on this platform"))
|
||||
}
|
||||
|
||||
#[expect(clippy::unused_async)]
|
||||
#[cfg(unix)]
|
||||
async fn run_command_with_pty(&self, cmd: std::process::Command) -> Result<RunResult> {
|
||||
use crate::util::{make_expectrl_output_readable, read_expectrl_log};
|
||||
use expectrl::{Expect, process::Termios as _};
|
||||
|
||||
let mut log = Vec::new();
|
||||
let writer = std::io::Cursor::new(&mut log);
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut p = expectrl::session::log(expectrl::Session::spawn(cmd)?, writer)?;
|
||||
p.set_echo(true)?;
|
||||
|
||||
if let Some(stdin) = &self.stdin {
|
||||
for line in stdin.lines() {
|
||||
if let Some(expectation) = line.strip_prefix("#expect:") {
|
||||
if let Err(inner) = p.expect(expectation) {
|
||||
return Ok(RunResult {
|
||||
exit_status: ExitStatus::from_raw(1),
|
||||
stdout: read_expectrl_log(log).unwrap_or_default(),
|
||||
stderr: std::format!("failed to expect '{expectation}': {inner}"),
|
||||
duration: start_time.elapsed(),
|
||||
});
|
||||
}
|
||||
} else if let Some(control_code) = line.strip_prefix("#send:") {
|
||||
match control_code.to_lowercase().as_str() {
|
||||
"ctrl+d" => p.send(expectrl::ControlCode::EndOfTransmission)?,
|
||||
"tab" => p.send(expectrl::ControlCode::HorizontalTabulation)?,
|
||||
"enter" => p.send(expectrl::ControlCode::LineFeed)?,
|
||||
_ => (),
|
||||
}
|
||||
} else if line.trim() == "#expect-prompt" {
|
||||
if let Err(inner) = p.expect("test$ ") {
|
||||
return Ok(RunResult {
|
||||
exit_status: ExitStatus::from_raw(1),
|
||||
stdout: read_expectrl_log(log).unwrap_or_default(),
|
||||
stderr: std::format!("failed to expect prompt: {inner}"),
|
||||
duration: start_time.elapsed(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
p.send(line)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(inner) = p.expect(expectrl::Eof) {
|
||||
return Ok(RunResult {
|
||||
exit_status: ExitStatus::from_raw(1),
|
||||
stdout: read_expectrl_log(log).unwrap_or_default(),
|
||||
stderr: std::format!("failed to expect EOF: {inner}"),
|
||||
duration: start_time.elapsed(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut wait_status = p.get_process().status()?;
|
||||
|
||||
if matches!(wait_status, expectrl::process::unix::WaitStatus::StillAlive) {
|
||||
// Try to terminate it safely.
|
||||
p.get_process_mut()
|
||||
.kill(expectrl::process::unix::Signal::SIGTERM)?;
|
||||
wait_status = p.get_process().wait()?;
|
||||
}
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
let output = read_expectrl_log(log)?;
|
||||
let cleaned = make_expectrl_output_readable(output);
|
||||
|
||||
match wait_status {
|
||||
expectrl::process::unix::WaitStatus::Exited(_, code) => Ok(RunResult {
|
||||
exit_status: ExitStatus::from_raw(code),
|
||||
stdout: cleaned,
|
||||
stderr: String::new(),
|
||||
duration,
|
||||
}),
|
||||
expectrl::process::unix::WaitStatus::Signaled(_, _, _) => {
|
||||
Err(anyhow::anyhow!("process was signaled"))
|
||||
}
|
||||
_ => Err(anyhow::anyhow!(
|
||||
"unexpected status for process: {wait_status:?}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(clippy::unused_async)]
|
||||
#[allow(unused_mut, reason = "only mutated on some platforms")]
|
||||
async fn run_command_with_stdin(&self, mut cmd: std::process::Command) -> Result<RunResult> {
|
||||
// SAFETY:
|
||||
// To avoid bash trying to directly access /dev/tty and generate tty-related signals,
|
||||
// we create a new session for the child process. The standard library has a setsid()
|
||||
// API but it's unstable, so we use nix here. Calling pre_exec can be unsafe as
|
||||
// it runs in the child process after fork() but before exec(), and there are constraints
|
||||
// around what can be safely done in that context. However, calling setsid() is generally
|
||||
// considered safe as it doesn't allocate memory or perform complex operations to forked
|
||||
// state.
|
||||
#[cfg(unix)]
|
||||
unsafe {
|
||||
cmd.pre_exec(|| {
|
||||
let _ = nix::unistd::setsid();
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
|
||||
let mut test_cmd = assert_cmd::Command::from_std(cmd);
|
||||
|
||||
test_cmd.timeout(std::time::Duration::from_secs(
|
||||
self.timeout_in_seconds
|
||||
.unwrap_or(DEFAULT_TIMEOUT_IN_SECONDS),
|
||||
));
|
||||
|
||||
if let Some(stdin) = &self.stdin {
|
||||
test_cmd.write_stdin(stdin.as_bytes());
|
||||
}
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
let cmd_result = test_cmd.output()?;
|
||||
let duration = start_time.elapsed();
|
||||
|
||||
Ok(RunResult {
|
||||
exit_status: cmd_result.status,
|
||||
stdout: String::from_utf8_lossy(cmd_result.stdout.as_slice()).to_string(),
|
||||
stderr: String::from_utf8_lossy(cmd_result.stderr.as_slice()).to_string(),
|
||||
duration,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
//! Test harness library for brush shell integration tests.
|
||||
//!
|
||||
//! This crate provides a unified framework for running YAML-based integration tests
|
||||
//! that support both oracle-based comparison (comparing brush output against bash)
|
||||
//! and expectation-based testing (inline expectations or insta snapshots).
|
||||
//!
|
||||
//! # Modes of Operation
|
||||
//!
|
||||
//! 1. **Oracle comparison**: Runs both an oracle shell (e.g., bash) and the test shell (brush),
|
||||
//! comparing their outputs. This is the traditional compatibility testing mode.
|
||||
//!
|
||||
//! 2. **Expectation-based**: Runs only the test shell and compares against inline expectations
|
||||
//! specified in the YAML or against insta snapshots.
|
||||
//!
|
||||
//! 3. **Hybrid**: Combines both modes - runs oracle comparison AND validates against expectations.
|
||||
//! Both must pass for the test to succeed.
|
||||
|
||||
#![cfg(any(unix, windows))]
|
||||
#![allow(clippy::missing_panics_doc)]
|
||||
#![allow(clippy::unwrap_used)]
|
||||
|
||||
mod comparison;
|
||||
mod config;
|
||||
mod execution;
|
||||
mod reporting;
|
||||
mod runner;
|
||||
mod testcase;
|
||||
pub mod util;
|
||||
|
||||
pub use comparison::{
|
||||
DirComparison, DirComparisonEntry, DurationComparison, ExitStatusComparison,
|
||||
ExpectationComparison, OracleComparison, SingleExpectationComparison, SnapshotResult,
|
||||
StringComparison, TestComparison,
|
||||
};
|
||||
pub use config::{
|
||||
OracleConfig, OutputFormat, RunnerConfig, ShellConfig, TestMode, TestOptions, WhichShell,
|
||||
};
|
||||
pub use execution::RunResult;
|
||||
pub use runner::TestRunner;
|
||||
pub use testcase::{ShellInvocation, TestCase, TestCaseSet, TestFile};
|
||||
@@ -0,0 +1,486 @@
|
||||
//! Reporting utilities for test results.
|
||||
|
||||
use crate::comparison::{
|
||||
DirComparison, DirComparisonEntry, ExitStatusComparison, ExpectationComparison,
|
||||
OracleComparison, SingleExpectationComparison, StringComparison, TestComparison,
|
||||
};
|
||||
use crate::config::{OutputFormat, TestOptions};
|
||||
use crate::util::{make_expectrl_output_readable, write_diff};
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
use std::io::Write;
|
||||
|
||||
/// Result of running a single test case.
|
||||
pub struct TestCaseResult {
|
||||
/// Name of the test case.
|
||||
pub name: Option<String>,
|
||||
/// Whether the test succeeded.
|
||||
pub success: bool,
|
||||
/// Whether the test was skipped.
|
||||
pub skip: bool,
|
||||
/// Whether this is a known failure.
|
||||
pub known_failure: bool,
|
||||
/// The comparison result.
|
||||
pub comparison: TestComparison,
|
||||
}
|
||||
|
||||
impl TestCaseResult {
|
||||
/// Reports this result in pretty format.
|
||||
pub fn report_pretty(&self, options: &TestOptions) -> Result<()> {
|
||||
self.write_details(std::io::stderr(), options)
|
||||
}
|
||||
|
||||
/// Writes the details of this result to a writer.
|
||||
pub fn write_details<W: Write>(&self, mut writer: W, options: &TestOptions) -> Result<()> {
|
||||
if self.skip {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !options.verbose {
|
||||
if (!self.comparison.is_failure() && !self.known_failure)
|
||||
|| (self.comparison.is_failure() && self.known_failure)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
write!(
|
||||
writer,
|
||||
"* {}: [{}]... ",
|
||||
"Test case".bright_yellow(),
|
||||
self.name
|
||||
.as_ref()
|
||||
.map_or_else(|| "(unnamed)", |n| n.as_str())
|
||||
.italic()
|
||||
)?;
|
||||
|
||||
if !self.comparison.is_failure() {
|
||||
if self.known_failure {
|
||||
writeln!(writer, "{}", "unexpected success.".bright_red())?;
|
||||
} else {
|
||||
writeln!(writer, "{}", "ok.".bright_green())?;
|
||||
return Ok(());
|
||||
}
|
||||
} else if self.known_failure {
|
||||
writeln!(writer, "{}", "known failure.".bright_magenta())?;
|
||||
if !options.display_known_failure_details {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(writer)?;
|
||||
|
||||
// Report oracle comparison if present
|
||||
if let Some(oracle) = &self.comparison.oracle {
|
||||
self.write_oracle_details(&mut writer, oracle, options)?;
|
||||
}
|
||||
|
||||
// Report expectation comparison
|
||||
if self.comparison.expectation.has_any_checks() {
|
||||
self.write_expectation_details(&mut writer, &self.comparison.expectation)?;
|
||||
}
|
||||
|
||||
if !self.success {
|
||||
writeln!(writer, " {}", "FAILED.".bright_red())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_lines)]
|
||||
#[expect(clippy::unused_self)]
|
||||
fn write_oracle_details<W: Write>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
oracle: &OracleComparison,
|
||||
options: &TestOptions,
|
||||
) -> Result<()> {
|
||||
writeln!(writer, " {} comparison:", "Oracle".cyan())?;
|
||||
|
||||
match oracle.exit_status {
|
||||
ExitStatusComparison::Ignored => writeln!(writer, " status {}", "ignored".cyan())?,
|
||||
ExitStatusComparison::Same(status) => {
|
||||
writeln!(
|
||||
writer,
|
||||
" status matches ({}) {}",
|
||||
format!("{status}").green(),
|
||||
"✔️".green()
|
||||
)?;
|
||||
}
|
||||
ExitStatusComparison::TestDiffers {
|
||||
test_exit_status,
|
||||
oracle_exit_status,
|
||||
} => {
|
||||
writeln!(
|
||||
writer,
|
||||
" status mismatch: {} from oracle vs. {} from test",
|
||||
format!("{oracle_exit_status}").cyan(),
|
||||
format!("{test_exit_status}").bright_red()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.stdout {
|
||||
StringComparison::Ignored {
|
||||
test_string,
|
||||
oracle_string,
|
||||
} => {
|
||||
writeln!(writer, " stdout {}", "ignored".cyan())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, oracle_string))?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle: stdout [cleaned]------------------------".cyan()
|
||||
)?;
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
indent::indent_all_by(10, make_expectrl_output_readable(oracle_string))
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, test_string))?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Test: stdout [cleaned]------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
indent::indent_all_by(10, make_expectrl_output_readable(test_string))
|
||||
)?;
|
||||
}
|
||||
StringComparison::Same(s) => {
|
||||
writeln!(writer, " stdout matches {}", "✔️".green())?;
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, s))?;
|
||||
}
|
||||
}
|
||||
StringComparison::TestDiffers {
|
||||
test_string: t,
|
||||
oracle_string: o,
|
||||
} => {
|
||||
writeln!(writer, " stdout {}", "DIFFERS:".bright_red())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
write_diff(writer, 10, o.as_str(), t.as_str())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------".cyan()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.stderr {
|
||||
StringComparison::Ignored { .. } => {
|
||||
writeln!(writer, " stderr {}", "ignored".cyan())?;
|
||||
}
|
||||
StringComparison::Same(s) => {
|
||||
writeln!(writer, " stderr matches {}", "✔️".green())?;
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stderr ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, s))?;
|
||||
}
|
||||
}
|
||||
StringComparison::TestDiffers {
|
||||
test_string: t,
|
||||
oracle_string: o,
|
||||
} => {
|
||||
writeln!(writer, " stderr {}", "DIFFERS:".bright_red())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stderr ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
write_diff(writer, 10, o.as_str(), t.as_str())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------".cyan()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.temp_dir {
|
||||
DirComparison::Ignored => writeln!(writer, " temp dir {}", "ignored".cyan())?,
|
||||
DirComparison::Same => writeln!(writer, " temp dir matches {}", "✔️".green())?,
|
||||
DirComparison::TestDiffers(entries) => {
|
||||
writeln!(writer, " temp dir {}", "DIFFERS".bright_red())?;
|
||||
|
||||
for entry in entries {
|
||||
const INDENT: &str = " ";
|
||||
match entry {
|
||||
DirComparisonEntry::Different(
|
||||
left_path,
|
||||
left_contents,
|
||||
right_path,
|
||||
right_contents,
|
||||
) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}oracle file {} differs from test file {}",
|
||||
left_path.to_string_lossy(),
|
||||
right_path.to_string_lossy()
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}{}",
|
||||
"------ Oracle <> Test: file ---------------------------------"
|
||||
.cyan()
|
||||
)?;
|
||||
|
||||
write_diff(
|
||||
writer,
|
||||
10,
|
||||
left_contents.as_str(),
|
||||
right_contents.as_str(),
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------"
|
||||
.cyan()
|
||||
)?;
|
||||
}
|
||||
DirComparisonEntry::LeftOnly(p) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}file missing from test dir: {}",
|
||||
p.to_string_lossy()
|
||||
)?;
|
||||
}
|
||||
DirComparisonEntry::RightOnly(p) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}unexpected file in test dir: {}",
|
||||
p.to_string_lossy()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[expect(clippy::unused_self)]
|
||||
fn write_expectation_details<W: Write>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
expectation: &ExpectationComparison,
|
||||
) -> Result<()> {
|
||||
writeln!(writer, " {} check:", "Expectation".cyan())?;
|
||||
|
||||
// Exit code
|
||||
match &expectation.exit_code {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " exit code matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(
|
||||
writer,
|
||||
" exit code {}: expected {}, got {}",
|
||||
"DIFFERS".bright_red(),
|
||||
expected.cyan(),
|
||||
actual.bright_red()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Stdout
|
||||
match &expectation.stdout {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " stdout matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(writer, " stdout {}", "DIFFERS:".bright_red())?;
|
||||
write_diff(writer, 8, expected.as_str(), actual.as_str())?;
|
||||
}
|
||||
}
|
||||
|
||||
// Stderr
|
||||
match &expectation.stderr {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " stderr matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(writer, " stderr {}", "DIFFERS:".bright_red())?;
|
||||
write_diff(writer, 8, expected.as_str(), actual.as_str())?;
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot
|
||||
if expectation.snapshot_used {
|
||||
if let Some(result) = &expectation.snapshot_result {
|
||||
match result {
|
||||
crate::comparison::SnapshotResult::Matches => {
|
||||
writeln!(writer, " snapshot matches {}", "✔️".green())?;
|
||||
}
|
||||
crate::comparison::SnapshotResult::Differs { message } => {
|
||||
writeln!(
|
||||
writer,
|
||||
" snapshot {}: {}",
|
||||
"DIFFERS".bright_red(),
|
||||
message
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Results from running a set of test cases.
|
||||
pub struct TestCaseSetResults {
|
||||
/// Name of the test case set.
|
||||
pub name: Option<String>,
|
||||
/// Name of the configuration used.
|
||||
pub config_name: String,
|
||||
/// Number of successful tests.
|
||||
pub success_count: u32,
|
||||
/// Number of skipped tests.
|
||||
pub skip_count: u32,
|
||||
/// Number of known failures.
|
||||
pub known_failure_count: u32,
|
||||
/// Number of failed tests.
|
||||
pub fail_count: u32,
|
||||
/// Individual test case results.
|
||||
pub test_case_results: Vec<TestCaseResult>,
|
||||
/// Total duration comparison for successful tests.
|
||||
pub success_duration: std::time::Duration,
|
||||
}
|
||||
|
||||
impl TestCaseSetResults {
|
||||
/// Reports these results in pretty format.
|
||||
pub fn report_pretty(&self, options: &TestOptions) -> Result<()> {
|
||||
self.write_details(std::io::stderr(), options)
|
||||
}
|
||||
|
||||
fn write_details<W: Write>(&self, mut writer: W, options: &TestOptions) -> Result<()> {
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
"=================== {}: [{}/{}] ===================",
|
||||
"Running test case set".blue(),
|
||||
self.name
|
||||
.as_ref()
|
||||
.map_or_else(|| "(unnamed)", |n| n.as_str())
|
||||
.italic(),
|
||||
self.config_name.magenta(),
|
||||
)?;
|
||||
}
|
||||
|
||||
for test_case_result in &self.test_case_results {
|
||||
test_case_result.report_pretty(options)?;
|
||||
}
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" successful cases ran in {:?}",
|
||||
self.success_duration
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports test results based on the configured output format.
|
||||
pub fn report_results(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
match options.format {
|
||||
OutputFormat::Pretty => report_results_pretty(results, options),
|
||||
OutputFormat::Junit => report_results_junit(results, options),
|
||||
OutputFormat::Terse => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn report_results_pretty(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
for result in results {
|
||||
result.report_pretty(options)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report_results_junit(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
let mut report = junit_report::Report::new();
|
||||
|
||||
for result in results {
|
||||
let mut suite = junit_report::TestSuite::new(result.name.unwrap_or(String::new()).as_str());
|
||||
for r in result.test_case_results {
|
||||
let test_case_name = r.name.as_deref().unwrap_or("");
|
||||
let mut test_case: junit_report::TestCase = if r.success {
|
||||
junit_report::TestCase::success(test_case_name, r.comparison.duration.try_into()?)
|
||||
} else if r.known_failure {
|
||||
junit_report::TestCase::skipped(test_case_name)
|
||||
} else {
|
||||
junit_report::TestCase::failure(
|
||||
test_case_name,
|
||||
r.comparison.duration.try_into()?,
|
||||
"test failure",
|
||||
"failed",
|
||||
)
|
||||
};
|
||||
|
||||
let mut output_buf: Vec<u8> = vec![];
|
||||
r.write_details(&mut output_buf, options)?;
|
||||
|
||||
let output_as_string = String::from_utf8(output_buf)?;
|
||||
test_case.set_system_out(strip_ansi_escapes::strip_str(output_as_string).as_str());
|
||||
|
||||
suite.add_testcase(test_case);
|
||||
}
|
||||
|
||||
report.add_testsuite(suite);
|
||||
}
|
||||
|
||||
report.write_xml(std::io::stdout())?;
|
||||
writeln!(std::io::stdout())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
//! Test runner implementation.
|
||||
|
||||
use crate::comparison::{
|
||||
DirComparison, DurationComparison, ExitStatusComparison, ExpectationComparison,
|
||||
OracleComparison, SingleExpectationComparison, SnapshotResult, StringComparison,
|
||||
TestComparison, diff_dirs, output_matches,
|
||||
};
|
||||
use crate::config::{RunnerConfig, TestMode, TestOptions};
|
||||
use crate::reporting::{TestCaseResult, TestCaseSetResults};
|
||||
use crate::testcase::{TestCase, TestCaseSet};
|
||||
use anyhow::{Context, Result};
|
||||
use colored::Colorize;
|
||||
|
||||
/// The main test runner.
|
||||
pub struct TestRunner {
|
||||
config: RunnerConfig,
|
||||
options: TestOptions,
|
||||
}
|
||||
|
||||
impl TestRunner {
|
||||
/// Creates a new test runner with the given configuration and options.
|
||||
pub const fn new(config: RunnerConfig, options: TestOptions) -> Self {
|
||||
Self { config, options }
|
||||
}
|
||||
|
||||
/// Runs all tests and returns success/failure.
|
||||
pub async fn run(&self) -> Result<bool> {
|
||||
let mut success_count = 0;
|
||||
let mut skip_count = 0;
|
||||
let mut known_failure_count = 0;
|
||||
let mut fail_count = 0;
|
||||
let mut join_handles = vec![];
|
||||
let mut success_duration = std::time::Duration::default();
|
||||
|
||||
// Generate a glob pattern to find all the YAML test case files.
|
||||
let glob_pattern = self
|
||||
.config
|
||||
.test_cases_dir
|
||||
.join("**/*.yaml")
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
|
||||
if self.options.verbose {
|
||||
eprintln!("Running test cases: {glob_pattern}");
|
||||
}
|
||||
|
||||
// Spawn each test case set separately.
|
||||
for entry in glob::glob(glob_pattern.as_ref()).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
|
||||
let yaml_file = std::fs::File::open(entry.as_path())?;
|
||||
let mut test_case_set: TestCaseSet = serde_yaml::from_reader(yaml_file)
|
||||
.context(format!("parsing {}", entry.to_string_lossy()))?;
|
||||
|
||||
test_case_set.source_dir = entry.parent().unwrap().to_path_buf();
|
||||
test_case_set.source_file.clone_from(&entry);
|
||||
|
||||
if self.options.list_tests_only {
|
||||
for test_case in &test_case_set.cases {
|
||||
let case_is_skipped = self.should_skip_test(&test_case_set, test_case)?;
|
||||
if case_is_skipped == self.options.skipped_tests_only {
|
||||
println!(
|
||||
"{}::{}: test",
|
||||
test_case_set.name.as_deref().unwrap_or("unnamed"),
|
||||
test_case.name.as_deref().unwrap_or("unnamed"),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let config = self.config.clone();
|
||||
let options = self.options.clone();
|
||||
|
||||
join_handles.push(tokio::spawn(async move {
|
||||
run_test_case_set(test_case_set, config, options).await
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if self.options.list_tests_only {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
// Await all results.
|
||||
let mut all_results = vec![];
|
||||
for join_handle in join_handles {
|
||||
let results = join_handle.await??;
|
||||
|
||||
success_count += results.success_count;
|
||||
skip_count += results.skip_count;
|
||||
known_failure_count += results.known_failure_count;
|
||||
fail_count += results.fail_count;
|
||||
success_duration += results.success_duration;
|
||||
|
||||
all_results.push(results);
|
||||
}
|
||||
|
||||
crate::reporting::report_results(all_results, &self.options)?;
|
||||
|
||||
if matches!(self.options.format, crate::config::OutputFormat::Pretty) {
|
||||
let formatted_fail_count = if fail_count > 0 {
|
||||
fail_count.to_string().red()
|
||||
} else {
|
||||
fail_count.to_string().green()
|
||||
};
|
||||
|
||||
let formatted_known_failure_count = if known_failure_count > 0 {
|
||||
known_failure_count.to_string().magenta()
|
||||
} else {
|
||||
known_failure_count.to_string().green()
|
||||
};
|
||||
|
||||
let formatted_skip_count = if skip_count > 0 {
|
||||
skip_count.to_string().cyan()
|
||||
} else {
|
||||
skip_count.to_string().green()
|
||||
};
|
||||
|
||||
eprintln!(
|
||||
"================================================================================"
|
||||
);
|
||||
eprintln!(
|
||||
"{} test case(s) ran: {} succeeded, {} failed, {} known to fail, {} skipped.",
|
||||
success_count + fail_count + known_failure_count,
|
||||
success_count.to_string().green(),
|
||||
formatted_fail_count,
|
||||
formatted_known_failure_count,
|
||||
formatted_skip_count,
|
||||
);
|
||||
eprintln!("duration of successful tests: {success_duration:?}");
|
||||
eprintln!(
|
||||
"================================================================================"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(fail_count == 0)
|
||||
}
|
||||
|
||||
fn should_skip_test(&self, test_case_set: &TestCaseSet, test_case: &TestCase) -> Result<bool> {
|
||||
// Check incompatible configs (set-level and per-case).
|
||||
if let Some(oracle) = &self.config.oracle {
|
||||
if test_case_set.incompatible_configs.contains(&oracle.name)
|
||||
|| test_case.incompatible_configs.contains(&oracle.name)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
// Check incompatible OS
|
||||
if let Some(host_os_id) = &self.config.host_os_id {
|
||||
if test_case.incompatible_os.contains(host_os_id) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
// Check incompatible runtime platforms (set-level and per-case).
|
||||
if test_case_set
|
||||
.incompatible_platforms
|
||||
.iter()
|
||||
.chain(test_case.incompatible_platforms.iter())
|
||||
.any(|p| self.config.platform_tags.contains(p))
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
// Check oracle version constraints
|
||||
if let Some(oracle) = &self.config.oracle {
|
||||
if test_case.min_oracle_version.is_some() || test_case.max_oracle_version.is_some() {
|
||||
if let Some(actual_oracle_version_str) = &oracle.version_str {
|
||||
let actual_oracle_version =
|
||||
version_compare::Version::from(actual_oracle_version_str.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("failed to parse oracle version"))?;
|
||||
|
||||
if let Some(min_oracle_version_str) = &test_case.min_oracle_version {
|
||||
let min_oracle_version = version_compare::Version::from(
|
||||
min_oracle_version_str,
|
||||
)
|
||||
.ok_or_else(|| anyhow::anyhow!("failed to parse min oracle version"))?;
|
||||
|
||||
if matches!(
|
||||
actual_oracle_version.compare(min_oracle_version),
|
||||
version_compare::Cmp::Lt
|
||||
) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(max_oracle_version_str) = &test_case.max_oracle_version {
|
||||
let max_oracle_version = version_compare::Version::from(
|
||||
max_oracle_version_str,
|
||||
)
|
||||
.ok_or_else(|| anyhow::anyhow!("failed to parse max oracle version"))?;
|
||||
|
||||
if matches!(
|
||||
actual_oracle_version.compare(max_oracle_version),
|
||||
version_compare::Cmp::Gt
|
||||
) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check filters
|
||||
let test_case_set_name = test_case_set.name.as_deref().unwrap_or("");
|
||||
let test_case_name = test_case.name.as_deref().unwrap_or("");
|
||||
|
||||
if test_case_set_name.is_empty() || test_case_name.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let qualified_name = format!("{test_case_set_name}::{test_case_name}");
|
||||
if !self.options.should_run_test(&qualified_name) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(test_case.skip)
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_test_case_set(
|
||||
test_case_set: TestCaseSet,
|
||||
config: RunnerConfig,
|
||||
options: TestOptions,
|
||||
) -> Result<TestCaseSetResults> {
|
||||
let mut success_count = 0;
|
||||
let mut skip_count = 0;
|
||||
let mut known_failure_count = 0;
|
||||
let mut fail_count = 0;
|
||||
let mut success_duration = std::time::Duration::default();
|
||||
let mut test_case_results = vec![];
|
||||
|
||||
for test_case in &test_case_set.cases {
|
||||
let runner = TestRunner::new(config.clone(), options.clone());
|
||||
let case_is_skipped = runner.should_skip_test(&test_case_set, test_case)?;
|
||||
|
||||
let test_case_result = if case_is_skipped == options.skipped_tests_only {
|
||||
run_single_test(&test_case_set, test_case, &config).await?
|
||||
} else {
|
||||
TestCaseResult {
|
||||
success: true,
|
||||
comparison: TestComparison::skipped(),
|
||||
name: test_case.name.clone(),
|
||||
skip: true,
|
||||
known_failure: test_case.known_failure,
|
||||
}
|
||||
};
|
||||
|
||||
if test_case_result.skip {
|
||||
skip_count += 1;
|
||||
} else if test_case_result.success {
|
||||
if test_case.known_failure {
|
||||
fail_count += 1;
|
||||
} else {
|
||||
success_count += 1;
|
||||
success_duration += test_case_result.comparison.duration;
|
||||
}
|
||||
} else if test_case.known_failure {
|
||||
known_failure_count += 1;
|
||||
} else {
|
||||
fail_count += 1;
|
||||
}
|
||||
|
||||
test_case_results.push(test_case_result);
|
||||
}
|
||||
|
||||
let config_name = config
|
||||
.oracle
|
||||
.map_or_else(|| String::from("brush"), |o| o.name);
|
||||
|
||||
Ok(TestCaseSetResults {
|
||||
name: test_case_set.name.clone(),
|
||||
config_name,
|
||||
test_case_results,
|
||||
success_count,
|
||||
skip_count,
|
||||
known_failure_count,
|
||||
fail_count,
|
||||
success_duration,
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_single_test(
|
||||
test_case_set: &TestCaseSet,
|
||||
test_case: &TestCase,
|
||||
config: &RunnerConfig,
|
||||
) -> Result<TestCaseResult> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
// Determine what comparisons to perform
|
||||
let should_run_oracle = config.oracle.is_some()
|
||||
&& !test_case.skip_oracle
|
||||
&& matches!(config.mode, TestMode::Oracle | TestMode::Hybrid);
|
||||
|
||||
let should_check_expectations = test_case.has_expectations()
|
||||
|| matches!(config.mode, TestMode::Expectation | TestMode::Hybrid);
|
||||
|
||||
// Run oracle comparison if needed
|
||||
let (oracle_comparison, test_result, test_temp_dir) = if should_run_oracle {
|
||||
let oracle_config = config.oracle.as_ref().unwrap();
|
||||
let (oracle_comp, test_res) =
|
||||
run_oracle_comparison(test_case_set, test_case, config, oracle_config).await?;
|
||||
(Some(oracle_comp), Some(test_res), None)
|
||||
} else {
|
||||
// Run test shell only
|
||||
let test_temp_dir = assert_fs::TempDir::new()?;
|
||||
test_case.create_test_files_in(&test_temp_dir, test_case_set)?;
|
||||
let test_res = test_case
|
||||
.run_shell(&config.test_shell, &test_temp_dir)
|
||||
.await?;
|
||||
(None, Some(test_res), Some(test_temp_dir))
|
||||
};
|
||||
|
||||
// Check expectations
|
||||
let expectation_comparison = if should_check_expectations {
|
||||
if let Some(test_res) = &test_result {
|
||||
check_expectations(
|
||||
test_case_set,
|
||||
test_case,
|
||||
test_res,
|
||||
test_temp_dir.as_ref(),
|
||||
config,
|
||||
)
|
||||
} else {
|
||||
ExpectationComparison::not_specified()
|
||||
}
|
||||
} else {
|
||||
ExpectationComparison::not_specified()
|
||||
};
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
|
||||
let comparison = TestComparison {
|
||||
oracle: oracle_comparison,
|
||||
expectation: expectation_comparison,
|
||||
duration,
|
||||
};
|
||||
|
||||
let success = !comparison.is_failure();
|
||||
|
||||
Ok(TestCaseResult {
|
||||
success,
|
||||
comparison,
|
||||
name: test_case.name.clone(),
|
||||
skip: false,
|
||||
known_failure: test_case.known_failure,
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_oracle_comparison(
|
||||
test_case_set: &TestCaseSet,
|
||||
test_case: &TestCase,
|
||||
config: &RunnerConfig,
|
||||
oracle_config: &crate::config::OracleConfig,
|
||||
) -> Result<(OracleComparison, crate::execution::RunResult)> {
|
||||
// Run oracle
|
||||
let oracle_temp_dir = assert_fs::TempDir::new()?;
|
||||
test_case.create_test_files_in(&oracle_temp_dir, test_case_set)?;
|
||||
let oracle_result = test_case
|
||||
.run_shell(&oracle_config.shell, &oracle_temp_dir)
|
||||
.await?;
|
||||
|
||||
// Run test shell
|
||||
let test_temp_dir = assert_fs::TempDir::new()?;
|
||||
test_case.create_test_files_in(&test_temp_dir, test_case_set)?;
|
||||
let test_result = test_case
|
||||
.run_shell(&config.test_shell, &test_temp_dir)
|
||||
.await?;
|
||||
|
||||
// Build comparison
|
||||
let mut comparison = OracleComparison {
|
||||
exit_status: ExitStatusComparison::Ignored,
|
||||
stdout: StringComparison::Ignored {
|
||||
test_string: String::new(),
|
||||
oracle_string: String::new(),
|
||||
},
|
||||
stderr: StringComparison::Ignored {
|
||||
test_string: String::new(),
|
||||
oracle_string: String::new(),
|
||||
},
|
||||
temp_dir: DirComparison::Ignored,
|
||||
duration: DurationComparison {
|
||||
oracle: oracle_result.duration,
|
||||
test: test_result.duration,
|
||||
},
|
||||
};
|
||||
|
||||
// Compare exit status
|
||||
if test_case.ignore_exit_status {
|
||||
comparison.exit_status = ExitStatusComparison::Ignored;
|
||||
} else if oracle_result.exit_status == test_result.exit_status {
|
||||
comparison.exit_status = ExitStatusComparison::Same(oracle_result.exit_status);
|
||||
} else {
|
||||
comparison.exit_status = ExitStatusComparison::TestDiffers {
|
||||
test_exit_status: test_result.exit_status,
|
||||
oracle_exit_status: oracle_result.exit_status,
|
||||
};
|
||||
}
|
||||
|
||||
// Compare stdout
|
||||
if test_case.ignore_stdout {
|
||||
comparison.stdout = StringComparison::Ignored {
|
||||
test_string: test_result.stdout.clone(),
|
||||
oracle_string: oracle_result.stdout,
|
||||
};
|
||||
} else if output_matches(
|
||||
&oracle_result.stdout,
|
||||
&test_result.stdout,
|
||||
test_case.ignore_whitespace,
|
||||
) {
|
||||
comparison.stdout = StringComparison::Same(oracle_result.stdout);
|
||||
} else {
|
||||
comparison.stdout = StringComparison::TestDiffers {
|
||||
test_string: test_result.stdout.clone(),
|
||||
oracle_string: oracle_result.stdout,
|
||||
};
|
||||
}
|
||||
|
||||
// Compare stderr
|
||||
if test_case.ignore_stderr {
|
||||
comparison.stderr = StringComparison::Ignored {
|
||||
test_string: test_result.stderr.clone(),
|
||||
oracle_string: oracle_result.stderr,
|
||||
};
|
||||
} else if output_matches(
|
||||
&oracle_result.stderr,
|
||||
&test_result.stderr,
|
||||
test_case.ignore_whitespace,
|
||||
) {
|
||||
comparison.stderr = StringComparison::Same(oracle_result.stderr);
|
||||
} else {
|
||||
comparison.stderr = StringComparison::TestDiffers {
|
||||
test_string: test_result.stderr.clone(),
|
||||
oracle_string: oracle_result.stderr,
|
||||
};
|
||||
}
|
||||
|
||||
// Compare temporary directory contents
|
||||
comparison.temp_dir = diff_dirs(oracle_temp_dir.path(), test_temp_dir.path())?;
|
||||
|
||||
Ok((comparison, test_result))
|
||||
}
|
||||
|
||||
fn check_expectations(
|
||||
test_case_set: &TestCaseSet,
|
||||
test_case: &TestCase,
|
||||
test_result: &crate::execution::RunResult,
|
||||
test_temp_dir: Option<&assert_fs::TempDir>,
|
||||
config: &RunnerConfig,
|
||||
) -> ExpectationComparison {
|
||||
let mut comparison = ExpectationComparison::not_specified();
|
||||
|
||||
// Check inline expectations
|
||||
if let Some(expected_exit_code) = test_case.expected_exit_code {
|
||||
let actual_code = test_result.exit_status.code().unwrap_or(-1);
|
||||
if actual_code == expected_exit_code {
|
||||
comparison.exit_code = SingleExpectationComparison::Matches;
|
||||
} else {
|
||||
comparison.exit_code = SingleExpectationComparison::Differs {
|
||||
expected: expected_exit_code.to_string(),
|
||||
actual: actual_code.to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(expected_stdout) = &test_case.expected_stdout {
|
||||
if output_matches(
|
||||
expected_stdout,
|
||||
&test_result.stdout,
|
||||
test_case.ignore_whitespace,
|
||||
) {
|
||||
comparison.stdout = SingleExpectationComparison::Matches;
|
||||
} else {
|
||||
comparison.stdout = SingleExpectationComparison::Differs {
|
||||
expected: expected_stdout.clone(),
|
||||
actual: test_result.stdout.clone(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(expected_stderr) = &test_case.expected_stderr {
|
||||
if output_matches(
|
||||
expected_stderr,
|
||||
&test_result.stderr,
|
||||
test_case.ignore_whitespace,
|
||||
) {
|
||||
comparison.stderr = SingleExpectationComparison::Matches;
|
||||
} else {
|
||||
comparison.stderr = SingleExpectationComparison::Differs {
|
||||
expected: expected_stderr.clone(),
|
||||
actual: test_result.stderr.clone(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Check snapshot if enabled
|
||||
if test_case.snapshot {
|
||||
comparison.snapshot_used = true;
|
||||
comparison.snapshot_result = Some(check_snapshot(
|
||||
test_case_set,
|
||||
test_case,
|
||||
test_result,
|
||||
test_temp_dir,
|
||||
config,
|
||||
));
|
||||
}
|
||||
|
||||
comparison
|
||||
}
|
||||
|
||||
#[cfg(feature = "insta")]
|
||||
fn check_snapshot(
|
||||
test_case_set: &TestCaseSet,
|
||||
test_case: &TestCase,
|
||||
test_result: &crate::execution::RunResult,
|
||||
test_temp_dir: Option<&assert_fs::TempDir>,
|
||||
config: &RunnerConfig,
|
||||
) -> SnapshotResult {
|
||||
// Collect files from the temp directory
|
||||
let files = collect_temp_dir_files(test_temp_dir, test_case);
|
||||
|
||||
// Build snapshot content manually with literal block style for better readability
|
||||
let snapshot_content = format_snapshot_yaml(
|
||||
test_result.exit_status.code().unwrap_or(-1),
|
||||
&test_result.stdout,
|
||||
&test_result.stderr,
|
||||
&files,
|
||||
);
|
||||
|
||||
// Compute snapshot path
|
||||
let snapshot_dir = test_case_set.source_dir.join(&config.snapshot_dir_name);
|
||||
let yaml_stem = test_case_set
|
||||
.source_file
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("unknown");
|
||||
let test_name = test_case.name.as_deref().unwrap_or("unnamed");
|
||||
|
||||
// Convert spaces to underscores in snapshot name for filesystem compatibility
|
||||
let snapshot_name = format!("{}_{}", yaml_stem, test_name.replace(' ', "_"));
|
||||
|
||||
// Use insta's settings to configure snapshot location
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
settings.set_snapshot_path(&snapshot_dir);
|
||||
settings.set_prepend_module_to_snapshot(false);
|
||||
|
||||
settings.bind(|| {
|
||||
// Use assert_snapshot with raw string for block-style YAML
|
||||
let result = std::panic::catch_unwind(|| {
|
||||
insta::assert_snapshot!(snapshot_name.clone(), snapshot_content);
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(()) => SnapshotResult::Matches,
|
||||
Err(_) => SnapshotResult::Differs {
|
||||
message: format!(
|
||||
"Snapshot '{snapshot_name}' differs or is new. Run `cargo insta review` to update."
|
||||
),
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Format snapshot data as YAML with literal block style for multiline strings.
|
||||
#[cfg(feature = "insta")]
|
||||
fn format_snapshot_yaml(
|
||||
exit_code: i32,
|
||||
stdout: &str,
|
||||
stderr: &str,
|
||||
files: &std::collections::BTreeMap<String, String>,
|
||||
) -> String {
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut output = String::new();
|
||||
|
||||
writeln!(output, "exit_code: {exit_code}").ok();
|
||||
|
||||
// Format stdout
|
||||
write!(output, "stdout: ").ok();
|
||||
format_yaml_string(&mut output, stdout, 0);
|
||||
|
||||
// Format stderr
|
||||
write!(output, "stderr: ").ok();
|
||||
format_yaml_string(&mut output, stderr, 0);
|
||||
|
||||
// Format files if any
|
||||
if !files.is_empty() {
|
||||
writeln!(output, "files:").ok();
|
||||
for (filename, contents) in files {
|
||||
write!(output, " {filename}: ").ok();
|
||||
format_yaml_string(&mut output, contents, 2);
|
||||
}
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Format a string as YAML, using literal block style for multiline content.
|
||||
#[cfg(feature = "insta")]
|
||||
fn format_yaml_string(output: &mut String, s: &str, indent: usize) {
|
||||
use std::fmt::Write;
|
||||
|
||||
if s.is_empty() {
|
||||
writeln!(output, "\"\"").ok();
|
||||
} else if s.contains('\n') {
|
||||
// Use literal block style for multiline strings
|
||||
if s.ends_with('\n') {
|
||||
writeln!(output, "|").ok();
|
||||
} else {
|
||||
writeln!(output, "|-").ok();
|
||||
}
|
||||
let indent_str = " ".repeat(indent + 2);
|
||||
for line in s.lines() {
|
||||
writeln!(output, "{indent_str}{line}").ok();
|
||||
}
|
||||
// If string ends with newline but lines() doesn't capture trailing empty line
|
||||
if s.ends_with('\n') && !s.ends_with("\n\n") {
|
||||
// Already handled by literal block indicator '|'
|
||||
} else if s.ends_with("\n\n") {
|
||||
// Multiple trailing newlines need explicit empty lines
|
||||
let trailing_newlines = s.len() - s.trim_end_matches('\n').len();
|
||||
for _ in 1..trailing_newlines {
|
||||
writeln!(output, "{indent_str}").ok();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Single line - use quoted style if contains special chars, otherwise plain
|
||||
if s.contains(':')
|
||||
|| s.contains('#')
|
||||
|| s.contains('\'')
|
||||
|| s.contains('"')
|
||||
|| s.starts_with(' ')
|
||||
|| s.ends_with(' ')
|
||||
|| s == "true"
|
||||
|| s == "false"
|
||||
|| s == "null"
|
||||
|| s.parse::<f64>().is_ok()
|
||||
{
|
||||
// Use double-quoted style with escapes
|
||||
let escaped = s
|
||||
.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('\t', "\\t");
|
||||
writeln!(output, "\"{escaped}\"").ok();
|
||||
} else {
|
||||
writeln!(output, "{s}").ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect files from the temp directory, excluding test input files.
|
||||
/// Returns a map of filename -> contents (as string for text files).
|
||||
#[cfg(feature = "insta")]
|
||||
fn collect_temp_dir_files(
|
||||
test_temp_dir: Option<&assert_fs::TempDir>,
|
||||
test_case: &TestCase,
|
||||
) -> std::collections::BTreeMap<String, String> {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
let mut files = BTreeMap::new();
|
||||
|
||||
let Some(temp_dir) = test_temp_dir else {
|
||||
return files;
|
||||
};
|
||||
|
||||
// Get the set of input test file names to exclude
|
||||
let input_files: std::collections::HashSet<_> = test_case
|
||||
.test_files
|
||||
.iter()
|
||||
.map(|f| f.path.to_string_lossy().to_string())
|
||||
.collect();
|
||||
|
||||
// Walk the temp directory and collect files
|
||||
let Ok(entries) = std::fs::read_dir(temp_dir.path()) else {
|
||||
return files;
|
||||
};
|
||||
|
||||
for entry in entries.flatten() {
|
||||
let filename = entry.file_name().to_string_lossy().to_string();
|
||||
|
||||
// Skip input test files
|
||||
if input_files.contains(&filename) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip coverage profile data
|
||||
if filename.ends_with(".profraw") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Read file contents - use lossy conversion for binary files
|
||||
if let Ok(contents) = std::fs::read(&path) {
|
||||
let text = String::from_utf8_lossy(&contents).to_string();
|
||||
files.insert(filename, text);
|
||||
}
|
||||
}
|
||||
|
||||
files
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "insta"))]
|
||||
fn check_snapshot(
|
||||
_test_case_set: &TestCaseSet,
|
||||
_test_case: &TestCase,
|
||||
_test_result: &crate::execution::RunResult,
|
||||
_test_temp_dir: Option<&assert_fs::TempDir>,
|
||||
_config: &RunnerConfig,
|
||||
) -> SnapshotResult {
|
||||
SnapshotResult::Differs {
|
||||
message: String::from("Snapshot testing requires the 'insta' feature to be enabled"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
//! Test case definitions and YAML schema.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
/// How to invoke the shell for a test case.
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
pub enum ShellInvocation {
|
||||
/// Execute the shell binary directly.
|
||||
#[default]
|
||||
ExecShellBinary,
|
||||
/// Execute a script file.
|
||||
ExecScript(String),
|
||||
}
|
||||
|
||||
/// A file to create in the test's temporary directory.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct TestFile {
|
||||
/// Relative path to test file within the temp directory.
|
||||
pub path: PathBuf,
|
||||
/// Contents to seed the file with.
|
||||
#[serde(default)]
|
||||
pub contents: String,
|
||||
/// Optionally provides relative path to the source file
|
||||
/// that should be used to populate this file.
|
||||
pub source_path: Option<PathBuf>,
|
||||
/// Whether the file should be executable.
|
||||
#[serde(default)]
|
||||
pub executable: bool,
|
||||
}
|
||||
|
||||
/// A single test case.
|
||||
#[allow(
|
||||
clippy::unsafe_derive_deserialize,
|
||||
reason = "the unsafe call is unrelated to deserialization"
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct TestCase {
|
||||
/// Name of the test case.
|
||||
pub name: Option<String>,
|
||||
|
||||
/// How to invoke the shell.
|
||||
#[serde(default)]
|
||||
pub invocation: ShellInvocation,
|
||||
|
||||
/// Command-line arguments to the shell.
|
||||
#[serde(default)]
|
||||
pub args: Vec<String>,
|
||||
|
||||
/// Command-line arguments to append for the shell-under-test only.
|
||||
#[serde(default)]
|
||||
pub additional_test_args: Vec<String>,
|
||||
|
||||
/// Default command-line shell arguments that should be *removed*.
|
||||
#[serde(default)]
|
||||
pub removed_default_args: HashSet<String>,
|
||||
|
||||
/// Environment variables for the shell.
|
||||
#[serde(default)]
|
||||
pub env: HashMap<String, String>,
|
||||
|
||||
/// Home directory to set for the test.
|
||||
#[serde(default)]
|
||||
pub home_dir: Option<PathBuf>,
|
||||
|
||||
/// Whether to skip this test.
|
||||
#[serde(default)]
|
||||
pub skip: bool,
|
||||
|
||||
/// Whether this test requires a PTY.
|
||||
#[serde(default)]
|
||||
pub pty: bool,
|
||||
|
||||
/// Input to provide via stdin.
|
||||
#[serde(default)]
|
||||
pub stdin: Option<String>,
|
||||
|
||||
/// Whether to ignore exit status differences.
|
||||
#[serde(default)]
|
||||
pub ignore_exit_status: bool,
|
||||
|
||||
/// Whether to ignore stderr differences.
|
||||
#[serde(default)]
|
||||
pub ignore_stderr: bool,
|
||||
|
||||
/// Whether to ignore stdout differences.
|
||||
#[serde(default)]
|
||||
pub ignore_stdout: bool,
|
||||
|
||||
/// Whether to normalize whitespace when comparing output.
|
||||
#[serde(default)]
|
||||
pub ignore_whitespace: bool,
|
||||
|
||||
/// Files to create in the test's temporary directory.
|
||||
#[serde(default)]
|
||||
pub test_files: Vec<TestFile>,
|
||||
|
||||
/// Whether this test is a known failure.
|
||||
#[serde(default)]
|
||||
pub known_failure: bool,
|
||||
|
||||
/// Configurations that are incompatible with this test.
|
||||
#[serde(default)]
|
||||
pub incompatible_configs: HashSet<String>,
|
||||
|
||||
/// Operating systems that are incompatible with this test.
|
||||
#[serde(default)]
|
||||
pub incompatible_os: HashSet<String>,
|
||||
|
||||
/// Runtime platform tags (e.g., "wasi", "wasm") that are incompatible
|
||||
/// with this test. The test is skipped when any of these tags is present
|
||||
/// in the runner's active platform tag set.
|
||||
#[serde(default)]
|
||||
pub incompatible_platforms: HashSet<String>,
|
||||
|
||||
/// Minimum oracle version required for this test.
|
||||
#[serde(default)]
|
||||
pub min_oracle_version: Option<String>,
|
||||
|
||||
/// Maximum oracle version allowed for this test.
|
||||
#[serde(default)]
|
||||
pub max_oracle_version: Option<String>,
|
||||
|
||||
/// Timeout for this test in seconds.
|
||||
#[serde(default)]
|
||||
pub timeout_in_seconds: Option<u64>,
|
||||
|
||||
// ==================== Expectation fields ====================
|
||||
/// Expected stdout content (for expectation-based testing).
|
||||
#[serde(default)]
|
||||
pub expected_stdout: Option<String>,
|
||||
|
||||
/// Expected stderr content (for expectation-based testing).
|
||||
#[serde(default)]
|
||||
pub expected_stderr: Option<String>,
|
||||
|
||||
/// Expected exit code (for expectation-based testing).
|
||||
#[serde(default)]
|
||||
pub expected_exit_code: Option<i32>,
|
||||
|
||||
/// Whether to use insta snapshot for this test's expectations.
|
||||
#[serde(default)]
|
||||
pub snapshot: bool,
|
||||
|
||||
/// Whether to skip oracle comparison even when an oracle is configured.
|
||||
#[serde(default)]
|
||||
pub skip_oracle: bool,
|
||||
}
|
||||
|
||||
impl TestCase {
|
||||
/// Returns whether this test case has any inline expectations defined.
|
||||
pub const fn has_inline_expectations(&self) -> bool {
|
||||
self.expected_stdout.is_some()
|
||||
|| self.expected_stderr.is_some()
|
||||
|| self.expected_exit_code.is_some()
|
||||
}
|
||||
|
||||
/// Returns whether this test case uses snapshots for expectations.
|
||||
pub const fn uses_snapshot(&self) -> bool {
|
||||
self.snapshot
|
||||
}
|
||||
|
||||
/// Returns whether this test case has any expectations (inline or snapshot).
|
||||
pub const fn has_expectations(&self) -> bool {
|
||||
self.has_inline_expectations() || self.uses_snapshot()
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of test cases loaded from a single YAML file.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct TestCaseSet {
|
||||
/// Name of the test case set.
|
||||
pub name: Option<String>,
|
||||
|
||||
/// The test cases in this set.
|
||||
pub cases: Vec<TestCase>,
|
||||
|
||||
/// Common test files applicable to all children test cases.
|
||||
#[serde(default)]
|
||||
pub common_test_files: Vec<TestFile>,
|
||||
|
||||
/// Configurations that are incompatible with this entire test set.
|
||||
#[serde(default)]
|
||||
pub incompatible_configs: HashSet<String>,
|
||||
|
||||
/// Runtime platform tags (e.g., "wasi", "wasm") that are incompatible
|
||||
/// with this entire test set.
|
||||
#[serde(default)]
|
||||
pub incompatible_platforms: HashSet<String>,
|
||||
|
||||
/// Directory containing the YAML file (computed at runtime).
|
||||
#[serde(skip)]
|
||||
pub source_dir: PathBuf,
|
||||
|
||||
/// Path to the YAML file (computed at runtime).
|
||||
#[serde(skip)]
|
||||
pub source_file: PathBuf,
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//! Utility functions for the test harness.
|
||||
|
||||
use anyhow::Result;
|
||||
use descape::UnescapeExt;
|
||||
|
||||
/// Get the OS ID from /etc/os-release file.
|
||||
/// Returns the value of the ID field, which is the canonical OS identifier.
|
||||
/// For example: "ubuntu", "opensuse-tumbleweed", "fedora", etc.
|
||||
pub fn get_host_os_id() -> Option<String> {
|
||||
os_release::OsRelease::new().ok().and_then(|info| {
|
||||
if info.id.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(info.id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads and processes the expectrl log output.
|
||||
#[cfg(unix)]
|
||||
pub fn read_expectrl_log(log: Vec<u8>) -> Result<String> {
|
||||
let output_str = String::from_utf8(log)?;
|
||||
let output: String = output_str
|
||||
.lines()
|
||||
.filter(|line| line.starts_with("read:"))
|
||||
.map(|line| {
|
||||
line.strip_prefix("read: \"")
|
||||
.unwrap()
|
||||
.strip_suffix('"')
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Makes expectrl output human-readable by unescaping and stripping ANSI codes.
|
||||
pub fn make_expectrl_output_readable<S: AsRef<str>>(output: S) -> String {
|
||||
// Unescape the escaping done by expectrl's logging mechanism.
|
||||
let unescaped = output.as_ref().to_unescaped().unwrap().to_string();
|
||||
|
||||
// Remove VT escape sequences.
|
||||
strip_ansi_escapes::strip_str(unescaped)
|
||||
}
|
||||
|
||||
/// Writes a diff between two strings to a writer.
|
||||
pub fn write_diff(
|
||||
writer: &mut impl std::io::Write,
|
||||
indent: usize,
|
||||
left: &str,
|
||||
right: &str,
|
||||
) -> Result<()> {
|
||||
use colored::Colorize;
|
||||
|
||||
let indent_str = " ".repeat(indent);
|
||||
|
||||
let diff = diff::lines(left, right);
|
||||
for d in diff {
|
||||
let formatted = match d {
|
||||
diff::Result::Left(l) => std::format!("{indent_str}- {l}").red(),
|
||||
diff::Result::Both(l, _) => std::format!("{indent_str} {l}").bright_black(),
|
||||
diff::Result::Right(r) => std::format!("{indent_str}+ {r}").green(),
|
||||
};
|
||||
|
||||
writeln!(writer, "{formatted}")?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolves the first element of the given launcher token list to an absolute path.
|
||||
///
|
||||
/// This is needed because the test harness clears env vars (including `PATH`)
|
||||
/// before spawning child processes, so a launcher binary referenced by name
|
||||
/// (e.g., `wasmtime`) would fail to resolve at exec time.
|
||||
pub fn resolve_launcher_path(tokens: &mut [String]) -> Result<()> {
|
||||
let Some(first) = tokens.first() else {
|
||||
return Ok(());
|
||||
};
|
||||
if std::path::Path::new(first.as_str()).is_absolute() {
|
||||
return Ok(());
|
||||
}
|
||||
let resolved = std::env::var_os("PATH")
|
||||
.into_iter()
|
||||
.flat_map(|p| std::env::split_paths(&p).collect::<Vec<_>>())
|
||||
.map(|d| d.join(first.as_str()))
|
||||
.find(|p| p.is_file())
|
||||
.ok_or_else(|| anyhow::anyhow!("could not resolve launcher binary '{first}' in PATH"))?;
|
||||
tokens[0] = resolved.to_string_lossy().into_owned();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Gets the bash version string from the given bash path.
|
||||
pub fn get_bash_version_str(bash_path: &std::path::Path) -> Result<String> {
|
||||
use anyhow::Context;
|
||||
|
||||
let output = std::process::Command::new(bash_path)
|
||||
.arg("--norc")
|
||||
.arg("--noprofile")
|
||||
.arg("-c")
|
||||
.arg("echo -n ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}.${BASH_VERSINFO[2]}")
|
||||
.output()
|
||||
.context("failed to retrieve bash version")?
|
||||
.stdout;
|
||||
|
||||
let ver_str = String::from_utf8(output)?;
|
||||
|
||||
Ok(ver_str)
|
||||
}
|
||||
Reference in New Issue
Block a user