Merge nucleic/sleek-thistle-egret-fyej into dev

2026-07-18 05:19:31 -07:00
commit b6be87b72d
677 changed files with 102939 additions and 0 deletions
@@ -0,0 +1,374 @@
+//! Comparison types for test results.
+
+use std::{io::Read, path::PathBuf, process::ExitStatus};
+
+/// Comparison of durations between oracle and test runs.
+#[derive(Default)]
+pub struct DurationComparison {
+    /// Duration of the oracle run.
+    pub oracle: std::time::Duration,
+    /// Duration of the test run.
+    pub test: std::time::Duration,
+}
+
+/// Comparison of exit statuses.
+pub enum ExitStatusComparison {
+    /// Exit status was ignored.
+    Ignored,
+    /// Exit statuses match.
+    Same(ExitStatus),
+    /// Exit statuses differ.
+    TestDiffers {
+        /// Exit status from the test shell.
+        test_exit_status: ExitStatus,
+        /// Exit status from the oracle shell.
+        oracle_exit_status: ExitStatus,
+    },
+}
+
+impl ExitStatusComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        matches!(
+            self,
+            Self::TestDiffers {
+                test_exit_status: _,
+                oracle_exit_status: _
+            }
+        )
+    }
+}
+
+/// Comparison of string outputs (stdout/stderr).
+pub enum StringComparison {
+    /// Output was ignored.
+    Ignored {
+        /// Output from the test shell.
+        test_string: String,
+        /// Output from the oracle shell.
+        oracle_string: String,
+    },
+    /// Outputs match.
+    Same(String),
+    /// Outputs differ.
+    TestDiffers {
+        /// Output from the test shell.
+        test_string: String,
+        /// Output from the oracle shell.
+        oracle_string: String,
+    },
+}
+
+impl StringComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        matches!(
+            self,
+            Self::TestDiffers {
+                test_string: _,
+                oracle_string: _
+            }
+        )
+    }
+}
+
+/// A single entry in a directory comparison.
+pub enum DirComparisonEntry {
+    /// File exists only in the left (oracle) directory.
+    LeftOnly(PathBuf),
+    /// File exists only in the right (test) directory.
+    RightOnly(PathBuf),
+    /// Files differ between directories.
+    Different(PathBuf, String, PathBuf, String),
+}
+
+/// Comparison of directory contents.
+pub enum DirComparison {
+    /// Directory comparison was ignored.
+    Ignored,
+    /// Directory contents match.
+    Same,
+    /// Directory contents differ.
+    TestDiffers(Vec<DirComparisonEntry>),
+}
+
+impl DirComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        matches!(self, Self::TestDiffers(_))
+    }
+}
+
+/// Full comparison between oracle and test shell runs.
+pub struct OracleComparison {
+    /// Comparison of exit statuses.
+    pub exit_status: ExitStatusComparison,
+    /// Comparison of stdout.
+    pub stdout: StringComparison,
+    /// Comparison of stderr.
+    pub stderr: StringComparison,
+    /// Comparison of temporary directory contents.
+    pub temp_dir: DirComparison,
+    /// Comparison of durations.
+    pub duration: DurationComparison,
+}
+
+impl OracleComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        self.exit_status.is_failure()
+            || self.stdout.is_failure()
+            || self.stderr.is_failure()
+            || self.temp_dir.is_failure()
+    }
+
+    /// Creates an ignored comparison (all fields ignored).
+    pub fn ignored() -> Self {
+        Self {
+            exit_status: ExitStatusComparison::Ignored,
+            stdout: StringComparison::Ignored {
+                test_string: String::new(),
+                oracle_string: String::new(),
+            },
+            stderr: StringComparison::Ignored {
+                test_string: String::new(),
+                oracle_string: String::new(),
+            },
+            temp_dir: DirComparison::Ignored,
+            duration: DurationComparison::default(),
+        }
+    }
+}
+
+/// Comparison of a single expectation.
+#[derive(Debug)]
+pub enum SingleExpectationComparison {
+    /// Expectation was not specified (ignored).
+    NotSpecified,
+    /// Actual matches expected.
+    Matches,
+    /// Actual differs from expected.
+    Differs {
+        /// The expected value.
+        expected: String,
+        /// The actual value.
+        actual: String,
+    },
+}
+
+impl SingleExpectationComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        matches!(self, Self::Differs { .. })
+    }
+}
+
+/// Comparison against inline expectations.
+#[derive(Debug)]
+pub struct ExpectationComparison {
+    /// Comparison of exit code.
+    pub exit_code: SingleExpectationComparison,
+    /// Comparison of stdout.
+    pub stdout: SingleExpectationComparison,
+    /// Comparison of stderr.
+    pub stderr: SingleExpectationComparison,
+    /// Whether snapshot comparison was used.
+    pub snapshot_used: bool,
+    /// Snapshot comparison result (if used).
+    pub snapshot_result: Option<SnapshotResult>,
+}
+
+impl ExpectationComparison {
+    /// Creates an empty expectation comparison (all not specified).
+    pub const fn not_specified() -> Self {
+        Self {
+            exit_code: SingleExpectationComparison::NotSpecified,
+            stdout: SingleExpectationComparison::NotSpecified,
+            stderr: SingleExpectationComparison::NotSpecified,
+            snapshot_used: false,
+            snapshot_result: None,
+        }
+    }
+
+    /// Returns whether this comparison indicates a failure.
+    pub fn is_failure(&self) -> bool {
+        self.exit_code.is_failure()
+            || self.stdout.is_failure()
+            || self.stderr.is_failure()
+            || self
+                .snapshot_result
+                .as_ref()
+                .is_some_and(|r| r.is_failure())
+    }
+
+    /// Returns whether any expectations were checked.
+    pub const fn has_any_checks(&self) -> bool {
+        !matches!(self.exit_code, SingleExpectationComparison::NotSpecified)
+            || !matches!(self.stdout, SingleExpectationComparison::NotSpecified)
+            || !matches!(self.stderr, SingleExpectationComparison::NotSpecified)
+            || self.snapshot_used
+    }
+}
+
+/// Result of a snapshot comparison.
+#[derive(Debug)]
+pub enum SnapshotResult {
+    /// Snapshot matches.
+    Matches,
+    /// Snapshot differs (new snapshot created or update needed).
+    Differs {
+        /// Description of the difference.
+        message: String,
+    },
+}
+
+impl SnapshotResult {
+    /// Returns whether this result indicates a failure.
+    pub const fn is_failure(&self) -> bool {
+        matches!(self, Self::Differs { .. })
+    }
+}
+
+/// Combined test comparison result.
+pub struct TestComparison {
+    /// Oracle comparison (if oracle mode was used).
+    pub oracle: Option<OracleComparison>,
+    /// Expectation comparison (if expectations were defined).
+    pub expectation: ExpectationComparison,
+    /// Duration of the test run.
+    pub duration: std::time::Duration,
+}
+
+impl TestComparison {
+    /// Returns whether this comparison indicates a failure.
+    pub fn is_failure(&self) -> bool {
+        self.oracle.as_ref().is_some_and(|o| o.is_failure()) || self.expectation.is_failure()
+    }
+
+    /// Creates a skipped comparison.
+    pub fn skipped() -> Self {
+        Self {
+            oracle: None,
+            expectation: ExpectationComparison::not_specified(),
+            duration: std::time::Duration::default(),
+        }
+    }
+}
+
+/// Compares two strings, optionally ignoring whitespace.
+pub fn output_matches(oracle: &str, test: &str, ignore_whitespace: bool) -> bool {
+    if ignore_whitespace {
+        let whitespace_re = regex::Regex::new(r"\s+").unwrap();
+
+        let cleaned_oracle = whitespace_re.replace_all(oracle, " ").to_string();
+        let cleaned_test = whitespace_re.replace_all(test, " ").to_string();
+
+        cleaned_oracle == cleaned_test
+    } else {
+        oracle == test
+    }
+}
+
+/// Compares directory contents between oracle and test.
+pub fn diff_dirs(
+    oracle_path: &std::path::Path,
+    test_path: &std::path::Path,
+) -> anyhow::Result<DirComparison> {
+    use std::collections::HashMap;
+    use std::fs;
+
+    fn get_dir_entries(
+        dir_path: &std::path::Path,
+    ) -> anyhow::Result<HashMap<String, fs::FileType>> {
+        let mut entries = HashMap::new();
+        for entry in fs::read_dir(dir_path)? {
+            let entry = entry?;
+            let file_type = entry.file_type()?;
+            let filename = entry.file_name().to_string_lossy().to_string();
+
+            // Ignore raw coverage profile data files.
+            if filename.ends_with(".profraw") {
+                continue;
+            }
+
+            entries.insert(filename, file_type);
+        }
+
+        Ok(entries)
+    }
+
+    let mut entries = vec![];
+
+    let oracle_entries = get_dir_entries(oracle_path)?;
+    let test_entries = get_dir_entries(test_path)?;
+
+    // Look through all the files in the oracle directory
+    for (filename, file_type) in &oracle_entries {
+        if !test_entries.contains_key(filename) {
+            for left_only_file in walkdir::WalkDir::new(oracle_path.join(filename)) {
+                let entry = left_only_file?;
+                let left_only_path = entry.path();
+                entries.push(DirComparisonEntry::LeftOnly(left_only_path.to_owned()));
+            }
+
+            continue;
+        }
+
+        let oracle_file_path = oracle_path.join(filename);
+        let test_file_path = test_path.join(filename);
+
+        if file_type.is_file() {
+            let mut oracle_file = std::fs::OpenOptions::new()
+                .read(true)
+                .open(&oracle_file_path)?;
+            let mut oracle_bytes = vec![];
+            oracle_file.read_to_end(&mut oracle_bytes)?;
+
+            let mut test_file = std::fs::OpenOptions::new()
+                .read(true)
+                .open(&test_file_path)?;
+            let mut test_bytes = vec![];
+            test_file.read_to_end(&mut test_bytes)?;
+
+            if oracle_bytes != test_bytes {
+                let oracle_display_text = String::from_utf8_lossy(&oracle_bytes);
+                let test_display_text = String::from_utf8_lossy(&test_bytes);
+
+                entries.push(DirComparisonEntry::Different(
+                    oracle_file_path,
+                    oracle_display_text.to_string(),
+                    test_file_path,
+                    test_display_text.to_string(),
+                ));
+            }
+        } else if file_type.is_dir() {
+            let subdir_comparison =
+                diff_dirs(oracle_file_path.as_path(), test_file_path.as_path())?;
+            if let DirComparison::TestDiffers(subdir_entries) = subdir_comparison {
+                entries.extend(subdir_entries);
+            }
+        }
+    }
+
+    for (filename, file_type) in &test_entries {
+        if oracle_entries.contains_key(filename) {
+            continue;
+        }
+
+        if file_type.is_dir() {
+            for right_only_file in walkdir::WalkDir::new(test_path.join(filename)) {
+                let entry = right_only_file?;
+                let right_only_path = entry.path();
+                entries.push(DirComparisonEntry::RightOnly(right_only_path.to_owned()));
+            }
+        } else {
+            entries.push(DirComparisonEntry::RightOnly(test_path.join(filename)));
+        }
+    }
+
+    if entries.is_empty() {
+        Ok(DirComparison::Same)
+    } else {
+        Ok(DirComparison::TestDiffers(entries))
+    }
+}