tests: Fix matching tests with test kinds

libtest_mimic adds test kind as a prefix (e.g. `[kind] test`
instead of `test`), which has to be taken into account when
filtering for possible test candidates. This patch ensures
compatibility with libtest_mimic when using test kinds.
This commit is contained in:
Kamil Jarosz 2024-04-13 02:25:44 +02:00 committed by Nathan Adams
parent e40374c403
commit 4715c2d459
3 changed files with 49 additions and 17 deletions

1
Cargo.lock generated
View File

@ -5248,6 +5248,7 @@ dependencies = [
"futures", "futures",
"image 0.25.1", "image 0.25.1",
"libtest-mimic", "libtest-mimic",
"regex",
"ruffle_core", "ruffle_core",
"ruffle_render_wgpu", "ruffle_render_wgpu",
"ruffle_test_framework", "ruffle_test_framework",

View File

@ -20,6 +20,7 @@ lzma = ["ruffle_test_framework/lzma"]
[dependencies] [dependencies]
ruffle_render_wgpu = { path = "../render/wgpu", optional = true } ruffle_render_wgpu = { path = "../render/wgpu", optional = true }
regex = "1.10.4"
[dev-dependencies] [dev-dependencies]
ruffle_core = { path = "../core", features = ["deterministic", "timeline_debug", "avm_debug", "audio", "mp3", "default_font"] } ruffle_core = { path = "../core", features = ["deterministic", "timeline_debug", "avm_debug", "audio", "mp3", "default_font"] }

View File

@ -8,6 +8,7 @@ use crate::shared_object::{shared_object_avm1, shared_object_avm2, shared_object
use anyhow::Context; use anyhow::Context;
use anyhow::Result; use anyhow::Result;
use libtest_mimic::{Arguments, Trial}; use libtest_mimic::{Arguments, Trial};
use regex::Regex;
use ruffle_test_framework::options::TestOptions; use ruffle_test_framework::options::TestOptions;
use ruffle_test_framework::runner::TestStatus; use ruffle_test_framework::runner::TestStatus;
use ruffle_test_framework::test::Test; use ruffle_test_framework::test::Test;
@ -22,19 +23,38 @@ mod shared_object;
const TEST_TOML_NAME: &str = "test.toml"; const TEST_TOML_NAME: &str = "test.toml";
/// Convert the filter (e.g. from the CLI) to a test name.
///
/// These two values may differ due to how
/// libtest_mimic handles test kind annotations:
/// a test may be named `test` or `[kind] test` when a kind is present.
/// This function removes the "kind" prefix from
/// the name to match tests similarly to libtest_mimic.
///
/// See [`Arguments::is_filtered_out()`].
/// See [`libtest_mimic::TestInfo::test_name_with_kind()`].
fn filter_to_test_name(filter: &str) -> String {
Regex::new("^\\[[^]]+] ")
.unwrap()
.replace(filter, "")
.to_string()
}
fn is_candidate(args: &Arguments, test_name: &str) -> bool { fn is_candidate(args: &Arguments, test_name: &str) -> bool {
if let Some(filter) = &args.filter { if let Some(filter) = &args.filter {
let expected_test_name = filter_to_test_name(filter);
match args.exact { match args.exact {
true if test_name != filter => return false, true if test_name != expected_test_name => return false,
false if !test_name.contains(filter) => return false, false if !test_name.contains(&expected_test_name) => return false,
_ => {} _ => {}
}; };
} }
for skip_filter in &args.skip { for skip_filter in &args.skip {
let skipped_test_name = filter_to_test_name(skip_filter);
match args.exact { match args.exact {
true if test_name == skip_filter => return false, true if test_name == skipped_test_name => return false,
false if test_name.contains(skip_filter) => return false, false if test_name.contains(&skipped_test_name) => return false,
_ => {} _ => {}
} }
} }
@ -65,19 +85,7 @@ fn main() {
let root = Path::new("tests/swfs"); let root = Path::new("tests/swfs");
let mut tests: Vec<Trial> = if filter_exact { let mut tests: Vec<Trial> = if filter_exact {
let name = args.filter.as_ref().unwrap(); look_up_test(root, &args).map_or_else(Vec::new, |trial| vec![trial])
let absolute_root = std::fs::canonicalize(root).unwrap();
let path = absolute_root
.join(name)
.join(TEST_TOML_NAME)
.canonicalize()
.expect("test should exist");
path.strip_prefix(absolute_root).expect("path traversal");
if path.is_file() {
vec![run_test(&args, &path, name)]
} else {
vec![]
}
} else { } else {
walkdir::WalkDir::new(root) walkdir::WalkDir::new(root)
.into_iter() .into_iter()
@ -123,6 +131,28 @@ fn main() {
libtest_mimic::run(&args, tests).exit() libtest_mimic::run(&args, tests).exit()
} }
fn look_up_test(root: &Path, args: &Arguments) -> Option<Trial> {
let name = filter_to_test_name(args.filter.as_ref().unwrap());
let absolute_root = std::fs::canonicalize(root).unwrap();
let path = absolute_root
.join(&name)
.join(TEST_TOML_NAME)
.canonicalize()
.ok()?;
// Make sure that:
// 1. There's no path traversal (e.g. `cargo test ../../test`)
// 2. The path is still exact (e.g. `cargo test avm1/../avm1/test`)
if path.strip_prefix(absolute_root).ok()? != Path::new(&name).join(TEST_TOML_NAME) {
return None;
}
if path.is_file() {
Some(run_test(args, &path, &name))
} else {
None
}
}
fn run_test(args: &Arguments, file: &Path, name: &str) -> Trial { fn run_test(args: &Arguments, file: &Path, name: &str) -> Trial {
let root = VfsPath::new(PhysicalFS::new(file.parent().unwrap())); let root = VfsPath::new(PhysicalFS::new(file.parent().unwrap()));
let test = Test::from_options( let test = Test::from_options(