2019-08-26 23:38:37 +00:00
|
|
|
//! Tests running SWFs in a headless Ruffle instance.
|
|
|
|
//!
|
2021-01-15 11:47:23 +00:00
|
|
|
//! Trace output can be compared with correct output from the official Flash Player.
|
2019-08-26 23:38:37 +00:00
|
|
|
|
2023-11-09 14:36:46 +00:00
|
|
|
use crate::environment::NativeEnvironment;
|
2023-01-27 18:42:41 +00:00
|
|
|
use crate::external_interface::tests::{external_interface_avm1, external_interface_avm2};
|
2023-07-03 01:58:18 +00:00
|
|
|
use crate::shared_object::{shared_object_avm1, shared_object_avm2, shared_object_self_ref_avm1};
|
2023-01-27 16:33:38 +00:00
|
|
|
use anyhow::Context;
|
2023-01-27 17:49:45 +00:00
|
|
|
use anyhow::Result;
|
2023-01-24 01:23:33 +00:00
|
|
|
use libtest_mimic::{Arguments, Trial};
|
2024-04-13 00:25:44 +00:00
|
|
|
use regex::Regex;
|
2023-11-12 21:57:44 +00:00
|
|
|
use ruffle_test_framework::options::TestOptions;
|
2024-01-18 18:47:25 +00:00
|
|
|
use ruffle_test_framework::runner::TestStatus;
|
2023-11-09 11:34:32 +00:00
|
|
|
use ruffle_test_framework::test::Test;
|
2023-11-12 21:57:44 +00:00
|
|
|
use ruffle_test_framework::vfs::{PhysicalFS, VfsPath};
|
|
|
|
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
|
2022-06-20 18:10:40 +00:00
|
|
|
use std::path::Path;
|
2024-01-18 18:47:25 +00:00
|
|
|
use std::thread::sleep;
|
2019-12-18 21:25:54 +00:00
|
|
|
|
2023-11-09 14:36:46 +00:00
|
|
|
mod environment;
|
2023-01-27 18:36:51 +00:00
|
|
|
mod external_interface;
|
2023-01-27 18:47:27 +00:00
|
|
|
mod shared_object;
|
2020-02-17 20:32:25 +00:00
|
|
|
|
2024-04-13 00:25:44 +00:00
|
|
|
const TEST_TOML_NAME: &str = "test.toml";
|
|
|
|
|
2024-04-13 00:25:44 +00:00
|
|
|
/// Convert the filter (e.g. from the CLI) to a test name.
|
|
|
|
///
|
|
|
|
/// These two values may differ due to how
|
|
|
|
/// libtest_mimic handles test kind annotations:
|
|
|
|
/// a test may be named `test` or `[kind] test` when a kind is present.
|
|
|
|
/// This function removes the "kind" prefix from
|
|
|
|
/// the name to match tests similarly to libtest_mimic.
|
|
|
|
///
|
|
|
|
/// See [`Arguments::is_filtered_out()`].
|
|
|
|
/// See [`libtest_mimic::TestInfo::test_name_with_kind()`].
|
|
|
|
fn filter_to_test_name(filter: &str) -> String {
|
|
|
|
Regex::new("^\\[[^]]+] ")
|
|
|
|
.unwrap()
|
|
|
|
.replace(filter, "")
|
|
|
|
.to_string()
|
|
|
|
}
|
|
|
|
|
2023-01-28 10:47:07 +00:00
|
|
|
fn is_candidate(args: &Arguments, test_name: &str) -> bool {
|
|
|
|
if let Some(filter) = &args.filter {
|
2024-04-13 00:25:44 +00:00
|
|
|
let expected_test_name = filter_to_test_name(filter);
|
2023-01-28 10:47:07 +00:00
|
|
|
match args.exact {
|
2024-04-13 00:25:44 +00:00
|
|
|
true if test_name != expected_test_name => return false,
|
|
|
|
false if !test_name.contains(&expected_test_name) => return false,
|
2023-01-28 10:47:07 +00:00
|
|
|
_ => {}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
for skip_filter in &args.skip {
|
2024-04-13 00:25:44 +00:00
|
|
|
let skipped_test_name = filter_to_test_name(skip_filter);
|
2023-01-28 10:47:07 +00:00
|
|
|
match args.exact {
|
2024-04-13 00:25:44 +00:00
|
|
|
true if test_name == skipped_test_name => return false,
|
|
|
|
false if test_name.contains(&skipped_test_name) => return false,
|
2023-01-28 10:47:07 +00:00
|
|
|
_ => {}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
true
|
|
|
|
}
|
|
|
|
|
2023-01-24 01:23:33 +00:00
|
|
|
fn main() {
|
|
|
|
let args = Arguments::from_args();
|
|
|
|
|
2024-01-17 22:52:21 +00:00
|
|
|
let _ = env_logger::Builder::from_env(
|
|
|
|
env_logger::Env::default().default_filter_or("info,wgpu_core=warn,wgpu_hal=warn"),
|
|
|
|
)
|
|
|
|
.format_timestamp(None)
|
|
|
|
.is_test(true)
|
|
|
|
.try_init();
|
|
|
|
|
|
|
|
let subscriber = tracing_subscriber::fmt::Subscriber::builder()
|
|
|
|
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
|
|
|
.finish();
|
|
|
|
// Ignore error if it's already been set
|
|
|
|
let _ = tracing::subscriber::set_global_default(subscriber);
|
|
|
|
|
2024-04-13 00:25:44 +00:00
|
|
|
// When this is true, we are looking for one specific test.
|
|
|
|
// This is an important optimization for nextest,
|
|
|
|
// as it executes tests one by one.
|
|
|
|
let filter_exact = args.exact && args.filter.is_some();
|
|
|
|
|
2023-01-24 01:23:33 +00:00
|
|
|
let root = Path::new("tests/swfs");
|
2024-04-13 00:25:44 +00:00
|
|
|
let mut tests: Vec<Trial> = if filter_exact {
|
2024-04-13 00:25:44 +00:00
|
|
|
look_up_test(root, &args).map_or_else(Vec::new, |trial| vec![trial])
|
2024-04-13 00:25:44 +00:00
|
|
|
} else {
|
|
|
|
walkdir::WalkDir::new(root)
|
|
|
|
.into_iter()
|
|
|
|
.map(Result::unwrap)
|
|
|
|
.filter(|entry| entry.file_type().is_file() && entry.file_name() == TEST_TOML_NAME)
|
|
|
|
.filter_map(|file| {
|
|
|
|
let name = file
|
|
|
|
.path()
|
|
|
|
.parent()?
|
|
|
|
.strip_prefix(root)
|
|
|
|
.context("Couldn't strip root prefix from test dir")
|
|
|
|
.unwrap()
|
|
|
|
.to_string_lossy()
|
|
|
|
.replace('\\', "/");
|
|
|
|
if is_candidate(&args, &name) {
|
|
|
|
Some(run_test(&args, file.path(), &name))
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect()
|
|
|
|
};
|
2023-01-24 01:23:33 +00:00
|
|
|
|
|
|
|
// Manual tests here, since #[test] doesn't work once we use our own test harness
|
2023-11-09 14:36:46 +00:00
|
|
|
tests.push(Trial::test("shared_object_avm1", || {
|
|
|
|
shared_object_avm1(&NativeEnvironment)
|
|
|
|
}));
|
|
|
|
tests.push(Trial::test("shared_object_self_ref_avm1", || {
|
|
|
|
shared_object_self_ref_avm1(&NativeEnvironment)
|
|
|
|
}));
|
|
|
|
tests.push(Trial::test("shared_object_avm2", || {
|
|
|
|
shared_object_avm2(&NativeEnvironment)
|
|
|
|
}));
|
|
|
|
tests.push(Trial::test("external_interface_avm1", || {
|
|
|
|
external_interface_avm1(&NativeEnvironment)
|
|
|
|
}));
|
|
|
|
tests.push(Trial::test("external_interface_avm2", || {
|
|
|
|
external_interface_avm2(&NativeEnvironment)
|
|
|
|
}));
|
2023-01-24 01:23:33 +00:00
|
|
|
|
|
|
|
tests.sort_unstable_by(|a, b| a.name().cmp(b.name()));
|
|
|
|
|
|
|
|
libtest_mimic::run(&args, tests).exit()
|
|
|
|
}
|
2023-11-12 22:00:52 +00:00
|
|
|
|
2024-04-13 00:25:44 +00:00
|
|
|
fn look_up_test(root: &Path, args: &Arguments) -> Option<Trial> {
|
|
|
|
let name = filter_to_test_name(args.filter.as_ref().unwrap());
|
|
|
|
let absolute_root = std::fs::canonicalize(root).unwrap();
|
|
|
|
let path = absolute_root
|
|
|
|
.join(&name)
|
|
|
|
.join(TEST_TOML_NAME)
|
|
|
|
.canonicalize()
|
|
|
|
.ok()?;
|
|
|
|
|
|
|
|
// Make sure that:
|
|
|
|
// 1. There's no path traversal (e.g. `cargo test ../../test`)
|
|
|
|
// 2. The path is still exact (e.g. `cargo test avm1/../avm1/test`)
|
|
|
|
if path.strip_prefix(absolute_root).ok()? != Path::new(&name).join(TEST_TOML_NAME) {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
if path.is_file() {
|
|
|
|
Some(run_test(args, &path, &name))
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-04-13 00:25:44 +00:00
|
|
|
fn run_test(args: &Arguments, file: &Path, name: &str) -> Trial {
|
|
|
|
let root = VfsPath::new(PhysicalFS::new(file.parent().unwrap()));
|
2023-11-12 22:00:52 +00:00
|
|
|
let test = Test::from_options(
|
|
|
|
TestOptions::read(&root.join("test.toml").unwrap())
|
|
|
|
.context("Couldn't load test options")
|
|
|
|
.unwrap(),
|
|
|
|
root,
|
2024-04-13 00:25:44 +00:00
|
|
|
name.to_string(),
|
2023-11-12 22:00:52 +00:00
|
|
|
)
|
|
|
|
.with_context(|| format!("Couldn't create test {name}"))
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let ignore = !test.should_run(!args.list, &NativeEnvironment);
|
|
|
|
|
|
|
|
let mut trial = Trial::test(test.name.to_string(), move || {
|
|
|
|
let test = AssertUnwindSafe(test);
|
2024-01-18 18:47:25 +00:00
|
|
|
let unwind_result = catch_unwind(|| {
|
|
|
|
let mut runner = test.create_test_runner(&NativeEnvironment)?;
|
|
|
|
|
|
|
|
loop {
|
2024-01-17 19:47:03 +00:00
|
|
|
runner.tick();
|
|
|
|
match runner.test()? {
|
2024-01-18 18:47:25 +00:00
|
|
|
TestStatus::Continue => {}
|
|
|
|
TestStatus::Sleep(duration) => sleep(duration),
|
|
|
|
TestStatus::Finished => break,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Result::<_>::Ok(())
|
|
|
|
});
|
2023-11-12 22:00:52 +00:00
|
|
|
if test.options.known_failure {
|
|
|
|
match unwind_result {
|
|
|
|
Ok(Ok(())) => Err(
|
|
|
|
format!("{} was known to be failing, but now passes successfully. Please update it and remove `known_failure = true`!", test.name).into()
|
|
|
|
),
|
|
|
|
Ok(Err(_)) | Err(_) => Ok(()),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
match unwind_result {
|
|
|
|
Ok(r) => Ok(r?),
|
|
|
|
Err(e) => resume_unwind(e),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
if ignore {
|
|
|
|
trial = trial.with_ignored_flag(true);
|
|
|
|
}
|
|
|
|
trial
|
|
|
|
}
|