tests: Move tests away from giant macro to individual test.toml files that describe the test

Cargo.lock

@@ -2351,6 +2351,17 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libtest-mimic"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7b603516767d1ab23d0de09d023e62966c3322f7148297c35cf3d97aa8b37fa"
+dependencies = [
+ "clap",
+ "termcolor",
+ "threadpool",
+]
+
 [[package]]
 name = "libz-sys"
 version = "1.1.8"
@@ -3620,6 +3631,7 @@ dependencies = [
  "log",
  "lyon",
  "png",
+ "serde",
  "smallvec",
  "swf",
  "thiserror",
@@ -4375,11 +4387,15 @@ dependencies = [
  "env_logger",
  "futures",
  "image",
+ "libtest-mimic",
  "pretty_assertions",
  "regex",
  "ruffle_core",
  "ruffle_input_format",
  "ruffle_render_wgpu",
+ "serde",
+ "toml",
+ "walkdir",
 ]
 
 [[package]]

render/Cargo.toml

@@ -20,6 +20,7 @@ thiserror = "1.0"
 wasm-bindgen = { version = "=0.2.83", optional = true }
 gc-arena = { workspace = true }
 enum-map = "2.4.2"
+serde = "1.0.152"
 
 [dependencies.jpeg-decoder]
 version = "0.3.0"

render/src/backend.rs

@@ -6,6 +6,7 @@ use crate::error::Error;
 use crate::shape_utils::DistilledShape;
 use downcast_rs::{impl_downcast, Downcast};
 use gc_arena::{Collect, GcCell, MutationContext};
+use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::rc::Rc;
 use swf;
@@ -205,7 +206,7 @@ pub enum Context3DCommand<'gc> {
 #[derive(Copy, Clone, Debug)]
 pub struct ShapeHandle(pub usize);
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
 pub struct ViewportDimensions {
     /// The dimensions of the stage's containing viewport.
     pub width: u32,

tests/Cargo.toml

@@ -25,3 +25,12 @@ imgtests = ["ruffle_render_wgpu"]
 approx = "0.5.1"
 pretty_assertions = "1.3.0"
 env_logger = "0.10.0"
+serde = "1.0"
+toml = "0.5.10"
+libtest-mimic = "0.6.0"
+walkdir = "2.3.2"
+
+[[test]]
+name = "tests"
+harness = false
+path = "tests/regression_tests.rs"

tests/README.md

@@ -0,0 +1,37 @@
+# SWF Regression Tests
+
+Inside [tests/swfs](tests/swfs) is a large collection of automated tests that are based around running a swf and seeing what happens.
+
+To create a test, make a directory that looks like the following, at minimum:
+
+- directory/
+  - test.swf
+  - test.toml
+  - output.txt
+
+As best practice, please also include any source used to make the swf - such as `test.fla` and any actionscript files.
+
+
+# Test Structure
+## test.toml
+Except for `num_frames`, every other field and section is optional.
+
+```toml
+num_frames = 1 # The amount of frames of the swf to run
+sleep_to_meet_frame_rate = false # If true, slow the tick rate to match the movies requested fps rate
+image = false # If true, capture a screenshot of the movie and compare it against a "known good" image
+ignore = false # If true, ignore this test. Please comment why, ideally link to an issue, so we know what's up
+
+# Sometimes floating point math doesn't exactly 100% match between flash and rust.
+# If you encounter this in a test, the following section will change the output testing from "exact" to "approximate"
+# (when it comes to floating point numbers, at least.)
+[approximations]
+number_patterns = [] # A list of regex patterns with capture groups to additionally treat as approximate numbers
+epsilon = 0.0 # The upper bound of any rounding errors. Default is the difference between 1.0 and the next largest representable number
+max_relative = 0.0 # The default relative tolerance for testing values that are far-apart. Default is the difference between 1.0 and the next largest representable number
+
+# Options for the player used to run this swf 
+[player_options]
+max_execution_duration = { secs = 15, nanos = 0} # How long can actionscript execute for before being forcefully stopped
+viewport_dimensions = { width = 100, height = 100, scale_factor = 1 } # The size of the player. Defaults to the swfs stage size
+```

tests/tests/swfs/avm1/action_to_integer/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/add/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/add2/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/add_property/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/add_swf4/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/add_swf5/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/arguments/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_call_method/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_concat/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_constructor/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_enumerate/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_length/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_properties/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_prototyping/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_slice/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_sort/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_splice/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/array_trivial/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as1_constructor_v6/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as1_constructor_v7/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as2_oop/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as2_super_and_this_v6/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as2_super_and_this_v8/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as2_super_via_manual_prototype/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_broadcaster/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_broadcaster_initialize/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version_swf5/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version_swf6/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version_swf7/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version_swf8/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_set_prop_flags_version_swf9/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/as_transformed_flag/test.toml

@@ -0,0 +1 @@
+num_frames = 3

tests/tests/swfs/avm1/attach_movie/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bad_placeobject_clipaction/test.toml

@@ -0,0 +1 @@
+num_frames = 2

tests/tests/swfs/avm1/bad_swf_tag_past_eof/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bevel_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitand/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data_compare/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data_copypixels/test.toml

@@ -0,0 +1,2 @@
+num_frames = 2
+image = true

tests/tests/swfs/avm1/bitmap_data_max_size_swf10/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data_max_size_swf9/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data_noise/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_data_threshold/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitmap_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitor/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/biturshift/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/biturshift_swf8/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/bitxor/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/blur_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/button_children/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/button_order/test.toml

@@ -0,0 +1 @@
+num_frames = 2

tests/tests/swfs/avm1/call/test.toml

@@ -0,0 +1 @@
+num_frames = 2

tests/tests/swfs/avm1/call_method_empty_name/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/clip_events/test.toml

@@ -0,0 +1 @@
+num_frames = 4

tests/tests/swfs/avm1/closure_scope/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/color/test.toml

@@ -0,0 +1,2 @@
+num_frames = 1
+image = true

tests/tests/swfs/avm1/color_matrix_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/color_transform/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/conflicting_instance_names/test.toml

@@ -0,0 +1 @@
+num_frames = 6

tests/tests/swfs/avm1/constructor_function/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/context_menu/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/context_menu_item/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/convolution_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/create_empty_movie_clip/test.toml

@@ -0,0 +1 @@
+num_frames = 2

tests/tests/swfs/avm1/cross_movie_root/test.toml

@@ -0,0 +1 @@
+num_frames = 5

tests/tests/swfs/avm1/custom_clip_methods/test.toml

@@ -0,0 +1 @@
+num_frames = 3

tests/tests/swfs/avm1/date/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/default_names/test.toml

@@ -0,0 +1 @@
+num_frames = 6

tests/tests/swfs/avm1/define_function2_preload/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/define_function2_preload_order/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/define_function_case_sensitive/test.toml

@@ -0,0 +1 @@
+num_frames = 2

tests/tests/swfs/avm1/define_local/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/define_local_with_paths/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/delete/test.toml

@@ -0,0 +1 @@
+num_frames = 3

tests/tests/swfs/avm1/displacement_map_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/divide_swf4/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/do_init_action/test.toml

@@ -0,0 +1 @@
+num_frames = 3

tests/tests/swfs/avm1/drag_drop/test.toml

@@ -0,0 +1 @@
+num_frames = 14

tests/tests/swfs/avm1/drop_shadow_filter/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/duplicate_movie_clip/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/duplicate_movie_clip_drawing/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_align/test.toml

@@ -0,0 +1,4 @@
+num_frames = 1
+
+[approximations]
+epsilon = 3.0

tests/tests/swfs/avm1/edittext_antialiastype/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_autosize/test.toml

@@ -0,0 +1,4 @@
+num_frames = 1
+
+[approximations]
+epsilon = 0.1

tests/tests/swfs/avm1/edittext_bullet/test.toml

@@ -0,0 +1,4 @@
+num_frames = 1
+
+[approximations]
+epsilon = 3.0

tests/tests/swfs/avm1/edittext_default_format/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_font_size/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_hscroll/test.toml

@@ -0,0 +1,4 @@
+num_frames = 1
+
+[approximations]
+epsilon = 3.0

tests/tests/swfs/avm1/edittext_html_entity/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_html_roundtrip/test.toml

@@ -0,0 +1 @@
+num_frames = 1

tests/tests/swfs/avm1/edittext_leading/test.toml

@@ -0,0 +1 @@
+num_frames = 1