core: Fix setting certain DisplayObject properties to NaN

This doesn't perfectly match Flash's behavior - I haven't been
able to reproduce the values produces when the DisplayObject
starts out with certain 'Matrix' values (a non-zero 'b' or 'd').

Howver, when the 'b' and 'd' matrix values are both 0, setting
'dobj.rotation = NaN' has no effect on the matrix, while
'dobj.scaleX = NaN' and 'dobj.scaleY = NaN' both treat 'NaN'
as 0 for the purposes of updating the matrix.

This fixes the tack shooter in Bloons Tower Defense 3, which
tries to set 'rotation = NaN' for spawned tacks.

core/src/display_object.rs

@@ -262,6 +262,20 @@ impl<'gc> DisplayObjectBase<'gc> {
         self.set_transformed_by_script(true);
         self.cache_scale_rotation();
         self.rotation = degrees;
+
+        // FIXME - this isn't quite correct. In Flash player,
+        // trying to set rotation to NaN does nothing if the current
+        // matrix 'b' and 'd' terms are both zero. However, if one
+        // of those terms is non-zero, then the entire matrix gets
+        // modified in a way that depends on its starting values.
+        // I haven't been able to figure out how to reproduce those
+        // values, so for now, we never modify the matrix if the
+        // rotation is NaN. Hopefully, there are no SWFs depending
+        // on the weird behavior when b or d is non-zero.
+        if degrees.into_radians().is_nan() {
+            return;
+        }
+
         let cos_x = f64::cos(degrees.into_radians());
         let sin_x = f64::sin(degrees.into_radians());
         let cos_y = f64::cos(degrees.into_radians() + self.skew);
@@ -278,10 +292,18 @@ impl<'gc> DisplayObjectBase<'gc> {
         self.scale_x
     }
 
-    fn set_scale_x(&mut self, value: Percent) {
+    fn set_scale_x(&mut self, mut value: Percent) {
         self.set_transformed_by_script(true);
         self.cache_scale_rotation();
         self.scale_x = value;
+
+        // Note - in order to match Flash's behavior, the 'scale_x' field is set to NaN
+        // (which gets reported back to ActionScript), but we treat it as 0 for
+        // the purposes of updating the matrix
+        if value.percent().is_nan() {
+            value = 0.0.into();
+        }
+
         let cos = f64::cos(self.rotation.into_radians());
         let sin = f64::sin(self.rotation.into_radians());
         let mut matrix = &mut self.transform.matrix;
@@ -294,10 +316,18 @@ impl<'gc> DisplayObjectBase<'gc> {
         self.scale_y
     }
 
-    fn set_scale_y(&mut self, value: Percent) {
+    fn set_scale_y(&mut self, mut value: Percent) {
         self.set_transformed_by_script(true);
         self.cache_scale_rotation();
         self.scale_y = value;
+
+        // Note - in order to match Flash's behavior, the 'scale_y' field is set to NaN
+        // (which gets reported back to ActionScript), but we treat it as 0 for
+        // the purposes of updating the matrix
+        if value.percent().is_nan() {
+            value = 0.0.into();
+        }
+
         let cos = f64::cos(self.rotation.into_radians() + self.skew);
         let sin = f64::sin(self.rotation.into_radians() + self.skew);
         let mut matrix = &mut self.transform.matrix;

tests/tests/swfs/avm2/displayobject_invalid_floats/Test.as

@@ -0,0 +1,45 @@
+
+package {
+	import flash.display.MovieClip;
+	import flash.geom.Matrix;
+
+	public class Test {
+		public function Test(main: MovieClip) {
+			
+			var props = ["rotation", "x", "y", "scaleX", "scaleY"];
+			// FIXME - we should also be testing Infinity and -Infinity here,
+			// but those give very weird values back in the matrix,
+			// and I havne't yet figured out how to reproduce them. Hopefully,
+			// there are no SWFs relying on the behavior.
+			for each (var prop in props) {
+				var clip = new MovieClip();
+				clip.transform.matrix = new Matrix(2, 0, 4, 0, 5, 6);
+				trace("Setting initial matrix: " + clip.transform.matrix);
+				trace("Initial value: clip[" + prop + "] = " + clip[prop]);
+				tryValue(clip, prop, NaN);
+				
+				clip = new MovieClip();
+				var newMat = new Matrix(2, 0, 4, 0, 5, 6);
+				clip.transform.matrix = newMat;
+				trace("Setting initial matrix: " + clip.transform.matrix);
+				trace("Initial value: clip[" + prop + "] = " + clip[prop]);
+				tryValue(clip, prop, NaN);
+				
+				clip = new MovieClip();
+				var newMat = new Matrix(7, 0, 9, 0, 11, 12);
+				clip.transform.matrix = newMat;
+				trace("Setting initial matrix: " + clip.transform.matrix);
+				trace("Initial value: clip[" + prop + "] = " + clip[prop]);
+				tryValue(clip, prop, NaN);
+			}
+		}
+	}
+}
+
+import flash.display.MovieClip;
+
+function tryValue(clip:MovieClip, prop:String, value:*) {
+	clip[prop] = value;
+	trace("Setting clip[" + prop + "] = " + value + " gave: " + clip[prop]);
+	trace("Matrix: " + clip.transform.matrix);
+}

tests/tests/swfs/avm2/displayobject_invalid_floats/output.txt

@@ -0,0 +1,60 @@
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[rotation] = 0
+Setting clip[rotation] = NaN gave: NaN
+Matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[rotation] = 0
+Setting clip[rotation] = NaN gave: NaN
+Matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Setting initial matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Initial value: clip[rotation] = 0
+Setting clip[rotation] = NaN gave: NaN
+Matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[x] = 5
+Setting clip[x] = NaN gave: 0
+Matrix: (a=2, b=0, c=4, d=0, tx=0, ty=6)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[x] = 5
+Setting clip[x] = NaN gave: 0
+Matrix: (a=2, b=0, c=4, d=0, tx=0, ty=6)
+Setting initial matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Initial value: clip[x] = 11
+Setting clip[x] = NaN gave: 0
+Matrix: (a=7, b=0, c=9, d=0, tx=0, ty=12)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[y] = 6
+Setting clip[y] = NaN gave: 0
+Matrix: (a=2, b=0, c=4, d=0, tx=5, ty=0)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[y] = 6
+Setting clip[y] = NaN gave: 0
+Matrix: (a=2, b=0, c=4, d=0, tx=5, ty=0)
+Setting initial matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Initial value: clip[y] = 12
+Setting clip[y] = NaN gave: 0
+Matrix: (a=7, b=0, c=9, d=0, tx=11, ty=0)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[scaleX] = 2
+Setting clip[scaleX] = NaN gave: NaN
+Matrix: (a=0, b=0, c=4, d=0, tx=5, ty=6)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[scaleX] = 2
+Setting clip[scaleX] = NaN gave: NaN
+Matrix: (a=0, b=0, c=4, d=0, tx=5, ty=6)
+Setting initial matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Initial value: clip[scaleX] = 7
+Setting clip[scaleX] = NaN gave: NaN
+Matrix: (a=0, b=0, c=9, d=0, tx=11, ty=12)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[scaleY] = 4
+Setting clip[scaleY] = NaN gave: NaN
+Matrix: (a=2, b=0, c=0, d=0, tx=5, ty=6)
+Setting initial matrix: (a=2, b=0, c=4, d=0, tx=5, ty=6)
+Initial value: clip[scaleY] = 4
+Setting clip[scaleY] = NaN gave: NaN
+Matrix: (a=2, b=0, c=0, d=0, tx=5, ty=6)
+Setting initial matrix: (a=7, b=0, c=9, d=0, tx=11, ty=12)
+Initial value: clip[scaleY] = 9
+Setting clip[scaleY] = NaN gave: NaN
+Matrix: (a=7, b=0, c=0, d=0, tx=11, ty=12)

tests/tests/swfs/avm2/displayobject_invalid_floats/test.toml

@@ -0,0 +1 @@
+num_frames = 1