From 1bb2422595de366a58421d9098278abc174d6847 Mon Sep 17 00:00:00 2001 From: Lord-McSweeney <84632019+Lord-McSweeney@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:59:26 -0800 Subject: [PATCH] avm2: Correctness fixes to AMF serialization (#13537) Fixes some cases where `amf::serialize_value` returns `None` --------- Co-authored-by: Lord-McSweeney --- core/src/avm2/amf.rs | 14 +++- core/src/avm2/globals/flash/net/socket.rs | 40 ++++++----- .../avm2/globals/flash/utils/byte_array.rs | 46 ++++++------ tests/tests/swfs/avm2/amf_function/Test.as | 66 ++++++++++++++++++ tests/tests/swfs/avm2/amf_function/output.txt | 46 ++++++++++++ tests/tests/swfs/avm2/amf_function/test.swf | Bin 0 -> 1271 bytes tests/tests/swfs/avm2/amf_function/test.toml | 1 + 7 files changed, 173 insertions(+), 40 deletions(-) create mode 100644 tests/tests/swfs/avm2/amf_function/Test.as create mode 100644 tests/tests/swfs/avm2/amf_function/output.txt create mode 100644 tests/tests/swfs/avm2/amf_function/test.swf create mode 100644 tests/tests/swfs/avm2/amf_function/test.toml diff --git a/core/src/avm2/amf.rs b/core/src/avm2/amf.rs index 017412344..b5bc3a7dc 100644 --- a/core/src/avm2/amf.rs +++ b/core/src/avm2/amf.rs @@ -110,7 +110,7 @@ pub fn serialize_value<'gc>( .iter() .map(|v| { serialize_value(activation, v, amf_version, object_table) - .expect("Unexpected non-object value in object vector") + .unwrap_or(AmfValue::Undefined) }) .collect(); @@ -421,7 +421,17 @@ pub fn deserialize_value<'gc>( let class = alias_to_class(activation, name)?; let storage = VectorStorage::from_values( vec.iter() - .map(|v| deserialize_value(activation, v)) + .map(|v| { + deserialize_value(activation, v).map(|value| { + // There's no Vector.: convert any + // Undefined items in the Vector to Null. + if matches!(value, Value::Undefined) { + Value::Null + } else { + value + } + }) + }) .collect::, _>>()?, *is_fixed, Some(class), diff --git a/core/src/avm2/globals/flash/net/socket.rs b/core/src/avm2/globals/flash/net/socket.rs index fa92ba35e..e75991122 100644 --- a/core/src/avm2/globals/flash/net/socket.rs +++ b/core/src/avm2/globals/flash/net/socket.rs @@ -646,24 +646,28 @@ pub fn write_object<'gc>( ObjectEncoding::Amf0 => AMFVersion::AMF0, ObjectEncoding::Amf3 => AMFVersion::AMF3, }; - if let Some(amf) = - crate::avm2::amf::serialize_value(activation, obj, amf_version, &mut Default::default()) - { - let element = Element::new("", Rc::new(amf)); - let mut lso = flash_lso::types::Lso::new(vec![element], "", amf_version); - let bytes = flash_lso::write::write_to_bytes(&mut lso) - .map_err(|_| "Failed to serialize object")?; - // This is kind of hacky: We need to strip out the header and any padding so that we only write - // the value. In the future, there should be a method to do this in the flash_lso crate. - let element_padding = match amf_version { - AMFVersion::AMF0 => 8, - AMFVersion::AMF3 => 7, - }; - socket.write_bytes( - &bytes[flash_lso::write::header_length(&lso.header) + element_padding - ..bytes.len() - 1], - ); - } + + let amf = crate::avm2::amf::serialize_value( + activation, + obj, + amf_version, + &mut Default::default(), + ) + .unwrap_or(flash_lso::types::Value::Undefined); + + let element = Element::new("", Rc::new(amf)); + let mut lso = flash_lso::types::Lso::new(vec![element], "", amf_version); + let bytes = + flash_lso::write::write_to_bytes(&mut lso).map_err(|_| "Failed to serialize object")?; + // This is kind of hacky: We need to strip out the header and any padding so that we only write + // the value. In the future, there should be a method to do this in the flash_lso crate. + let element_padding = match amf_version { + AMFVersion::AMF0 => 8, + AMFVersion::AMF3 => 7, + }; + socket.write_bytes( + &bytes[flash_lso::write::header_length(&lso.header) + element_padding..bytes.len() - 1], + ); } Ok(Value::Undefined) diff --git a/core/src/avm2/globals/flash/utils/byte_array.rs b/core/src/avm2/globals/flash/utils/byte_array.rs index 799064b6d..bd58837ee 100644 --- a/core/src/avm2/globals/flash/utils/byte_array.rs +++ b/core/src/avm2/globals/flash/utils/byte_array.rs @@ -758,6 +758,7 @@ pub fn read_object<'gc>( let bytes = bytearray .read_at(bytearray.bytes_available(), bytearray.position()) .map_err(|e| e.to_avm(activation))?; + let (bytes_left, value) = match bytearray.object_encoding() { ObjectEncoding::Amf0 => { let mut decoder = AMF0Decoder::default(); @@ -799,26 +800,31 @@ pub fn write_object<'gc>( ObjectEncoding::Amf0 => AMFVersion::AMF0, ObjectEncoding::Amf3 => AMFVersion::AMF3, }; - if let Some(amf) = - crate::avm2::amf::serialize_value(activation, obj, amf_version, &mut Default::default()) - { - let element = Element::new("", Rc::new(amf)); - let mut lso = flash_lso::types::Lso::new(vec![element], "", amf_version); - let bytes = flash_lso::write::write_to_bytes(&mut lso) - .map_err(|_| "Failed to serialize object")?; - // This is kind of hacky: We need to strip out the header and any padding so that we only write - // the value. In the future, there should be a method to do this in the flash_lso crate. - let element_padding = match amf_version { - AMFVersion::AMF0 => 8, - AMFVersion::AMF3 => 7, - }; - bytearray - .write_bytes( - &bytes[flash_lso::write::header_length(&lso.header) + element_padding - ..bytes.len() - 1], - ) - .map_err(|e| e.to_avm(activation))?; - } + + let amf = crate::avm2::amf::serialize_value( + activation, + obj, + amf_version, + &mut Default::default(), + ) + .unwrap_or(flash_lso::types::Value::Undefined); + + let element = Element::new("", Rc::new(amf)); + let mut lso = flash_lso::types::Lso::new(vec![element], "", amf_version); + let bytes = + flash_lso::write::write_to_bytes(&mut lso).map_err(|_| "Failed to serialize object")?; + // This is kind of hacky: We need to strip out the header and any padding so that we only write + // the value. In the future, there should be a method to do this in the flash_lso crate. + let element_padding = match amf_version { + AMFVersion::AMF0 => 8, + AMFVersion::AMF3 => 7, + }; + bytearray + .write_bytes( + &bytes[flash_lso::write::header_length(&lso.header) + element_padding + ..bytes.len() - 1], + ) + .map_err(|e| e.to_avm(activation))?; } Ok(Value::Undefined) diff --git a/tests/tests/swfs/avm2/amf_function/Test.as b/tests/tests/swfs/avm2/amf_function/Test.as new file mode 100644 index 000000000..c03d30a15 --- /dev/null +++ b/tests/tests/swfs/avm2/amf_function/Test.as @@ -0,0 +1,66 @@ +package { + import flash.display.MovieClip; + import flash.utils.*; + + public class Test extends MovieClip { + public function Test() { + var t1:Function = new Function(); + var t2:Vector. = Vector.([new Function(),new Function(),new Function()]); + var t3:Vector. = Vector.([null,new Function()]); + var t4:Vector. = Vector.([]); + var t5:Array = [new Function(),new Function()]; + runTest("Just function",0,t1); + runTest("Just function",3,t1); + runTest("Function vector",3,t2); + runTest("Function vector with null element",3,t3); + runTest("Empty function vector",3,t4); + runTest("Array with two function elements",3,t5); + } + + public function printByteArray(name:String, array:ByteArray):void { + trace("Printing ByteArray:"); + var str:* = ""; + var i:* = 0; + while(i < array.length) + { + str += array[i]; + if(i != array.length - 1) + { + str += ", "; + } + i++; + } + trace(str); + } + + public function runTest(name:String, amfversion:*, obj:*):void { + trace("Running test \"" + name + "\" with AMF" + amfversion); + var bytearray:* = new ByteArray(); + bytearray.objectEncoding = amfversion; + bytearray.writeObject(obj); + printByteArray(name,bytearray); + bytearray.position = 0; + var read:* = bytearray.readObject(); + trace("read back: " + getQualifiedClassName(read)); + if(read && read instanceof Vector.<*>) + { + trace("Was vector, length " + read.length + ". Elements:"); + for(var i in read) + { + trace(i + "th element: " + read[i]); + } + } + if(read && read instanceof Array) + { + trace("Was array, length " + read.length + ". Elements:"); + for(i in read) + { + trace(i + "th element: " + read[i]); + } + } + trace(read); + trace("Done with test!"); + } + } +} + diff --git a/tests/tests/swfs/avm2/amf_function/output.txt b/tests/tests/swfs/avm2/amf_function/output.txt new file mode 100644 index 000000000..a21ac573f --- /dev/null +++ b/tests/tests/swfs/avm2/amf_function/output.txt @@ -0,0 +1,46 @@ +Running test "Just function" with AMF0 +Printing ByteArray: +6 +read back: void +undefined +Done with test! +Running test "Just function" with AMF3 +Printing ByteArray: +0 +read back: void +undefined +Done with test! +Running test "Function vector" with AMF3 +Printing ByteArray: +16, 7, 0, 1, 0, 0, 0 +read back: __AS3__.vec::Vector. +Was vector, length 3. Elements: +0th element: null +1th element: null +2th element: null +null,null,null +Done with test! +Running test "Function vector with null element" with AMF3 +Printing ByteArray: +16, 5, 0, 1, 1, 0 +read back: __AS3__.vec::Vector. +Was vector, length 2. Elements: +0th element: null +1th element: null +null,null +Done with test! +Running test "Empty function vector" with AMF3 +Printing ByteArray: +16, 1, 0, 1 +read back: __AS3__.vec::Vector. +Was vector, length 0. Elements: + +Done with test! +Running test "Array with two function elements" with AMF3 +Printing ByteArray: +9, 1, 1 +read back: Array +Was array, length 0. Elements: + +Done with test! +should be undefined: undefined diff --git a/tests/tests/swfs/avm2/amf_function/test.swf b/tests/tests/swfs/avm2/amf_function/test.swf new file mode 100644 index 0000000000000000000000000000000000000000..1c869d1bbf891f5a9321e74447050e518e09b5c2 GIT binary patch literal 1271 zcmV>S5p!02mke1P>Ll69dLc;@gwvBI`k2G*{i;SV^5puMW>IzBhX)nV3TyZOs4`J{mb|7 zzu+50cMyv75K3dTJRLv?Es?=s@KgLpkA~}!xu-5W<%dfsit?UdbFRU8t5hgxHAgq| zhO=8JZ`~*?6pObDddF&V%SPg3)8*2p{@gTpxYf37{&1h0<#t=!XYG#dX=C0|nAFLh zEU&rQqO`?9^GqJAg7v26Hu4>AHC@WvnqkI2^=tv8jzQ`?o+mj~YMS;l-VkkV*QA=K z5M18IR%jS<6hFAvwz%o17-%`J#VyBH+NM_XWrCwWmv!*0jWa8FM_{#M8y;5L29(nd z?(9JMJ3INVX-utmT&~tffXFMP+8rY2UgOn0i#Jre(`>3{)7&*}o>F5ojP8OkPap zk_(v|nVXqgncJDg%pHmeBbW-J#HQdGnhHmv(;gQFE^x|z#fZR&C<~ZT%BaMM%mRuW zBC}v0m6JqH5jhR?c_v@D5W(SBHzQY>@-9=}V@jSW1*Q~P@VXolW(7GZ$SFZi3uoc~ z$1R7%Sy4`ka!QobqFfgLwS$i)OyuZ4^1#R^jHDPzQ-lb7B*u_{B!m+1+dhjtA;LH= zi%1|MMwntmL<9r|jv|7weEFzY5vrtEJy)!zi`7)Ink-fm#cCYBPtgVcYTysSkwBH; z)1zV^Pb1XFGZ>B-d8?1(LbkG)ySurFbL8-479AR5^dds?$+0fh@AM8|>f&)-$>Q&0 zkl9Ht0+MgVp)B$1U{F~+0ZZ@?qgR68k#T?(FTgnhM@G2W$M?m%5Q^T1P^A~W+;Z=@ z2X+;<&b)x&PFvH(lg+!hBEHbYO_Atg7HK72Tzww^6wDq}?&);(9`@{XdaXMnBXm`# z)rh)8a<%1i-(3(S=6Q6y`D~W35o;QLV!*G zJ{SZ70XFn1`CenMTz>qYT!M~zbuQ*U@Y^I%rUZY(UYoKg+{bU@NFTpLqJ4Z>nC{~% z;!GbuqM4&LR0*$#(Lwml9Lcs3aNCA>#)cfk zpp`UtDT`igQej95BTCH*$38We6;6C=J}ba1hU7IMsYiMfJ79*SOIft}1bYwdDb%;2 hORHa^o?oR1tEmV_MIhxEhE4YVuF(S^{s7I2MD-WMbkzU= literal 0 HcmV?d00001 diff --git a/tests/tests/swfs/avm2/amf_function/test.toml b/tests/tests/swfs/avm2/amf_function/test.toml new file mode 100644 index 000000000..dbee897f5 --- /dev/null +++ b/tests/tests/swfs/avm2/amf_function/test.toml @@ -0,0 +1 @@ +num_frames = 1