avm2: Fix Array.sort(function)

core/src/avm2/globals/array.rs

@@ -1014,16 +1014,16 @@ pub fn sort<'gc>(
                     .coerce_to_u32(activation)? as u8,
             ),
         )
+    } else {
+        let arg = args.get(0).cloned().unwrap_or(Value::Undefined);
+        if let Ok(callable) = arg.as_callable(activation, None, None) {
+            (Some(callable), SortOptions::empty())
         } else {
             (
                 None,
-            SortOptions::from_bits_truncate(
-                args.get(0)
-                    .cloned()
-                    .unwrap_or_else(|| 0.into())
-                    .coerce_to_u32(activation)? as u8,
-            ),
+                SortOptions::from_bits_truncate(arg.coerce_to_u32(activation)? as u8),
             )
+        }
     };
 
     let mut values = if let Some(values) = extract_array_values(activation, this.into())? {

tests/tests/swfs/avm2/array_sort/Test.as

@@ -26,7 +26,7 @@ function length_based_comparison(a, b) {
 }
 
 function sub_comparison(a, b) {
-	return a - b;
+	return b - a;
 }
 
 function lbc(a, b) {

tests/tests/swfs/avm2/array_sort/output.txt

@@ -263,17 +263,17 @@ false
 //Array.prototype[10] = "hole in slot 10";
 //var a = new Array(7,2,1,"3","4")
 //(contents of a.sort(sub_comparison))
-1
-2
-3
-4
 7
+4
+3
+2
+1
 //(contents of a.sort(sub_comparison, 2))
-7
-4
-3
-2
 1
+2
+3
+4
+7
 //(contents of a.sort(sub_comparison, Array.RETURNINDEXEDARRAY))
 4
 3
@@ -293,5 +293,5 @@ false
 true
 //var d = new Array(3,"4")
 //(contents of d.sort(sub_comparison, 4))
-3
 4
+3