avm2: Make optimizer lookup callee return type

This provides us with more optimization opportunities.
Since we optimize methods one at a time, we need to look up
the return type in the domain rather than relying on the
resolved return type in the target `BytecodeMethod`

core/src/avm2/method.rs

@@ -8,6 +8,7 @@ use crate::avm2::value::{abc_default_value, Value};
 use crate::avm2::verify::{resolve_param_config, VerifiedMethodInfo};
 use crate::avm2::Error;
 use crate::avm2::Multiname;
+use crate::context::UpdateContext;
 use crate::string::AvmString;
 use crate::tag_utils::SwfMovie;
 use gc_arena::barrier::unlock;
@@ -23,6 +24,8 @@ use swf::avm2::types::{
     MethodFlags as AbcMethodFlags, MethodParam as AbcMethodParam,
 };
 
+use super::Domain;
+
 /// Represents a function defined in Ruffle's code.
 ///
 /// Parameters are as follows:
@@ -472,4 +475,11 @@ impl<'gc> Method<'gc> {
             Method::Bytecode(bm) => bm.method().flags.contains(AbcMethodFlags::NEED_ARGUMENTS),
         }
     }
+
+    pub fn domain(&self, context: &mut UpdateContext<'gc>) -> Domain<'gc> {
+        match self {
+            Method::Native(_) => context.avm2.playerglobals_domain(),
+            Method::Bytecode(bm) => bm.translation_unit().domain(),
+        }
+    }
 }

core/src/avm2/optimize.rs

@@ -305,6 +305,25 @@ fn has_simple_scope_structure(
     true
 }
 
+fn try_resolve_method_return_type<'gc>(
+    vtable: &VTable<'gc>,
+    disp_id: u32,
+    activation: &mut Activation<'_, 'gc>,
+) -> Option<Class<'gc>> {
+    if let Some(target_method) = vtable.get_method(disp_id) {
+        let target_return_type = &target_method.return_type();
+        if !target_return_type.has_lazy_component() && !target_return_type.is_any_name() {
+            if let Some(target_ret_class) = target_method
+                .domain(activation.context)
+                .get_class(activation.context, target_return_type)
+            {
+                return Some(target_ret_class);
+            }
+        }
+    }
+    None
+}
+
 pub fn optimize<'gc>(
     activation: &mut Activation<'_, 'gc>,
     method: &BytecodeMethod<'gc>,
@@ -1097,6 +1116,12 @@ pub fn optimize<'gc>(
                                     index: disp_id,
                                     push_return_value: true,
                                 };
+                                if let Some(ret_type) =
+                                    try_resolve_method_return_type(&vtable, disp_id, activation)
+                                {
+                                    stack_push_done = true;
+                                    stack.push_class(ret_type);
+                                }
                             }
                             _ => {}
                         }
@@ -1318,6 +1343,12 @@ pub fn optimize<'gc>(
                                     index: disp_id,
                                     push_return_value: true,
                                 };
+                                if let Some(ret_type) =
+                                    try_resolve_method_return_type(&vtable, disp_id, activation)
+                                {
+                                    stack_push_done = true;
+                                    stack.push_class(ret_type);
+                                }
                             }
                             Some(Property::Slot { slot_id })
                             | Some(Property::ConstSlot { slot_id }) => {