core: Use HTTP response encoding if existing

If System#useCodepage has been set to true, the form loader now uses the
encoding specified in the HTTP response content type field, if existing,
to decode remote text files. chardetng is now (only) used if the HTTP
response doesn't specify any encoding or if the file is local.

core/src/backend/navigator.rs

@@ -4,6 +4,7 @@ use crate::loader::Error;
 use crate::socket::{ConnectionState, SocketAction, SocketHandle};
 use crate::string::WStr;
 use async_channel::{Receiver, Sender};
+use encoding_rs::Encoding;
 use indexmap::IndexMap;
 use std::borrow::Cow;
 use std::fmt;
@@ -197,6 +198,9 @@ pub trait SuccessResponse {
     /// This method consumes the response.
     fn body(self: Box<Self>) -> OwnedFuture<Vec<u8>, Error>;
 
+    /// The text encoding listed in the HTTP response header if existing.
+    fn text_encoding(&self) -> Option<&'static Encoding>;
+
     /// The status code of the response.
     fn status(&self) -> u16;
 
@@ -589,6 +593,10 @@ pub fn fetch_path<NavigatorType: NavigatorBackend>(
             })
         }
 
+        fn text_encoding(&self) -> Option<&'static Encoding> {
+            None
+        }
+
         fn status(&self) -> u16 {
             self.status
         }
@@ -682,3 +690,13 @@ pub fn fetch_path<NavigatorType: NavigatorBackend>(
         Ok(response)
     })
 }
+
+/// Parses and returns the encoding out of an HTTP header content type string
+/// if existing.
+pub fn get_encoding(content_type: &str) -> Option<&'static Encoding> {
+    if let Some((_, encoding_string)) = content_type.split_once("charset=") {
+        Encoding::for_label(encoding_string.as_bytes())
+    } else {
+        None
+    }
+}

core/src/loader.rs

@@ -1217,6 +1217,7 @@ impl<'gc> Loader<'gc> {
             let fetch = player.lock().unwrap().navigator().fetch(request);
 
             let response = fetch.await.map_err(|e| e.error)?;
+            let response_encoding = response.text_encoding();
             let body = response.body().await?;
 
             // Fire the load handler.
@@ -1235,9 +1236,16 @@ impl<'gc> Loader<'gc> {
 
                 let utf8_string;
                 let utf8_body = if activation.context.system.use_codepage {
-                    let mut encoding_detector = EncodingDetector::new();
-                    encoding_detector.feed(&body, true);
-                    let encoding = encoding_detector.guess(None, true);
+                    // Determine the encoding
+                    let encoding = if let Some(encoding) = response_encoding {
+                        encoding
+                    } else {
+                        let mut encoding_detector = EncodingDetector::new();
+                        encoding_detector.feed(&body, true);
+                        encoding_detector.guess(None, true)
+                    };
+
+                    // Convert the text into UTF-8
                     utf8_string = encoding.decode(&body).0;
                     utf8_string.as_bytes()
                 } else {

frontend-utils/src/backends/navigator.rs

@@ -8,8 +8,8 @@ use async_io::Timer;
 use futures_lite::FutureExt;
 use reqwest::{cookie, header, Proxy};
 use ruffle_core::backend::navigator::{
-    async_return, create_fetch_error, ErrorResponse, NavigationMethod, NavigatorBackend,
-    OpenURLMode, OwnedFuture, Request, SocketMode, SuccessResponse,
+    async_return, create_fetch_error, get_encoding, ErrorResponse, NavigationMethod,
+    NavigatorBackend, OpenURLMode, OwnedFuture, Request, SocketMode, SuccessResponse,
 };
 use ruffle_core::indexmap::IndexMap;
 use ruffle_core::loader::Error;
@@ -232,6 +232,7 @@ impl<F: FutureSpawner, I: NavigatorInterface> NavigatorBackend for ExternalNavig
                     let response: Box<dyn SuccessResponse> = Box::new(Response {
                         url: response_url.to_string(),
                         response_body: ResponseBody::File(contents),
+                        text_encoding: None,
                         status: 0,
                         redirected: false,
                     });
@@ -270,7 +271,11 @@ impl<F: FutureSpawner, I: NavigatorInterface> NavigatorBackend for ExternalNavig
                 })?;
 
                 let url = response.url().to_string();
-
+                let text_encoding = response
+                    .headers()
+                    .get("Content-Type")
+                    .and_then(|content_type| content_type.to_str().ok())
+                    .and_then(get_encoding);
                 let status = response.status().as_u16();
                 let redirected = *response.url() != processed_url;
                 if !response.status().is_success() {
@@ -286,6 +291,7 @@ impl<F: FutureSpawner, I: NavigatorInterface> NavigatorBackend for ExternalNavig
                 let response: Box<dyn SuccessResponse> = Box::new(Response {
                     url,
                     response_body: ResponseBody::Network(Arc::new(Mutex::new(Some(response)))),
+                    text_encoding,
                     status,
                     redirected,
                 });

frontend-utils/src/backends/navigator/fetch.rs

@@ -1,6 +1,7 @@
 use reqwest::Response as ReqwestResponse;
 use ruffle_core::backend::navigator::{OwnedFuture, SuccessResponse};
 use ruffle_core::loader::Error;
+use ruffle_core::swf::Encoding;
 use std::sync::{Arc, Mutex};
 
 pub enum ResponseBody {
@@ -18,6 +19,7 @@ pub enum ResponseBody {
 pub struct Response {
     pub url: String,
     pub response_body: ResponseBody,
+    pub text_encoding: Option<&'static Encoding>,
     pub status: u16,
     pub redirected: bool,
 }
@@ -47,6 +49,10 @@ impl SuccessResponse for Response {
         }
     }
 
+    fn text_encoding(&self) -> Option<&'static Encoding> {
+        self.text_encoding
+    }
+
     fn status(&self) -> u16 {
         self.status
     }

tests/framework/src/backends/navigator.rs

@@ -10,6 +10,7 @@ use ruffle_core::backend::navigator::{
 use ruffle_core::indexmap::IndexMap;
 use ruffle_core::loader::Error;
 use ruffle_core::socket::{ConnectionState, SocketAction, SocketHandle};
+use ruffle_core::swf::Encoding;
 use ruffle_socket_format::SocketEvent;
 use std::borrow::Cow;
 use std::time::Duration;
@@ -54,6 +55,10 @@ impl SuccessResponse for TestResponse {
     fn expected_length(&self) -> Result<Option<u64>, Error> {
         Ok(Some(self.body.len() as u64))
     }
+
+    fn text_encoding(&self) -> Option<&'static Encoding> {
+        None
+    }
 }
 
 /// A `NavigatorBackend` used by tests that supports logging fetch requests.

web/src/navigator.rs

@@ -6,13 +6,14 @@ use futures_util::{future, SinkExt, StreamExt};
 use gloo_net::websocket::{futures::WebSocket, Message};
 use js_sys::{Array, Uint8Array};
 use ruffle_core::backend::navigator::{
-    async_return, create_fetch_error, create_specific_fetch_error, ErrorResponse, NavigationMethod,
-    NavigatorBackend, OpenURLMode, OwnedFuture, Request, SuccessResponse,
+    async_return, create_fetch_error, create_specific_fetch_error, get_encoding, ErrorResponse,
+    NavigationMethod, NavigatorBackend, OpenURLMode, OwnedFuture, Request, SuccessResponse,
 };
 use ruffle_core::config::NetworkingAccessMode;
 use ruffle_core::indexmap::IndexMap;
 use ruffle_core::loader::Error;
 use ruffle_core::socket::{ConnectionState, SocketAction, SocketHandle};
+use ruffle_core::swf::Encoding;
 use std::borrow::Cow;
 use std::cell::RefCell;
 use std::rc::Rc;
@@ -481,6 +482,14 @@ impl SuccessResponse for WebResponseWrapper {
         })
     }
 
+    fn text_encoding(&self) -> Option<&'static Encoding> {
+        if let Ok(Some(content_type)) = self.response.headers().get("Content-Type") {
+            get_encoding(&content_type)
+        } else {
+            None
+        }
+    }
+
     fn status(&self) -> u16 {
         self.response.status()
     }