video/external: Add OpenH264 decoder

Cargo.lock

@@ -654,6 +654,27 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 
+[[package]]
+name = "bzip2"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
+dependencies = [
+ "bzip2-sys",
+ "libc",
+]
+
+[[package]]
+name = "bzip2-sys"
+version = "0.1.11+1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "calloop"
 version = "0.12.4"
@@ -3161,6 +3182,16 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
+[[package]]
+name = "md-5"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
+dependencies = [
+ "cfg-if",
+ "digest",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.2"
@@ -4142,6 +4173,7 @@ dependencies = [
  "cookie",
  "cookie_store",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2",
@@ -4541,11 +4573,17 @@ dependencies = [
 name = "ruffle_video_external"
 version = "0.1.0"
 dependencies = [
+ "bzip2",
+ "hex",
+ "libloading 0.8.3",
+ "md-5",
+ "reqwest",
  "ruffle_render",
  "ruffle_video",
  "ruffle_video_software",
  "slotmap",
  "swf",
+ "tracing",
 ]
 
 [[package]]

video/external/Cargo.toml

@@ -12,4 +12,18 @@ ruffle_render = { path = "../../render" }
 ruffle_video = { path = ".." }
 swf = { path = "../../swf" }
 slotmap = { workspace = true }
+tracing = { workspace = true }
 ruffle_video_software = { path = "../software" }
+
+# Needed for OpenH264:
+libloading = "0.8.3"
+md-5 = "0.10.6"
+reqwest = { version = "0.12.3", default-features = false, features = ["blocking"] }
+hex = "0.4.3"
+bzip2 = { version = "0.4.4", features = ["static"] }
+
+[package.metadata.cargo-machete]
+ignored = [
+    # Renames itself to "md5", see: https://github.com/bnjbvr/cargo-machete/issues/8
+    "md-5"
+]

video/external/src/BINARY_LICENSE.txt

@@ -0,0 +1,59 @@
+-------------------------------------------------------
+About The Cisco-Provided Binary of OpenH264 Video Codec
+-------------------------------------------------------
+
+Cisco provides this program under the terms of the BSD license.  
+
+Additionally, this binary is licensed under Cisco’s AVC/H.264 Patent Portfolio License from MPEG LA, at no cost to you, provided that the requirements and conditions shown below in the AVC/H.264 Patent Portfolio sections are met.  
+
+As with all AVC/H.264 codecs, you may also obtain your own patent license from MPEG LA or from the individual patent owners, or proceed at your own risk.  Your rights from Cisco under the BSD license are not affected by this choice.  
+
+For more information on the OpenH264 binary licensing, please see the OpenH264 FAQ found at http://www.openh264.org/faq.html#binary 
+
+A corresponding source code to this binary program is available under the same BSD terms, which can be found at http://www.openh264.org
+
+-----------
+BSD License
+-----------
+
+Copyright © 2014 Cisco Systems, Inc.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-----------------------------------------
+AVC/H.264 Patent Portfolio License Notice
+-----------------------------------------
+
+The binary form of this Software is distributed by Cisco under the AVC/H.264 Patent Portfolio License from MPEG LA, and is subject to the following requirements, which may or may not be applicable to your use of this software: 
+
+THIS PRODUCT IS LICENSED UNDER THE AVC PATENT PORTFOLIO LICENSE FOR THE PERSONAL USE OF A CONSUMER OR OTHER USES IN WHICH IT DOES NOT RECEIVE REMUNERATION TO (i) ENCODE VIDEO IN COMPLIANCE WITH THE AVC STANDARD (“AVC VIDEO”) AND/OR (ii) DECODE AVC VIDEO THAT WAS ENCODED BY A CONSUMER ENGAGED IN A PERSONAL ACTIVITY AND/OR WAS OBTAINED FROM A VIDEO PROVIDER LICENSED TO PROVIDE AVC VIDEO.  NO LICENSE IS GRANTED OR SHALL BE IMPLIED FOR ANY OTHER USE.  ADDITIONAL INFORMATION MAY BE OBTAINED FROM MPEG LA, L.L.C. SEE HTTP://WWW.MPEGLA.COM
+
+Accordingly, please be advised that content providers and broadcasters using AVC/H.264 in their service may be required to obtain a separate use license from MPEG LA, referred to as "(b) sublicenses" in the SUMMARY OF AVC/H.264 LICENSE TERMS from MPEG LA found at http://www.openh264.org/mpegla
+
+---------------------------------------------
+AVC/H.264 Patent Portfolio License Conditions
+---------------------------------------------
+
+In addition, the Cisco-provided binary of this Software is licensed under Cisco's license from MPEG LA only if the following conditions are met:
+
+1. The Cisco-provided binary is separately downloaded to an end user’s device, and not integrated into or combined with third party software prior to being downloaded to the end user’s device;
+
+2. The end user must have the ability to control (e.g., to enable, disable, or re-enable) the use of the Cisco-provided binary;
+
+3. Third party software, in the location where end users can control the use of the Cisco-provided binary, must display the following text:
+
+       "OpenH264 Video Codec provided by Cisco Systems, Inc."
+
+4.  Any third-party software that makes use of the Cisco-provided binary must reproduce all of the above text, as well as this last condition, in the EULA and/or in another location where licensing information is to be presented to the end user.  
+ 
+
+
+                          v1.0

video/external/src/backend.rs

@@ -1,4 +1,6 @@
 use crate::decoder::VideoDecoder;
+use bzip2::read::BzDecoder;
+use md5::{Digest, Md5};
 use ruffle_render::backend::RenderBackend;
 use ruffle_render::bitmap::{BitmapHandle, BitmapInfo, PixelRegion};
 use ruffle_video::backend::VideoBackend;
@@ -7,6 +9,9 @@ use ruffle_video::frame::{EncodedFrame, FrameDependency};
 use ruffle_video::VideoStreamHandle;
 use ruffle_video_software::backend::SoftwareVideoBackend;
 use slotmap::SlotMap;
+use std::fs::File;
+use std::io::{copy, Write};
+use std::path::PathBuf;
 use swf::{VideoCodec, VideoDeblocking};
 
 enum ProxyOrStream {
@@ -23,19 +28,99 @@ enum ProxyOrStream {
 /// except for H.264, for which it uses an external decoder.
 pub struct ExternalVideoBackend {
     streams: SlotMap<VideoStreamHandle, ProxyOrStream>,
+    openh264_lib_filepath: Option<PathBuf>,
     software: SoftwareVideoBackend,
 }
 
 impl Default for ExternalVideoBackend {
     fn default() -> Self {
-        Self::new()
+        Self::new(None)
     }
 }
 
+/// Source: https://www.openh264.org/BINARY_LICENSE.txt
+const BINARY_LICENSE: &[u8] = include_bytes!("BINARY_LICENSE.txt");
+
 impl ExternalVideoBackend {
-    pub fn new() -> Self {
+    fn get_openh264_data() -> Result<(&'static str, &'static str), Box<dyn std::error::Error>> {
+        // Source: https://github.com/cisco/openh264/releases/tag/v2.4.1
+        match (std::env::consts::OS, std::env::consts::ARCH) {
+            ("linux", "x86") => Ok((
+                "libopenh264-2.4.1-linux32.7.so",
+                "dd0743066117d63e1b2abc56a86506e5",
+            )),
+            ("linux", "x86_64") => Ok((
+                "libopenh264-2.4.1-linux64.7.so",
+                "19c561386a9564f8510fcb7586b9d402",
+            )),
+            ("linux", "arm") => Ok((
+                "libopenh264-2.4.1-linux-arm.7.so",
+                "2274a1bbd13f32b7afe22092e44fa2b5",
+            )),
+            ("linux", "aarch64") => Ok((
+                "libopenh264-2.4.1-linux-arm64.7.so",
+                "2aa205f08077aa2d049032e0b56c5b84",
+            )),
+            ("macos", "x86_64") => Ok((
+                "libopenh264-2.4.1-mac-x64.dylib",
+                "9fefa1e0279a49b8a4e9cf6fc148bc0c",
+            )),
+            ("macos", "aarch64") => Ok((
+                "libopenh264-2.4.1-mac-arm64.dylib",
+                "41f59bb5696ffeadbfba3a8a95ec39b7",
+            )),
+            ("windows", "x86") => Ok((
+                "openh264-2.4.1-win32.dll",
+                "a9360e6dd1e24320c3d65a0c65bf14a4",
+            )),
+            ("windows", "x86_64") => Ok((
+                "openh264-2.4.1-win64.dll",
+                "c85406e6b73812ec3fb9da5f898c6a9e",
+            )),
+            (os, arch) => Err(format!("Unsupported OS/ARCH: {} {}", os, arch).into()),
+        }
+    }
+
+    pub fn get_openh264() -> Result<PathBuf, Box<dyn std::error::Error>> {
+        const URL_BASE: &str = "http://ciscobinary.openh264.org/";
+        const URL_SUFFIX: &str = ".bz2";
+
+        let (filename, md5sum) = Self::get_openh264_data()?;
+
+        let filepath = std::env::current_exe()?
+            .parent()
+            .ok_or("Could not determine Ruffle location.")?
+            .join(filename);
+
+        // If the binary doesn't exist in the expected location, download it.
+        if !filepath.is_file() {
+            File::create("OpenH264-license.txt")?.write_all(BINARY_LICENSE)?;
+
+            let url = format!("{}{}{}", URL_BASE, filename, URL_SUFFIX);
+            let response = reqwest::blocking::get(url)?;
+            let bytes = response.bytes()?;
+            let mut bzip2_reader = BzDecoder::new(bytes.as_ref());
+
+            let mut file = File::create(filepath.clone())?;
+            copy(&mut bzip2_reader, &mut file)?;
+        }
+
+        // Regardless of whether the library was already there, or we just downloaded it, let's check the MD5 hash.
+        let mut md5 = Md5::new();
+        copy(&mut File::open(filepath.clone())?, &mut md5)?;
+        let result: [u8; 16] = md5.finalize().into();
+
+        if result[..] != hex::decode(md5sum)?[..] {
+            return Err(format!("MD5 checksum mismatch for {}", filename).into());
+        }
+
+        Ok(filepath)
+    }
+
+    pub fn new(openh264_lib_filepath: Option<PathBuf>) -> Self {
         Self {
             streams: SlotMap::with_key(),
+            openh264_lib_filepath,
             software: SoftwareVideoBackend::new(),
         }
     }
@@ -52,7 +137,15 @@ impl VideoBackend for ExternalVideoBackend {
         filter: VideoDeblocking,
     ) -> Result<VideoStreamHandle, Error> {
         let proxy_or_stream = if codec == VideoCodec::H264 {
-            todo!();
+            let openh264 = &self.openh264_lib_filepath;
+            if let Some(openh264) = openh264 {
+                tracing::info!("Using OpenH264 at {:?}", openh264);
+                let decoder = Box::new(crate::decoder::openh264::H264Decoder::new(openh264));
+                let stream = VideoStream::new(decoder);
+                ProxyOrStream::Owned(stream)
+            } else {
+                return Err(Error::DecoderError("No OpenH264".into()));
+            }
         } else {
             ProxyOrStream::Proxied(
                 self.software

video/external/src/decoder.rs

@@ -1 +1,11 @@
+// bindgen ../openh264/codec/api/wels/codec_api.h --no-prepend-enum-name \
+//         --dynamic-loading OpenH264 -o openh264_sys.rs
+#[allow(non_upper_case_globals)]
+#[allow(non_camel_case_types)]
+#[allow(non_snake_case)]
+#[allow(dead_code)]
+mod openh264_sys;
+
+pub mod openh264;
+
 pub use ruffle_video_software::decoder::VideoDecoder;

video/external/src/decoder/README.md

@@ -0,0 +1,10 @@
+To update to a new OpenH264 release:
+
+ - Open the new release on https://github.com/cisco/openh264/releases
+ - Update the binary file names and MD5 hashes
+ - Add/remove supported architectures and platforms if added/dropped
+ - Update the base URL and license if changed
+ - Download the OpenH264 sources (at least `codec_api.h`)
+ - Regenerate the bindings using the command in `decoder.rs`
+ - Follow the API changes if necessary
+ - Update the version number sanity check

video/external/src/decoder/openh264.rs

@@ -0,0 +1,233 @@
+use core::slice;
+use std::ffi::{c_int, c_uchar};
+use std::ptr;
+
+use crate::decoder::openh264_sys::{self, videoFormatI420, ISVCDecoder, OpenH264};
+use crate::decoder::VideoDecoder;
+
+use ruffle_render::bitmap::BitmapFormat;
+use ruffle_video::error::Error;
+use ruffle_video::frame::{DecodedFrame, EncodedFrame, FrameDependency};
+
+/// H264 video decoder.
+pub struct H264Decoder {
+    /// How many bytes are used to store the length of the NALU (1, 2, 3, or 4).
+    length_size: u8,
+
+    openh264: OpenH264,
+    decoder: *mut ISVCDecoder,
+}
+
+impl H264Decoder {
+    /// `extradata` should hold "AVCC (MP4) format" decoder configuration, including PPS and SPS.
+    /// Make sure it has any start code emulation prevention "three bytes" removed.
+    pub fn new(openh264_lib_filename: &std::path::Path) -> Self {
+        let mut decoder: *mut ISVCDecoder = ptr::null_mut();
+        unsafe {
+            let openh264 = OpenH264::new(openh264_lib_filename).unwrap();
+
+            let version = openh264.WelsGetCodecVersion();
+
+            assert_eq!(
+                (version.uMajor, version.uMinor, version.uRevision),
+                (2, 4, 1),
+                "Unexpected OpenH264 version"
+            );
+
+            openh264.WelsCreateDecoder(&mut decoder);
+
+            let decoder_vtbl = (*decoder).as_ref().unwrap();
+
+            let mut dec_param: openh264_sys::SDecodingParam = std::mem::zeroed();
+            dec_param.sVideoProperty.eVideoBsType = openh264_sys::VIDEO_BITSTREAM_AVC;
+
+            (decoder_vtbl.Initialize.unwrap())(decoder, &dec_param);
+
+            Self {
+                length_size: 0,
+                openh264,
+                decoder,
+            }
+        }
+    }
+}
+
+impl Drop for H264Decoder {
+    fn drop(&mut self) {
+        unsafe {
+            let decoder_vtbl = (*self.decoder).as_ref().unwrap();
+
+            (decoder_vtbl.Uninitialize.unwrap())(self.decoder);
+            self.openh264.WelsDestroyDecoder(self.decoder);
+        }
+    }
+}
+
+impl VideoDecoder for H264Decoder {
+    fn configure_decoder(&mut self, configuration_data: &[u8]) -> Result<(), Error> {
+        unsafe {
+            // TODO: Check whether the "start code emulation prevention" needs to be
+            // undone here before looking into the data. (i.e. conversion from SODB
+            // into RBSP, by replacing each 0x00000301 byte sequence with 0x000001)
+
+            assert_eq!(configuration_data[0], 1, "Invalid configuration version");
+            // extradata[0]: configuration version, always 1
+            // extradata[1]: profile
+            // extradata[2]: compatibility
+            // extradata[3]: level
+            // extradata[4]: 6 reserved bits | NALU length size - 1
+
+            self.length_size = (configuration_data[4] & 0b0000_0011) + 1;
+
+            let decoder_vtbl = (*self.decoder).as_ref().unwrap();
+
+            //input: encoded bitstream start position; should include start code prefix
+            let mut buffer: Vec<c_uchar> = Vec::new();
+
+            // Converting from AVCC to Annex B (stream-like) format,
+            // putting the PPS and SPS into a NALU.
+
+            buffer.extend_from_slice(&[0, 0, 0, 1]);
+
+            let sps_length = configuration_data[6] as usize * 256 + configuration_data[7] as usize;
+
+            for i in 0..sps_length {
+                buffer.push(configuration_data[8 + i]);
+            }
+
+            let num_pps = configuration_data[8 + sps_length] as usize;
+
+            assert_eq!(num_pps, 1, "More than one PPS is not supported");
+
+            buffer.extend_from_slice(&[0, 0, 0, 1]);
+
+            let pps_length = configuration_data[8 + sps_length + 1] as usize * 256
+                + configuration_data[8 + sps_length + 2] as usize;
+
+            for i in 0..pps_length {
+                buffer.push(configuration_data[8 + sps_length + 3 + i]);
+            }
+
+            //output: [0~2] for Y,U,V buffer for Decoding only
+            let mut output = [ptr::null_mut() as *mut c_uchar; 3];
+            //in-out: for Decoding only: declare and initialize the output buffer info
+            let mut dest_buf_info: openh264_sys::SBufferInfo = std::mem::zeroed();
+
+            let _ret = decoder_vtbl.DecodeFrameNoDelay.unwrap()(
+                self.decoder,
+                buffer.as_mut_ptr(),
+                buffer.len() as c_int,
+                output.as_mut_ptr(),
+                &mut dest_buf_info as *mut openh264_sys::SBufferInfo,
+            );
+        }
+        Ok(())
+    }
+
+    fn preload_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result<FrameDependency, Error> {
+        assert!(self.length_size > 0, "Decoder not configured");
+
+        let nal_unit_type = encoded_frame.data[self.length_size as usize] & 0b0001_1111;
+
+        // 3.62 instantaneous decoding refresh (IDR) picture:
+        // After the decoding of an IDR picture all following coded pictures in decoding order can
+        // be decoded without inter prediction from any picture decoded prior to the IDR picture.
+        if nal_unit_type == openh264_sys::NAL_SLICE_IDR as u8 {
+            Ok(FrameDependency::None)
+        } else {
+            Ok(FrameDependency::Past)
+        }
+    }
+
+    fn decode_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result<DecodedFrame, Error> {
+        assert!(self.length_size > 0, "Decoder not configured");
+        unsafe {
+            let decoder_vtbl = (*self.decoder).as_ref().unwrap();
+
+            // input: encoded bitstream start position; should include start code prefix
+            // converting from AVCC (file-like) to Annex B (stream-like) format
+            // Thankfully the start code emulation prevention is there in both.
+            let mut buffer: Vec<c_uchar> = Vec::with_capacity(encoded_frame.data.len());
+
+            let mut i = 0;
+            while i < encoded_frame.data.len() {
+                let mut length = 0;
+                for j in 0..self.length_size {
+                    length = (length << 8) | encoded_frame.data[i + j as usize] as usize;
+                }
+                i += self.length_size as usize;
+                buffer.extend_from_slice(&[0, 0, 1]);
+                buffer.extend_from_slice(&encoded_frame.data[i..i + length]);
+                i += length;
+            }
+
+            // output: [0~2] for Y,U,V buffer
+            let mut output = [ptr::null_mut() as *mut c_uchar; 3];
+            let mut dest_buf_info: openh264_sys::SBufferInfo = std::mem::zeroed();
+
+            let ret = decoder_vtbl.DecodeFrameNoDelay.unwrap()(
+                self.decoder,
+                buffer.as_mut_ptr(),
+                buffer.len() as c_int,
+                output.as_mut_ptr(),
+                &mut dest_buf_info as *mut openh264_sys::SBufferInfo,
+            );
+
+            if ret != 0 {
+                return Err(Error::DecoderError(
+                    format!("Decoding failed with status code: {}", ret).into(),
+                ));
+            }
+            if dest_buf_info.iBufferStatus != 1 {
+                return Err(Error::DecoderError(
+                    "No output frame produced by the decoder".into(),
+                ));
+            }
+            let buffer_info = dest_buf_info.UsrData.sSystemBuffer;
+            if buffer_info.iFormat != videoFormatI420 as c_int {
+                return Err(Error::DecoderError(
+                    format!("Unexpected output format: {}", buffer_info.iFormat).into(),
+                ));
+            }
+
+            let mut yuv: Vec<u8> = Vec::with_capacity(
+                buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2,
+            );
+
+            // Copying Y
+            for i in 0..buffer_info.iHeight {
+                yuv.extend_from_slice(slice::from_raw_parts(
+                    output[0].offset((i * buffer_info.iStride[0]) as isize),
+                    buffer_info.iWidth as usize,
+                ));
+            }
+
+            // Copying U
+            for i in 0..buffer_info.iHeight / 2 {
+                yuv.extend_from_slice(slice::from_raw_parts(
+                    output[1].offset((i * buffer_info.iStride[1]) as isize),
+                    buffer_info.iWidth as usize / 2,
+                ));
+            }
+
+            // Copying V
+            for i in 0..buffer_info.iHeight / 2 {
+                yuv.extend_from_slice(slice::from_raw_parts(
+                    output[2].offset((i * buffer_info.iStride[1]) as isize),
+                    buffer_info.iWidth as usize / 2,
+                ));
+            }
+
+            // TODO: Check whether frames are being squished/stretched, or cropped,
+            // when encoded image size doesn't match declared video tag size.
+            // NOTE: This will always use the BT.601 coefficients, which may or may
+            // not be correct. So far I haven't seen anything to the contrary in FP.
+            Ok(DecodedFrame::new(
+                buffer_info.iWidth as u32,
+                buffer_info.iHeight as u32,
+                BitmapFormat::Yuv420p,
+                yuv,
+            ))
+        }
+    }
+}