From a05f81b393cbb5cb15d4c4a50d88df6ba77ecbc4 Mon Sep 17 00:00:00 2001 From: Mike Welsh Date: Tue, 20 Apr 2021 23:56:48 -0700 Subject: [PATCH] video: Add VideoDecoder trait Change VideoStream to use a VideoDecoder trait instead of an enum. This will make it a little easier as more codecs are added, and allow us to easily enable/disable codecs behind features. --- core/src/backend/video.rs | 7 ++ core/src/backend/video/software.rs | 190 ++++++++++++++++++++--------- 2 files changed, 136 insertions(+), 61 deletions(-) diff --git a/core/src/backend/video.rs b/core/src/backend/video.rs index 38724282a..6a3817d68 100644 --- a/core/src/backend/video.rs +++ b/core/src/backend/video.rs @@ -32,6 +32,13 @@ impl<'a> EncodedFrame<'a> { } } +/// A decoded frame of video in RGBA format. +struct DecodedFrame { + width: u16, + height: u16, + rgba: Vec, +} + /// What dependencies a given video frame has on any previous frames. #[derive(Copy, Clone, Debug)] pub enum FrameDependency { diff --git a/core/src/backend/video/software.rs b/core/src/backend/video/software.rs index 4537108b3..9151bf6b7 100644 --- a/core/src/backend/video/software.rs +++ b/core/src/backend/video/software.rs @@ -2,20 +2,11 @@ use crate::backend::render::{BitmapHandle, BitmapInfo, RenderBackend}; use crate::backend::video::{ - EncodedFrame, Error, FrameDependency, VideoBackend, VideoStreamHandle, + DecodedFrame, EncodedFrame, Error, FrameDependency, VideoBackend, VideoStreamHandle, }; use generational_arena::Arena; -use h263_rs::parser::{decode_picture, H263Reader}; -use h263_rs::{DecoderOption, H263State, PictureTypeCode}; -use h263_rs_yuv::bt601::yuv420_to_rgba; use swf::{VideoCodec, VideoDeblocking}; -/// A single preloaded video stream. -pub enum VideoStream { - /// An H.263 video stream. - H263(H263State, Option), -} - /// Software video backend that proxies to CPU-only codec implementations that /// ship with Ruffle. pub struct SoftwareVideoBackend { @@ -44,13 +35,13 @@ impl VideoBackend for SoftwareVideoBackend { codec: VideoCodec, _filter: VideoDeblocking, ) -> Result { - match codec { - VideoCodec::H263 => Ok(self.streams.insert(VideoStream::H263( - H263State::new(DecoderOption::SORENSON_SPARK_BITSTREAM), - None, - ))), - _ => Err(format!("Unsupported video codec type {:?}", codec).into()), - } + let decoder: Box = match codec { + VideoCodec::H263 => Box::new(h263::H263Decoder::new()), + _ => return Err(format!("Unsupported video codec type {:?}", codec).into()), + }; + let stream = VideoStream::new(decoder); + let stream_handle = self.streams.insert(stream); + Ok(stream_handle) } fn preload_video_stream_frame( @@ -63,21 +54,7 @@ impl VideoBackend for SoftwareVideoBackend { .get_mut(stream) .ok_or("Unregistered video stream")?; - match stream { - VideoStream::H263(_state, _last_bitmap) => { - let mut reader = H263Reader::from_source(encoded_frame.data()); - let picture = - decode_picture(&mut reader, DecoderOption::SORENSON_SPARK_BITSTREAM, None)? - .ok_or("Picture in video stream is not a picture")?; - - match picture.picture_type { - PictureTypeCode::IFrame => Ok(FrameDependency::None), - PictureTypeCode::PFrame => Ok(FrameDependency::Past), - PictureTypeCode::DisposablePFrame => Ok(FrameDependency::Past), - _ => Err("Invalid picture type code!".into()), - } - } - } + stream.decoder.preload_frame(encoded_frame) } fn decode_video_stream_frame( @@ -91,38 +68,129 @@ impl VideoBackend for SoftwareVideoBackend { .get_mut(stream) .ok_or("Unregistered video stream")?; - match stream { - VideoStream::H263(state, last_bitmap) => { - let mut reader = H263Reader::from_source(encoded_frame.data()); + let frame = stream.decoder.decode_frame(encoded_frame)?; + let handle = if let Some(bitmap) = stream.bitmap { + renderer.update_texture(bitmap, frame.width.into(), frame.height.into(), frame.rgba)? + } else { + renderer.register_bitmap_raw(frame.width.into(), frame.height.into(), frame.rgba)? + }; + stream.bitmap = Some(handle); - state.decode_next_picture(&mut reader)?; + Ok(BitmapInfo { + handle, + width: frame.width, + height: frame.height, + }) + } +} - let picture = state - .get_last_picture() - .expect("Decoding a picture should let us grab that picture"); +/// A single preloaded video stream. +struct VideoStream { + bitmap: Option, + decoder: Box, +} - let (width, height) = picture - .format() - .into_width_and_height() - .ok_or("H.263 decoder error!")?; - let chroma_width = picture.chroma_samples_per_row(); - let (y, b, r) = picture.as_yuv(); - let rgba = yuv420_to_rgba(y, b, r, width.into(), chroma_width); - - let handle = if let Some(lb) = last_bitmap { - renderer.update_texture(*lb, width.into(), height.into(), rgba)? - } else { - renderer.register_bitmap_raw(width.into(), height.into(), rgba)? - }; - - *last_bitmap = Some(handle); - - Ok(BitmapInfo { - handle, - width, - height, - }) - } +impl VideoStream { + fn new(decoder: Box) -> Self { + Self { + decoder, + bitmap: None, + } + } +} + +/// Trait for video decoders. +/// This should be implemented for each video codec. +trait VideoDecoder { + /// Preload a frame. + /// + /// No decoding is intended to happen at this point in time. Instead, the + /// video data should be inspected to determine inter-frame dependencies + /// between this and any previous frames in the stream. + /// + /// Frames should be preloaded in the order that they are recieved. + /// + /// Any dependencies listed here are inherent to the video bitstream. The + /// containing video stream is also permitted to introduce additional + /// interframe dependencies. + fn preload_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result; + + /// Decode a frame of a given video stream. + /// + /// This function is provided the external index of the frame, the codec + /// used to decode the data, and what codec to decode it with. The codec + /// provided here must match the one used to register the video stream. + /// + /// Frames may be decoded in any order that does not violate the frame + /// dependencies declared by the output of `preload_video_stream_frame`. + /// + /// The decoded frame should be returned. An `Error` can be returned if + /// a drawable bitmap can not be produced. + fn decode_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result; +} + +mod h263 { + use crate::backend::video::software::VideoDecoder; + use crate::backend::video::{DecodedFrame, EncodedFrame, Error, FrameDependency}; + use h263_rs::parser::{decode_picture, H263Reader}; + use h263_rs::{DecoderOption, H263State, PictureTypeCode}; + use h263_rs_yuv::bt601::yuv420_to_rgba; + + /// H263 video decoder. + pub struct H263Decoder(H263State); + + impl H263Decoder { + pub fn new() -> Self { + Self(H263State::new(DecoderOption::SORENSON_SPARK_BITSTREAM)) + } + } + + impl VideoDecoder for H263Decoder { + fn preload_frame( + &mut self, + encoded_frame: EncodedFrame<'_>, + ) -> Result { + let mut reader = H263Reader::from_source(encoded_frame.data()); + let picture = + decode_picture(&mut reader, DecoderOption::SORENSON_SPARK_BITSTREAM, None)? + .ok_or("Picture in video stream is not a picture")?; + + match picture.picture_type { + PictureTypeCode::IFrame => Ok(FrameDependency::None), + PictureTypeCode::PFrame => Ok(FrameDependency::Past), + PictureTypeCode::DisposablePFrame => Ok(FrameDependency::Past), + _ => Err("Invalid picture type code!".into()), + } + } + + fn decode_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result { + let mut reader = H263Reader::from_source(encoded_frame.data()); + + self.0.decode_next_picture(&mut reader)?; + + let picture = self + .0 + .get_last_picture() + .expect("Decoding a picture should let us grab that picture"); + + let (width, height) = picture + .format() + .into_width_and_height() + .ok_or("H.263 decoder error!")?; + let chroma_width = picture.chroma_samples_per_row(); + let (y, b, r) = picture.as_yuv(); + let rgba = yuv420_to_rgba(y, b, r, width.into(), chroma_width); + Ok(DecodedFrame { + width, + height, + rgba, + }) + } + } + + impl Default for H263Decoder { + fn default() -> Self { + Self::new() } } }