diff --git a/Cargo.lock b/Cargo.lock index 5aec38900..9f523f022 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2283,7 +2283,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" dependencies = [ - "libloading 0.8.4", + "libloading 0.7.4", ] [[package]] @@ -3905,8 +3905,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" -version = "0.7.1" -source = "git+https://github.com/rustdesk-org/hwcodec#398e5a8938dd8768ade0fcdc27ea80e8b4b38738" +version = "0.8.0" +source = "git+https://github.com/21pages/hwcodec?branch=rc#ca80368a813caddcb88f077de7c02df9e7316001" dependencies = [ "bindgen 0.59.2", "cc", diff --git a/flutter/lib/common/widgets/overlay.dart b/flutter/lib/common/widgets/overlay.dart index 3fb63616d..e614215df 100644 --- a/flutter/lib/common/widgets/overlay.dart +++ b/flutter/lib/common/widgets/overlay.dart @@ -601,8 +601,9 @@ class QualityMonitor extends StatelessWidget { "Delay", "${qualityMonitorModel.data.delay == null ? '-' : (qualityMonitorModel.data.fps ?? "").replaceAll(' ', '').replaceAll('0', '').isEmpty ? 0 : qualityMonitorModel.data.delay}ms", rightColor: Colors.green), - _row("Target Bitrate", - "${qualityMonitorModel.data.targetBitrate ?? '-'}kb"), + if ((int.tryParse(qualityMonitorModel.data.targetBitrate ?? '') ?? 0) >= 1) + _row("Target Bitrate", + "${qualityMonitorModel.data.targetBitrate}kb"), _row( "Codec", qualityMonitorModel.data.codecFormat ?? '-'), _row("Chroma", qualityMonitorModel.data.chroma ?? '-'), diff --git a/libs/scrap/Cargo.toml b/libs/scrap/Cargo.toml index 505eca2de..86ceb6062 100644 --- a/libs/scrap/Cargo.toml +++ b/libs/scrap/Cargo.toml @@ -60,7 +60,8 @@ gstreamer-video = { version = "0.16", optional = true } zbus = { version = "3.15", optional = true } [dependencies.hwcodec] -git = "https://github.com/rustdesk-org/hwcodec" +git = "https://github.com/21pages/hwcodec" +branch = "rc" optional = true [target.'cfg(any(target_os = "windows", target_os = "linux"))'.dependencies] diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index a867a2d3f..69de3d841 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -93,6 +93,7 @@ fn test_vpx( width: width as _, height: height as _, quality, + image_quality: hbb_common::message_proto::ImageQuality::Balanced, codec: codec_id, keyframe_interval: None, }); @@ -263,6 +264,7 @@ mod hw { width, height, quality, + image_quality: hbb_common::message_proto::ImageQuality::Balanced, keyframe_interval: None, }), false, diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index ca620608a..0a68d749a 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -105,6 +105,7 @@ fn main() -> io::Result<()> { quality, codec: vpx_codec, keyframe_interval: None, + image_quality: hbb_common::message_proto::ImageQuality::Balanced, }), false, ) diff --git a/libs/scrap/src/common/aom.rs b/libs/scrap/src/common/aom.rs index e5093e54b..7cb5f011b 100644 --- a/libs/scrap/src/common/aom.rs +++ b/libs/scrap/src/common/aom.rs @@ -14,6 +14,7 @@ use hbb_common::{ anyhow::{anyhow, Context}, bytes::Bytes, log, + message_proto::ImageQuality, message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame}, ResultType, }; @@ -266,9 +267,19 @@ impl EncoderApi for AomEncoder { Ok(()) } - fn bitrate(&self) -> u32 { + fn rc_state(&self) -> crate::codec::RcState { let c = unsafe { *self.ctx.config.enc.to_owned() }; - c.rc_target_bitrate + crate::codec::RcState { + bitrate: c.rc_target_bitrate, + qp: ((c.rc_min_quantizer + c.rc_max_quantizer) / 2) as i32, + qp_min: c.rc_min_quantizer as i32, + qp_max: c.rc_max_quantizer as i32, + qp_mode: false, + } + } + + fn rc_changed(&self, _image_quality: ImageQuality) -> bool { + false } fn support_changing_quality(&self) -> bool { @@ -287,7 +298,12 @@ impl EncoderApi for AomEncoder { } impl AomEncoder { - pub fn encode<'a>(&'a mut self, ms: i64, data: &[u8], stride_align: usize) -> Result> { + pub fn encode<'a>( + &'a mut self, + ms: i64, + data: &[u8], + stride_align: usize, + ) -> Result> { let bpp = if self.i444 { 24 } else { 12 }; if data.len() < self.width * self.height * bpp / 8 { return Err(Error::FailedCall("len not enough".to_string())); diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 7f6a3e61a..bf7b2be04 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -32,7 +32,7 @@ use hbb_common::{ lazy_static, log, message_proto::{ supported_decoding::PreferCodec, video_frame, Chroma, CodecAbility, EncodedVideoFrames, - SupportedDecoding, SupportedEncoding, VideoFrame, + ImageQuality, SupportedDecoding, SupportedEncoding, VideoFrame, }, sysinfo::System, ResultType, @@ -47,6 +47,15 @@ lazy_static::lazy_static! { pub const ENCODE_NEED_SWITCH: &'static str = "ENCODE_NEED_SWITCH"; +#[derive(Debug, Clone, Copy, Default)] +pub struct RcState { + pub bitrate: u32, + pub qp: i32, + pub qp_min: i32, + pub qp_max: i32, + pub qp_mode: bool, +} + #[derive(Debug, Clone)] pub enum EncoderCfg { VPX(VpxEncoderConfig), @@ -71,10 +80,12 @@ pub trait EncoderApi { fn set_quality(&mut self, ratio: f32) -> ResultType<()>; - fn bitrate(&self) -> u32; + fn rc_state(&self) -> RcState; fn support_changing_quality(&self) -> bool; + fn rc_changed(&self, image_quality: ImageQuality) -> bool; + fn latency_free(&self) -> bool; fn is_hardware(&self) -> bool; @@ -896,6 +907,75 @@ pub const BR_BEST: f32 = 1.5; pub const BR_BALANCED: f32 = 1.0; pub const BR_SPEED: f32 = 0.5; +// Piecewise linear interpolation of QP over the ratio range. +// Working QP range: 18-44. Although H.264/HEVC uses 0-51 and VPX/AOM uses 0-63, +// we use the same model for all encoders — the 18-44 range provides a good +// perceptual quality spread and the encoder's own min/max quantizer settings +// handle the actual codec-specific mapping. +// Anchors: at_max (lowest ratio) → at_speed → at_balanced → at_best → at_min (highest ratio). +// Ratio above 2.0 is clamped to 2.0 (already at QP_MIN=18, no further improvement). +fn interpolate_qp( + ratio: f32, + at_min: i32, + at_best: i32, + at_balanced: i32, + at_speed: i32, + at_max: i32, +) -> i32 { + const QP_RATIO_MIN: f32 = 0.2; + const QP_RATIO_MAX: f32 = 2.0; + const QP_MIN: i32 = 18; + const QP_MAX: i32 = 44; + + let r = ratio.clamp(QP_RATIO_MIN, QP_RATIO_MAX); + let qp = if r <= BR_SPEED { + let t = (r - QP_RATIO_MIN) / (BR_SPEED - QP_RATIO_MIN); + at_max as f32 - (at_max - at_speed) as f32 * t + } else if r <= BR_BALANCED { + let t = (r - BR_SPEED) / (BR_BALANCED - BR_SPEED); + at_speed as f32 - (at_speed - at_balanced) as f32 * t + } else if r <= BR_BEST { + let t = (r - BR_BALANCED) / (BR_BEST - BR_BALANCED); + at_balanced as f32 - (at_balanced - at_best) as f32 * t + } else { + // BR_BEST(1.5) ~ QP_RATIO_MAX(2.0): continue decreasing QP toward at_min + let t = (r - BR_BEST) / (QP_RATIO_MAX - BR_BEST); + at_best as f32 - (at_best - at_min) as f32 * t + }; + (qp.round() as i32).clamp(QP_MIN, QP_MAX) +} + +// Map bitrate ratio to (qp, qmin, qmax) for QP-driven encoders (working range 18-44). +// Used by all encoder backends (hwcodec, VPX, VRAM). Lower QP means better quality. +// Ratio above 2.0 is treated as 2.0. +pub fn qp_for_ratio(ratio: f32) -> (i32, i32, i32) { + // at_min at_best at_balanced at_speed at_max + let qp = interpolate_qp(ratio, 18, 22, 28, 34, 44); + let qmin = interpolate_qp(ratio, 18, 18, 22, 28, 38); + let qmax = interpolate_qp(ratio, 22, 28, 34, 40, 44); + (qp, qmin, qmax) +} + +// Reverse lookup: find the ratio that produces the given target QP. +// Since qp_for_ratio is monotonically decreasing (higher ratio → lower QP), +// we use binary search within [ratio_min, ratio_max]. +pub fn ratio_for_qp(target_qp: i32, ratio_min: f32, ratio_max: f32) -> f32 { + let mut lo = ratio_min; + let mut hi = ratio_max; + for _ in 0..12 { + let mid = (lo + hi) / 2.0; + let (qp, _, _) = qp_for_ratio(mid); + if qp == target_qp { + return mid; + } else if qp > target_qp { + lo = mid; + } else { + hi = mid; + } + } + (lo + hi) / 2.0 +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum Quality { Best, @@ -1157,3 +1237,175 @@ pub fn test_av1() { }); }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_qp_for_ratio_monotonic() { + // Higher ratio should produce lower or equal QP (better quality) + let mut prev_qp = i32::MAX; + let mut prev_qmin = i32::MAX; + let mut prev_qmax = i32::MAX; + for i in 1..=200 { + let ratio = i as f32 * 0.1; + let (qp, qmin, qmax) = qp_for_ratio(ratio); + assert!( + qp <= prev_qp, + "qp not monotonic at ratio {}: {} > {}", + ratio, + qp, + prev_qp + ); + assert!( + qmin <= prev_qmin, + "qmin not monotonic at ratio {}: {} > {}", + ratio, + qmin, + prev_qmin + ); + assert!( + qmax <= prev_qmax, + "qmax not monotonic at ratio {}: {} > {}", + ratio, + qmax, + prev_qmax + ); + assert!( + qmin <= qp, + "qmin > qp at ratio {}: {} > {}", + ratio, + qmin, + qp + ); + assert!( + qp <= qmax, + "qp > qmax at ratio {}: {} > {}", + ratio, + qp, + qmax + ); + prev_qp = qp; + prev_qmin = qmin; + prev_qmax = qmax; + } + } + + #[test] + fn test_qp_for_ratio_known_values() { + let (qp, qmin, qmax) = qp_for_ratio(BR_SPEED); + assert_eq!(qp, 34); + assert_eq!(qmin, 28); + assert_eq!(qmax, 40); + + let (qp, qmin, qmax) = qp_for_ratio(BR_BALANCED); + assert_eq!(qp, 28); + assert_eq!(qmin, 22); + assert_eq!(qmax, 34); + + let (qp, qmin, qmax) = qp_for_ratio(BR_BEST); + assert_eq!(qp, 22); + assert_eq!(qmin, 18); + assert_eq!(qmax, 28); + } + + #[test] + fn test_qp_for_ratio_boundary() { + // Very low ratio (clamped to 0.2) + let (qp, qmin, qmax) = qp_for_ratio(0.0); + assert_eq!(qp, 44); + assert_eq!(qmin, 38); + assert_eq!(qmax, 44); + + // Very high ratio (clamped to 2.0) — same as ratio=2.0 + let (qp, qmin, qmax) = qp_for_ratio(100.0); + assert_eq!(qp, 18); + assert_eq!(qmin, 18); + assert_eq!(qmax, 22); + + // Ratio=2.0 should give QP_MIN + let (qp2, qmin2, qmax2) = qp_for_ratio(2.0); + assert_eq!((qp, qmin, qmax), (qp2, qmin2, qmax2)); + } + + #[test] + fn test_ratio_for_qp_roundtrip() { + for target_qp in 18..=44 { + let ratio = ratio_for_qp(target_qp, 0.2, 2.0); + let (qp, _, _) = qp_for_ratio(ratio); + assert!( + (qp - target_qp).abs() <= 1, + "roundtrip failed for target_qp={}, ratio={}, got qp={}", + target_qp, + ratio, + qp + ); + } + } + + #[test] + fn test_ratio_for_qp_boundary() { + let ratio = ratio_for_qp(28, 0.5, 1.5); + let (qp, _, _) = qp_for_ratio(ratio); + assert!( + (qp - 28).abs() <= 1, + "expected qp near 28, got qp={} at ratio={}", + qp, + ratio + ); + + // target_qp lower than anything in range should return near ratio_max + let ratio = ratio_for_qp(18, 0.2, 2.0); + assert!(ratio > 1.5, "expected high ratio for low qp, got {}", ratio); + + // target_qp higher than anything in range should return near ratio_min + let ratio = ratio_for_qp(44, 0.2, 2.0); + assert!(ratio < 0.5, "expected low ratio for high qp, got {}", ratio); + } + + #[test] + fn test_ratio_for_qp_monotonic() { + let mut prev_ratio = f32::MAX; + for qp in 18..=44 { + let ratio = ratio_for_qp(qp, 0.2, 2.0); + assert!( + ratio < prev_ratio, + "ratio not monotonic at qp={}: {} >= {}", + qp, + ratio, + prev_ratio + ); + prev_ratio = ratio; + } + } + + #[test] + fn test_ratio_for_qp_max_iterations() { + let mut max_iters = 0u32; + for target_qp in 18..=44 { + let mut lo = 0.2f32; + let mut hi = 2.0f32; + for i in 1..=32u32 { + let mid = (lo + hi) / 2.0; + let (qp, _, _) = qp_for_ratio(mid); + if qp == target_qp { + if i > max_iters { + max_iters = i; + } + break; + } else if qp > target_qp { + lo = mid; + } else { + hi = mid; + } + } + } + println!("max iterations needed: {}", max_iters); + assert!( + max_iters <= 12, + "needed {} iterations, expected <= 12", + max_iters + ); + } +} diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index a26d88a6c..fca4e6f6d 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -1,5 +1,7 @@ use crate::{ - codec::{base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg}, + codec::{ + base_bitrate, codec_thread_num, enable_hwcodec_option, qp_for_ratio, EncoderApi, EncoderCfg, + }, convert::*, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN, }; @@ -7,14 +9,13 @@ use hbb_common::{ anyhow::{anyhow, bail, Context}, bytes::Bytes, log, - message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame}, + message_proto::{EncodedVideoFrame, EncodedVideoFrames, ImageQuality, VideoFrame}, serde_derive::{Deserialize, Serialize}, serde_json, ResultType, }; use hwcodec::{ common::{ DataFormat, HwcodecErrno, - Quality::{self, *}, RateControl::{self, *}, }, ffmpeg::AVPixelFormat, @@ -28,8 +29,11 @@ use hwcodec::{ const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12; pub const DEFAULT_FPS: i32 = 30; const DEFAULT_GOP: i32 = i32::MAX; -const DEFAULT_HW_QUALITY: Quality = Quality_Default; pub const ERR_HEVC_POC: i32 = HwcodecErrno::HWCODEC_ERR_HEVC_COULD_NOT_FIND_POC as i32; +// AMF native encoder VBR may look blurry during initialization and when the first frame appears. It requires careful configuration and thorough testing before use. +pub const RC_BITRATE_MODE: RateControl = RateControl::RC_CBR; +// AMD encoders produce worse image quality at low bitrates, multiply to compensate +pub const AMD_BITRATE_MULTIPLIER: f32 = 1.5; crate::generate_call_macro!(call_yuv, false); @@ -46,6 +50,7 @@ pub struct HwRamEncoderConfig { pub width: usize, pub height: usize, pub quality: f32, + pub image_quality: ImageQuality, pub keyframe_interval: Option, } @@ -54,7 +59,11 @@ pub struct HwRamEncoder { pub format: DataFormat, pub pixfmt: AVPixelFormat, bitrate: u32, //kbs + qp: i32, + qp_min: i32, + qp_max: i32, config: HwRamEncoderConfig, + rc: RateControl, } impl EncoderApi for HwRamEncoder { @@ -64,11 +73,12 @@ impl EncoderApi for HwRamEncoder { { match cfg { EncoderCfg::HWRAM(config) => { - let rc = Self::rate_control(&config); + let rc = Self::rate_control(&config.name, config.image_quality); let mut bitrate = Self::bitrate(&config.name, config.width, config.height, config.quality); bitrate = Self::check_bitrate_range(&config, bitrate); let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32; + let (qp, qp_min, qp_max) = qp_for_ratio(config.quality); let ctx = EncodeContext { name: config.name.clone(), mc_name: config.mc_name.clone(), @@ -79,10 +89,10 @@ impl EncoderApi for HwRamEncoder { kbs: bitrate as i32, fps: DEFAULT_FPS, gop, - quality: DEFAULT_HW_QUALITY, rc, - q: -1, - thread_count: codec_thread_num(16) as _, // ffmpeg's thread_count is used for cpu + qp, + qp_min, + qp_max, }; let format = match Encoder::format_from_name(config.name.clone()) { Ok(format) => format, @@ -99,7 +109,11 @@ impl EncoderApi for HwRamEncoder { format, pixfmt: ctx.pixfmt, bitrate, + qp, + qp_min, + qp_max, config, + rc, }), Err(_) => Err(anyhow!(format!("Failed to create encoder"))), } @@ -179,15 +193,34 @@ impl EncoderApi for HwRamEncoder { ); if bitrate > 0 { bitrate = Self::check_bitrate_range(&self.config, bitrate); - self.encoder.set_bitrate(bitrate as _).ok(); - self.bitrate = bitrate; + if bitrate != self.bitrate { + self.encoder.set_bitrate(bitrate as _).ok(); + self.bitrate = bitrate; + } } self.config.quality = ratio; + let (qp, qp_min, qp_max) = qp_for_ratio(ratio); + if qp != self.qp || qp_min != self.qp_min || qp_max != self.qp_max { + self.encoder.set_qp(qp, qp_min, qp_max).ok(); + self.qp = qp; + self.qp_min = qp_min; + self.qp_max = qp_max; + } Ok(()) } - fn bitrate(&self) -> u32 { - self.bitrate + fn rc_state(&self) -> crate::codec::RcState { + crate::codec::RcState { + bitrate: self.bitrate, + qp: self.qp, + qp_min: self.qp_min, + qp_max: self.qp_max, + qp_mode: self.rc == RC_CQP, + } + } + + fn rc_changed(&self, image_quality: ImageQuality) -> bool { + Self::rate_control(&self.config.name, image_quality) != self.rc } fn support_changing_quality(&self) -> bool { @@ -240,19 +273,39 @@ impl HwRamEncoder { } } - fn rate_control(_config: &HwRamEncoderConfig) -> RateControl { + fn rate_control(name: &str, image_quality: ImageQuality) -> RateControl { #[cfg(target_os = "android")] - if _config.name.contains("mediacodec") { + if name.contains("mediacodec") { return RC_VBR; } - RC_CBR + // In balanced mode, CQP makes VS Code Inline Blame appear blurry, so CQP is only enabled at the highest quality level. + if image_quality == ImageQuality::Best + && ["qsv", "nvenc", "amf"].iter().any(|&x| name.contains(x)) + { + return RC_CQP; + } + if name.contains("qsv") { + return RC_VBR; + } + RC_BITRATE_MODE } pub fn bitrate(name: &str, width: usize, height: usize, ratio: f32) -> u32 { - Self::calc_bitrate(width, height, ratio, name.contains("h264")) + let multiplier = if name.contains("amf") { + AMD_BITRATE_MULTIPLIER + } else { + 1.0 + }; + Self::calc_bitrate(width, height, ratio, name.contains("h264"), multiplier) } - pub fn calc_bitrate(width: usize, height: usize, ratio: f32, h264: bool) -> u32 { + pub fn calc_bitrate( + width: usize, + height: usize, + ratio: f32, + h264: bool, + multiplier: f32, + ) -> u32 { let base = base_bitrate(width as _, height as _) as f32; // Monotonically increasing formula: factor = 1.0 + C / (1.0 + base / scale) // d(bitrate)/d(base) = 1 + C/(1+base/scale)² > 0, always positive @@ -267,7 +320,7 @@ impl HwRamEncoder { 1.0 + 1.2 / (1.0 + base / scale) }; // ratio is multiplied at the end for linear scaling with quality settings - (base * factor * ratio) as u32 + (base * factor * ratio * multiplier) as u32 } pub fn check_bitrate_range(_config: &HwRamEncoderConfig, bitrate: u32) -> u32 { @@ -671,6 +724,7 @@ impl HwCodecConfig { pub fn check_available_hwcodec() -> String { #[cfg(any(target_os = "linux", target_os = "macos"))] hwcodec::common::setup_parent_death_signal(); + let (qp, qp_min, qp_max) = crate::codec::qp_for_ratio(crate::codec::Quality::default().ratio()); let ctx = EncodeContext { name: String::from(""), mc_name: None, @@ -681,10 +735,10 @@ pub fn check_available_hwcodec() -> String { kbs: 1000, fps: DEFAULT_FPS, gop: DEFAULT_GOP, - quality: DEFAULT_HW_QUALITY, - rc: RC_CBR, - q: -1, - thread_count: 4, + rc: RC_BITRATE_MODE, + qp, + qp_min, + qp_max, }; #[cfg(feature = "vram")] let vram = crate::vram::check_available_vram(); diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index f41dfb134..8601c03f7 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -5,10 +5,12 @@ use hbb_common::anyhow::{anyhow, Context}; use hbb_common::log; -use hbb_common::message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame}; +use hbb_common::message_proto::{ + Chroma, EncodedVideoFrame, EncodedVideoFrames, ImageQuality, VideoFrame, +}; use hbb_common::ResultType; -use crate::codec::{base_bitrate, codec_thread_num, EncoderApi}; +use crate::codec::{base_bitrate, codec_thread_num, qp_for_ratio, EncoderApi}; use crate::{EncodeInput, EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN}; use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; @@ -17,6 +19,9 @@ use hbb_common::bytes::Bytes; use std::os::raw::{c_int, c_uint}; use std::{ptr, slice}; +// https://github.com/webmproject/libvpx/blob/6845d7229e1b1d9315f506a7c559ea1003d832ab/vpx/vpx_encoder.h#L243 +const RC_QP_MODE: vpx_rc_mode = vpx_rc_mode::VPX_CQ; + generate_call_macro!(call_vpx, false); generate_call_ptr_macro!(call_vpx_ptr); @@ -39,6 +44,8 @@ pub struct VpxEncoder { id: VpxVideoCodecId, i444: bool, yuvfmt: EncodeYuvFormat, + qp: i32, + rc: vpx_rc_mode, } pub struct VpxDecoder { @@ -73,9 +80,8 @@ impl EncoderApi for VpxEncoder { c.rc_dropframe_thresh = 25; c.g_threads = codec_thread_num(64) as _; c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; - // https://developers.google.com/media/vp9/bitrate-modes/ - // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. - c.rc_end_usage = vpx_rc_mode::VPX_CBR; + let rc = Self::rate_control(config.image_quality); + c.rc_end_usage = rc; if let Some(keyframe_interval) = config.keyframe_interval { c.kf_min_dist = 0; c.kf_max_dist = keyframe_interval as _; @@ -83,11 +89,11 @@ impl EncoderApi for VpxEncoder { c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot } - let (q_min, q_max) = Self::calc_q_values(config.quality); - c.rc_min_quantizer = q_min; - c.rc_max_quantizer = q_max; + let (qp, qp_min, qp_max) = qp_for_ratio(config.quality); + c.rc_min_quantizer = qp_min as _; + c.rc_max_quantizer = qp_max as _; c.rc_target_bitrate = - Self::bitrate(config.width as _, config.height as _, config.quality); + Self::bitrate(config.width as _, config.height as _, config.quality, rc); // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp9/common/vp9_enums.h#29 // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp8/vp8_cx_iface.c#282 c.g_profile = if i444 && config.codec == VpxVideoCodecId::VP9 { @@ -113,6 +119,16 @@ impl EncoderApi for VpxEncoder { VPX_ENCODER_ABI_VERSION as _ )); + // Set CQ level for Constrained Quality mode + if rc == RC_QP_MODE { + // https://github.com/webmproject/libvpx/blob/6845d7229e1b1d9315f506a7c559ea1003d832ab/vpx/vp8cx.h#L260 + call_vpx!(vpx_codec_control_( + &mut ctx, + VP8E_SET_CQ_LEVEL as _, + qp as c_int + )); + } + if config.codec == VpxVideoCodecId::VP9 { // set encoder internal speed settings // in ffmpeg, it is --speed option @@ -165,6 +181,8 @@ impl EncoderApi for VpxEncoder { id: config.codec, i444, yuvfmt: Self::get_yuvfmt(config.width, config.height, i444), + qp, + rc, }) } _ => Err(anyhow!("encoder type mismatch")), @@ -202,17 +220,35 @@ impl EncoderApi for VpxEncoder { fn set_quality(&mut self, ratio: f32) -> ResultType<()> { let mut c = unsafe { *self.ctx.config.enc.to_owned() }; - let (q_min, q_max) = Self::calc_q_values(ratio); - c.rc_min_quantizer = q_min; - c.rc_max_quantizer = q_max; - c.rc_target_bitrate = Self::bitrate(self.width as _, self.height as _, ratio); + let (qp, qp_min, qp_max) = qp_for_ratio(ratio); + c.rc_min_quantizer = qp_min as _; + c.rc_max_quantizer = qp_max as _; + c.rc_target_bitrate = Self::bitrate(self.width as _, self.height as _, ratio, self.rc); call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c)); + if self.rc == RC_QP_MODE { + call_vpx!(vpx_codec_control_( + &mut self.ctx, + VP8E_SET_CQ_LEVEL as _, + qp as c_int + )); + } + self.qp = qp; Ok(()) } - fn bitrate(&self) -> u32 { + fn rc_state(&self) -> crate::codec::RcState { let c = unsafe { *self.ctx.config.enc.to_owned() }; - c.rc_target_bitrate + crate::codec::RcState { + bitrate: c.rc_target_bitrate, + qp: self.qp, + qp_min: c.rc_min_quantizer as i32, + qp_max: c.rc_max_quantizer as i32, + qp_mode: self.rc == RC_QP_MODE, + } + } + + fn rc_changed(&self, image_quality: ImageQuality) -> bool { + Self::rate_control(image_quality) != self.rc } fn support_changing_quality(&self) -> bool { @@ -231,7 +267,12 @@ impl EncoderApi for VpxEncoder { } impl VpxEncoder { - pub fn encode<'a>(&'a mut self, pts: i64, data: &[u8], stride_align: usize) -> Result> { + pub fn encode<'a>( + &'a mut self, + pts: i64, + data: &[u8], + stride_align: usize, + ) -> Result> { let bpp = if self.i444 { 24 } else { 12 }; if data.len() < self.width * self.height * bpp / 8 { return Err(Error::FailedCall("len not enough".to_string())); @@ -311,29 +352,22 @@ impl VpxEncoder { } } - fn bitrate(width: u32, height: u32, ratio: f32) -> u32 { - let bitrate = base_bitrate(width, height) as f32; - (bitrate * ratio) as u32 + fn rate_control(image_quality: ImageQuality) -> vpx_rc_mode { + if image_quality == ImageQuality::Best { + RC_QP_MODE + } else { + vpx_rc_mode::VPX_CBR + } } - #[inline] - fn calc_q_values(ratio: f32) -> (u32, u32) { - let b = (ratio * 100.0) as u32; - let b = std::cmp::min(b, 200); - let q_min1 = 36; - let q_min2 = 0; - let q_max1 = 56; - let q_max2 = 37; - - let t = b as f32 / 200.0; - - let mut q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32; - let mut q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32; - - q_min = q_min.clamp(q_min2, q_min1); - q_max = q_max.clamp(q_max2, q_max1); - - (q_min, q_max) + fn bitrate(width: u32, height: u32, ratio: f32, rc: vpx_rc_mode) -> u32 { + let bitrate = base_bitrate(width, height) as f32; + let multiplier = if rc == RC_QP_MODE { + 2.0 // CQ mode: bitrate is ceiling, set higher to let CQ level control quality + } else { + 1.0 + }; + (bitrate * ratio * multiplier) as u32 } fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat { @@ -394,6 +428,8 @@ pub struct VpxEncoderConfig { pub height: c_uint, /// The bitrate ratio pub quality: f32, + /// The image quality + pub image_quality: ImageQuality, /// The codec pub codec: VpxVideoCodecId, /// keyframe interval diff --git a/libs/scrap/src/common/vram.rs b/libs/scrap/src/common/vram.rs index 22645d92b..d9b5b56c6 100644 --- a/libs/scrap/src/common/vram.rs +++ b/libs/scrap/src/common/vram.rs @@ -5,19 +5,19 @@ use std::{ }; use crate::{ - codec::{enable_vram_option, EncoderApi, EncoderCfg}, - hwcodec::HwCodecConfig, + codec::{enable_vram_option, EncoderApi, EncoderCfg, Quality}, + hwcodec::{HwCodecConfig, AMD_BITRATE_MULTIPLIER, RC_BITRATE_MODE}, AdapterDevice, CodecFormat, EncodeInput, EncodeYuvFormat, Pixfmt, }; use hbb_common::{ anyhow::{anyhow, bail, Context}, bytes::Bytes, log, - message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame}, + message_proto::{EncodedVideoFrame, EncodedVideoFrames, ImageQuality, VideoFrame}, ResultType, }; use hwcodec::{ - common::{DataFormat, Driver, MAX_GOP}, + common::{DataFormat, Driver, RateControl, MAX_GOP}, vram::{ decode::{self, DecodeFrame, Decoder}, encode::{self, EncodeFrame, Encoder}, @@ -40,6 +40,7 @@ pub struct VRamEncoderConfig { pub width: usize, pub height: usize, pub quality: f32, + pub image_quality: ImageQuality, pub feature: FeatureContext, pub keyframe_interval: Option, } @@ -49,8 +50,13 @@ pub struct VRamEncoder { pub format: DataFormat, ctx: EncodeContext, bitrate: u32, + qp: i32, + qp_min: i32, + qp_max: i32, last_frame_len: usize, same_bad_len_counter: usize, + rc: RateControl, + config: VRamEncoderConfig, } impl EncoderApi for VRamEncoder { @@ -60,13 +66,11 @@ impl EncoderApi for VRamEncoder { { match cfg { EncoderCfg::VRAM(config) => { - let bitrate = Self::bitrate( - config.feature.data_format, - config.width, - config.height, - config.quality, - ); + let bitrate = + Self::bitrate(&config.feature, config.width, config.height, config.quality); let gop = config.keyframe_interval.unwrap_or(MAX_GOP as _) as i32; + let (qp, qp_min, qp_max) = crate::codec::qp_for_ratio(config.quality); + let rc = Self::rate_control(&config.feature, config.image_quality); let ctx = EncodeContext { f: config.feature.clone(), d: DynamicContext { @@ -76,6 +80,10 @@ impl EncoderApi for VRamEncoder { kbitrate: bitrate as _, framerate: 30, gop, + qp, + qp_min, + qp_max, + rc, }, }; match Encoder::new(ctx.clone()) { @@ -84,8 +92,13 @@ impl EncoderApi for VRamEncoder { ctx, format: config.feature.data_format, bitrate, + qp, + qp_min, + qp_max, last_frame_len: 0, same_bad_len_counter: 0, + rc, + config, }), Err(_) => Err(anyhow!(format!("Failed to create encoder"))), } @@ -172,21 +185,38 @@ impl EncoderApi for VRamEncoder { fn set_quality(&mut self, ratio: f32) -> ResultType<()> { let bitrate = Self::bitrate( - self.ctx.f.data_format, + &self.ctx.f, self.ctx.d.width as _, self.ctx.d.height as _, ratio, ); - if bitrate > 0 { + if bitrate > 0 && bitrate != self.bitrate { if self.encoder.set_bitrate((bitrate) as _).is_ok() { self.bitrate = bitrate; } } + let (qp, qp_min, qp_max) = crate::codec::qp_for_ratio(ratio); + if qp != self.qp || qp_min != self.qp_min || qp_max != self.qp_max { + self.encoder.set_qp(qp, qp_min, qp_max).ok(); + self.qp = qp; + self.qp_min = qp_min; + self.qp_max = qp_max; + } Ok(()) } - fn bitrate(&self) -> u32 { - self.bitrate + fn rc_state(&self) -> crate::codec::RcState { + crate::codec::RcState { + bitrate: self.bitrate, + qp: self.qp, + qp_min: self.qp_min, + qp_max: self.qp_max, + qp_mode: self.rc == RateControl::RC_CQP, + } + } + + fn rc_changed(&self, image_quality: ImageQuality) -> bool { + Self::rate_control(&self.config.feature, image_quality) != self.rc } fn support_changing_quality(&self) -> bool { @@ -283,8 +313,19 @@ impl VRamEncoder { } } - pub fn bitrate(fmt: DataFormat, width: usize, height: usize, ratio: f32) -> u32 { - crate::hwcodec::HwRamEncoder::calc_bitrate(width, height, ratio, fmt == DataFormat::H264) + pub fn bitrate(f: &FeatureContext, width: usize, height: usize, ratio: f32) -> u32 { + let multiplier = if f.vendor == Driver::AMF { + AMD_BITRATE_MULTIPLIER + } else { + 1.0 + }; + crate::hwcodec::HwRamEncoder::calc_bitrate( + width, + height, + ratio, + f.data_format == DataFormat::H264, + multiplier, + ) } pub fn set_not_use(video_service_name: String, not_use: bool) { @@ -308,6 +349,16 @@ impl VRamEncoder { .remove(&video_service_name); } } + + fn rate_control(f: &FeatureContext, image_quality: ImageQuality) -> RateControl { + if image_quality == ImageQuality::Best { + return RateControl::RC_CQP; + } + if f.vendor == Driver::MFX { + return RateControl::RC_VBR; + } + RC_BITRATE_MODE + } } pub struct VRamDecoder { @@ -382,6 +433,7 @@ pub struct VRamDecoderImage<'a> { impl VRamDecoderImage<'_> {} pub(crate) fn check_available_vram() -> (Vec, Vec, String) { + let (qp, qp_min, qp_max) = crate::codec::qp_for_ratio(Quality::default().ratio()); let d = DynamicContext { device: None, width: 1280, @@ -389,6 +441,10 @@ pub(crate) fn check_available_vram() -> (Vec, Vec kbitrate: 5000, framerate: 60, gop: MAX_GOP as _, + qp, + qp_min, + qp_max, + rc: RC_BITRATE_MODE, }; let encoders = encode::available(d); let decoders = decode::available(); diff --git a/res/vcpkg/ffmpeg/patch/0013-nvenc-support-dynamic-constQP-reconfigure.patch b/res/vcpkg/ffmpeg/patch/0013-nvenc-support-dynamic-constQP-reconfigure.patch new file mode 100644 index 000000000..2d28e8b3c --- /dev/null +++ b/res/vcpkg/ffmpeg/patch/0013-nvenc-support-dynamic-constQP-reconfigure.patch @@ -0,0 +1,91 @@ +From 6209f4bd3a476a115040c9a603f5528a3cfc7d2e Mon Sep 17 00:00:00 2001 +From: 21pages +Date: Mon, 23 Feb 2026 17:22:47 +0800 +Subject: [PATCH] nvenc support dynamic constQP reconfigure + +Signed-off-by: 21pages +--- + libavcodec/nvenc.c | 53 +++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 52 insertions(+), 1 deletion(-) + +diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c +index f4c559b7ce..b7ceaadf02 100644 +--- a/libavcodec/nvenc.c ++++ b/libavcodec/nvenc.c +@@ -2689,7 +2689,7 @@ static void reconfig_encoder(AVCodecContext *avctx, const AVFrame *frame) + NV_ENC_RECONFIGURE_PARAMS params = { 0 }; + int needs_reconfig = 0; + int needs_encode_config = 0; +- int reconfig_bitrate = 0, reconfig_dar = 0; ++ int reconfig_bitrate = 0, reconfig_dar = 0, reconfig_qp = 0; + int dw, dh; + + params.version = NV_ENC_RECONFIGURE_PARAMS_VER; +@@ -2749,6 +2749,54 @@ static void reconfig_encoder(AVCodecContext *avctx, const AVFrame *frame) + } + } + ++ if (ctx->rc == NV_ENC_PARAMS_RC_CONSTQP) { ++ NV_ENC_RC_PARAMS *rc = ¶ms.reInitEncodeParams.encodeConfig->rcParams; ++ NV_ENC_QP new_qp = ctx->encode_config.rcParams.constQP; ++#if CONFIG_AV1_NVENC_ENCODER ++ int qmax = avctx->codec->id == AV_CODEC_ID_AV1 ? 255 : 51; ++#else ++ int qmax = 51; ++#endif ++ ++ if (ctx->init_qp_p >= 0) { ++ new_qp.qpInterP = ctx->init_qp_p; ++ if (ctx->init_qp_i >= 0 && ctx->init_qp_b >= 0) { ++ new_qp.qpIntra = ctx->init_qp_i; ++ new_qp.qpInterB = ctx->init_qp_b; ++ } else if (avctx->i_quant_factor != 0.0 && avctx->b_quant_factor != 0.0) { ++ new_qp.qpIntra = av_clip( ++ new_qp.qpInterP * fabs(avctx->i_quant_factor) + avctx->i_quant_offset + 0.5, 0, qmax); ++ new_qp.qpInterB = av_clip( ++ new_qp.qpInterP * fabs(avctx->b_quant_factor) + avctx->b_quant_offset + 0.5, 0, qmax); ++ } else { ++ new_qp.qpIntra = new_qp.qpInterP; ++ new_qp.qpInterB = new_qp.qpInterP; ++ } ++ } else if (ctx->cqp >= 0) { ++ new_qp.qpInterP = new_qp.qpInterB = new_qp.qpIntra = ctx->cqp; ++ if (avctx->b_quant_factor != 0.0) ++ new_qp.qpInterB = av_clip(ctx->cqp * fabs(avctx->b_quant_factor) + avctx->b_quant_offset + 0.5, 0, qmax); ++ if (avctx->i_quant_factor != 0.0) ++ new_qp.qpIntra = av_clip(ctx->cqp * fabs(avctx->i_quant_factor) + avctx->i_quant_offset + 0.5, 0, qmax); ++ } ++ ++ if (new_qp.qpInterP != ctx->encode_config.rcParams.constQP.qpInterP || ++ new_qp.qpIntra != ctx->encode_config.rcParams.constQP.qpIntra || ++ new_qp.qpInterB != ctx->encode_config.rcParams.constQP.qpInterB) { ++ av_log(avctx, AV_LOG_VERBOSE, ++ "constQP change: P %d->%d I %d->%d B %d->%d\n", ++ ctx->encode_config.rcParams.constQP.qpInterP, new_qp.qpInterP, ++ ctx->encode_config.rcParams.constQP.qpIntra, new_qp.qpIntra, ++ ctx->encode_config.rcParams.constQP.qpInterB, new_qp.qpInterB); ++ ++ rc->constQP = new_qp; ++ reconfig_qp = 1; ++ ++ needs_encode_config = 1; ++ needs_reconfig = 1; ++ } ++ } ++ + if (!needs_encode_config) + params.reInitEncodeParams.encodeConfig = NULL; + +@@ -2768,6 +2816,9 @@ static void reconfig_encoder(AVCodecContext *avctx, const AVFrame *frame) + ctx->encode_config.rcParams.vbvBufferSize = params.reInitEncodeParams.encodeConfig->rcParams.vbvBufferSize; + } + ++ if (reconfig_qp) { ++ ctx->encode_config.rcParams.constQP = params.reInitEncodeParams.encodeConfig->rcParams.constQP; ++ } + } + } + } +-- +2.43.0.windows.1 + diff --git a/res/vcpkg/ffmpeg/patch/0014-amfenc-support-dynamic-QP-and-qmin-qmax-reconfigure.patch b/res/vcpkg/ffmpeg/patch/0014-amfenc-support-dynamic-QP-and-qmin-qmax-reconfigure.patch new file mode 100644 index 000000000..269931d3f --- /dev/null +++ b/res/vcpkg/ffmpeg/patch/0014-amfenc-support-dynamic-QP-and-qmin-qmax-reconfigure.patch @@ -0,0 +1,176 @@ +From 817066f5804d914762410d1fd07c36cc45462ed5 Mon Sep 17 00:00:00 2001 +From: 21pages +Date: Mon, 23 Feb 2026 22:18:39 +0800 +Subject: [PATCH] amfenc: support dynamic QP and qmin/qmax reconfigure + +Signed-off-by: 21pages +--- + libavcodec/amfenc.c | 119 +++++++++++++++++++++++++++++++++++++++++--- + libavcodec/amfenc.h | 11 +++- + 2 files changed, 121 insertions(+), 9 deletions(-) + +diff --git a/libavcodec/amfenc.c b/libavcodec/amfenc.c +index a53a05b16b..eb960d0e8a 100644 +--- a/libavcodec/amfenc.c ++++ b/libavcodec/amfenc.c +@@ -275,7 +275,14 @@ static int amf_init_context(AVCodecContext *avctx) + + ctx->hwsurfaces_in_queue = 0; + ctx->hwsurfaces_in_queue_max = 16; +- ctx->av_bitrate = avctx->bit_rate; ++ ctx->last_bit_rate = avctx->bit_rate; ++ ctx->last_qp_i = ctx->qp_i; ++ ctx->last_qp_p = ctx->qp_p; ++ ctx->last_qp_b = ctx->qp_b; ++ ctx->last_min_qp_i = ctx->min_qp_i; ++ ctx->last_max_qp_i = ctx->max_qp_i; ++ ctx->last_min_qp_p = ctx->min_qp_p; ++ ctx->last_max_qp_p = ctx->max_qp_p; + + // configure AMF logger + // the return of these functions indicates old state and do not affect behaviour +@@ -645,16 +652,112 @@ static int reconfig_encoder(AVCodecContext *avctx) + { + AmfContext *ctx = avctx->priv_data; + AMF_RESULT res = AMF_OK; ++ int is_cqp = (avctx->codec->id == AV_CODEC_ID_H264 && ++ ctx->rate_control_mode == AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_CONSTANT_QP) || ++ (avctx->codec->id == AV_CODEC_ID_HEVC && ++ ctx->rate_control_mode == AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD_CONSTANT_QP); ++ ++ if (ctx->last_bit_rate != avctx->bit_rate) { ++ if (!is_cqp) { ++ av_log(ctx, AV_LOG_INFO, "change bitrate from %d to %d\n", ctx->last_bit_rate, avctx->bit_rate); ++ if (avctx->codec->id == AV_CODEC_ID_H264) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_TARGET_BITRATE, avctx->bit_rate); ++ } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_TARGET_BITRATE, avctx->bit_rate); ++ } ++ } ++ ctx->last_bit_rate = avctx->bit_rate; ++ } ++ ++ // Dynamic QP/qmin/qmax reconfiguration ++ // Reference: https://amd.github.io/ama-sdk/latest/tuning_video_quality.html#dynamic-encoder-parameters ++ // CQP mode: dynamic QP, qmin, qmax ++ // CBR/VBR modes: dynamic qmin, qmax (not QP) ++ if (avctx->codec->id == AV_CODEC_ID_H264) { ++ ++ // Dynamic QP (CQP mode only) ++ if (is_cqp) { ++ if (ctx->qp_i >= 0 && ctx->qp_i != ctx->last_qp_i) { ++ av_log(ctx, AV_LOG_VERBOSE, "change QP I from %d to %d\n", ctx->last_qp_i, ctx->qp_i); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_QP_I, ctx->qp_i); ++ ctx->last_qp_i = ctx->qp_i; ++ } ++ if (ctx->qp_p >= 0 && ctx->qp_p != ctx->last_qp_p) { ++ av_log(ctx, AV_LOG_VERBOSE, "change QP P from %d to %d\n", ctx->last_qp_p, ctx->qp_p); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_QP_P, ctx->qp_p); ++ ctx->last_qp_p = ctx->qp_p; ++ } ++ if (ctx->qp_b >= 0 && ctx->qp_b != ctx->last_qp_b) { ++ av_log(ctx, AV_LOG_VERBOSE, "change QP B from %d to %d\n", ctx->last_qp_b, ctx->qp_b); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_QP_B, ctx->qp_b); ++ ctx->last_qp_b = ctx->qp_b; ++ } ++ } ++ ++ // Dynamic qmin/qmax (all modes) ++ if (avctx->qmin >= 0 && avctx->qmin != ctx->last_min_qp_i) { ++ int qval = FFMIN(avctx->qmin, 51); ++ av_log(ctx, AV_LOG_VERBOSE, "change min QP from %d to %d\n", ctx->last_min_qp_i, qval); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_MIN_QP, qval); ++ ctx->last_min_qp_i = avctx->qmin; ++ } ++ if (avctx->qmax >= 0 && avctx->qmax != ctx->last_max_qp_i) { ++ int qval = FFMIN(avctx->qmax, 51); ++ av_log(ctx, AV_LOG_VERBOSE, "change max QP from %d to %d\n", ctx->last_max_qp_i, qval); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_MAX_QP, qval); ++ ctx->last_max_qp_i = avctx->qmax; ++ } ++ } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { ++ ++ // Dynamic QP (CQP mode only) ++ if (is_cqp) { ++ if (ctx->qp_i >= 0 && ctx->qp_i != ctx->last_qp_i) { ++ av_log(ctx, AV_LOG_VERBOSE, "change QP I from %d to %d\n", ctx->last_qp_i, ctx->qp_i); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_QP_I, ctx->qp_i); ++ ctx->last_qp_i = ctx->qp_i; ++ } ++ if (ctx->qp_p >= 0 && ctx->qp_p != ctx->last_qp_p) { ++ av_log(ctx, AV_LOG_VERBOSE, "change QP P from %d to %d\n", ctx->last_qp_p, ctx->qp_p); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_QP_P, ctx->qp_p); ++ ctx->last_qp_p = ctx->qp_p; ++ } ++ } + +- if (ctx->av_bitrate != avctx->bit_rate) { +- av_log(ctx, AV_LOG_INFO, "change bitrate from %d to %d\n", ctx->av_bitrate, avctx->bit_rate); +- ctx->av_bitrate = avctx->bit_rate; +- if (avctx->codec->id == AV_CODEC_ID_H264) { +- AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_TARGET_BITRATE, avctx->bit_rate); +- } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { +- AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_TARGET_BITRATE, avctx->bit_rate); ++ // Dynamic qmin/qmax (all modes) - HEVC has per-frame-type min/max ++ { ++ int new_val = ctx->min_qp_i >= 0 ? ctx->min_qp_i : (avctx->qmin >= 0 ? FFMIN(avctx->qmin, 51) : -1); ++ if (new_val >= 0 && new_val != ctx->last_min_qp_i) { ++ av_log(ctx, AV_LOG_VERBOSE, "change min QP I from %d to %d\n", ctx->last_min_qp_i, new_val); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_MIN_QP_I, new_val); ++ ctx->last_min_qp_i = new_val; ++ } ++ } ++ { ++ int new_val = ctx->max_qp_i >= 0 ? ctx->max_qp_i : (avctx->qmax >= 0 ? FFMIN(avctx->qmax, 51) : -1); ++ if (new_val >= 0 && new_val != ctx->last_max_qp_i) { ++ av_log(ctx, AV_LOG_VERBOSE, "change max QP I from %d to %d\n", ctx->last_max_qp_i, new_val); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_MAX_QP_I, new_val); ++ ctx->last_max_qp_i = new_val; ++ } ++ } ++ { ++ int new_val = ctx->min_qp_p >= 0 ? ctx->min_qp_p : (avctx->qmin >= 0 ? FFMIN(avctx->qmin, 51) : -1); ++ if (new_val >= 0 && new_val != ctx->last_min_qp_p) { ++ av_log(ctx, AV_LOG_VERBOSE, "change min QP P from %d to %d\n", ctx->last_min_qp_p, new_val); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_MIN_QP_P, new_val); ++ ctx->last_min_qp_p = new_val; ++ } ++ } ++ { ++ int new_val = ctx->max_qp_p >= 0 ? ctx->max_qp_p : (avctx->qmax >= 0 ? FFMIN(avctx->qmax, 51) : -1); ++ if (new_val >= 0 && new_val != ctx->last_max_qp_p) { ++ av_log(ctx, AV_LOG_VERBOSE, "change max QP P from %d to %d\n", ctx->last_max_qp_p, new_val); ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_MAX_QP_P, new_val); ++ ctx->last_max_qp_p = new_val; ++ } + } + } ++ + return 0; + } + +diff --git a/libavcodec/amfenc.h b/libavcodec/amfenc.h +index 481e0fb75d..8d4ee064dd 100644 +--- a/libavcodec/amfenc.h ++++ b/libavcodec/amfenc.h +@@ -115,7 +115,16 @@ typedef struct AmfContext { + int max_b_frames; + int qvbr_quality_level; + int hw_high_motion_quality_boost; +- int64_t av_bitrate; ++ int64_t last_bit_rate; ++ ++ // Dynamic reconfiguration tracking ++ int last_qp_i; ++ int last_qp_p; ++ int last_qp_b; ++ int last_min_qp_i; ++ int last_max_qp_i; ++ int last_min_qp_p; ++ int last_max_qp_p; + + // HEVC - specific options + +-- +2.43.0.windows.1 + diff --git a/res/vcpkg/ffmpeg/patch/0015-amfenc-support-dynamic-VBV-buffer-size-initial-fulln.patch b/res/vcpkg/ffmpeg/patch/0015-amfenc-support-dynamic-VBV-buffer-size-initial-fulln.patch new file mode 100644 index 000000000..75a5c65dc --- /dev/null +++ b/res/vcpkg/ffmpeg/patch/0015-amfenc-support-dynamic-VBV-buffer-size-initial-fulln.patch @@ -0,0 +1,92 @@ +From f7c7fbbbcf1559fb8e044b07250b0bc8c7e55930 Mon Sep 17 00:00:00 2001 +From: 21pages +Date: Thu, 26 Feb 2026 16:56:28 +0800 +Subject: [PATCH] amfenc: support dynamic VBV buffer size, initial fullness and + peak bitrate reconfigure + +Signed-off-by: 21pages +--- + libavcodec/amfenc.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ + libavcodec/amfenc.h | 3 +++ + 2 files changed, 48 insertions(+) + +diff --git a/libavcodec/amfenc.c b/libavcodec/amfenc.c +index eb960d0e8a..ae36bf328d 100644 +--- a/libavcodec/amfenc.c ++++ b/libavcodec/amfenc.c +@@ -283,6 +283,9 @@ static int amf_init_context(AVCodecContext *avctx) + ctx->last_max_qp_i = ctx->max_qp_i; + ctx->last_min_qp_p = ctx->min_qp_p; + ctx->last_max_qp_p = ctx->max_qp_p; ++ ctx->last_rc_buffer_size = avctx->rc_buffer_size; ++ ctx->last_rc_initial_buffer_occupancy = avctx->rc_initial_buffer_occupancy; ++ ctx->last_rc_max_rate = avctx->rc_max_rate; + + // configure AMF logger + // the return of these functions indicates old state and do not affect behaviour +@@ -669,6 +672,48 @@ static int reconfig_encoder(AVCodecContext *avctx) + ctx->last_bit_rate = avctx->bit_rate; + } + ++ // Dynamic VBV buffer size reconfiguration ++ if (avctx->rc_buffer_size && avctx->rc_buffer_size != ctx->last_rc_buffer_size) { ++ if (!is_cqp) { ++ av_log(ctx, AV_LOG_INFO, "change rc_buffer_size from %d to %d\n", ctx->last_rc_buffer_size, avctx->rc_buffer_size); ++ if (avctx->codec->id == AV_CODEC_ID_H264) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_VBV_BUFFER_SIZE, avctx->rc_buffer_size); ++ } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_VBV_BUFFER_SIZE, avctx->rc_buffer_size); ++ } ++ } ++ ctx->last_rc_buffer_size = avctx->rc_buffer_size; ++ } ++ ++ // Dynamic initial VBV buffer fullness reconfiguration ++ if (avctx->rc_initial_buffer_occupancy && avctx->rc_initial_buffer_occupancy != ctx->last_rc_initial_buffer_occupancy) { ++ if (!is_cqp && avctx->rc_buffer_size > 0) { ++ int amf_buffer_fullness = avctx->rc_initial_buffer_occupancy * 64 / avctx->rc_buffer_size; ++ if (amf_buffer_fullness > 64) ++ amf_buffer_fullness = 64; ++ av_log(ctx, AV_LOG_INFO, "change rc_initial_buffer_occupancy from %d to %d\n", ctx->last_rc_initial_buffer_occupancy, avctx->rc_initial_buffer_occupancy); ++ if (avctx->codec->id == AV_CODEC_ID_H264) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_INITIAL_VBV_BUFFER_FULLNESS, amf_buffer_fullness); ++ } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_INITIAL_VBV_BUFFER_FULLNESS, amf_buffer_fullness); ++ } ++ } ++ ctx->last_rc_initial_buffer_occupancy = avctx->rc_initial_buffer_occupancy; ++ } ++ ++ // Dynamic peak bitrate reconfiguration ++ if (avctx->rc_max_rate && avctx->rc_max_rate != ctx->last_rc_max_rate) { ++ if (!is_cqp) { ++ av_log(ctx, AV_LOG_INFO, "change rc_max_rate from %"PRId64" to %"PRId64"\n", ctx->last_rc_max_rate, avctx->rc_max_rate); ++ if (avctx->codec->id == AV_CODEC_ID_H264) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_PEAK_BITRATE, avctx->rc_max_rate); ++ } else if (avctx->codec->id == AV_CODEC_ID_HEVC) { ++ AMF_ASSIGN_PROPERTY_INT64(res, ctx->encoder, AMF_VIDEO_ENCODER_HEVC_PEAK_BITRATE, avctx->rc_max_rate); ++ } ++ } ++ ctx->last_rc_max_rate = avctx->rc_max_rate; ++ } ++ + // Dynamic QP/qmin/qmax reconfiguration + // Reference: https://amd.github.io/ama-sdk/latest/tuning_video_quality.html#dynamic-encoder-parameters + // CQP mode: dynamic QP, qmin, qmax +diff --git a/libavcodec/amfenc.h b/libavcodec/amfenc.h +index 8d4ee064dd..dc9ad9942b 100644 +--- a/libavcodec/amfenc.h ++++ b/libavcodec/amfenc.h +@@ -125,6 +125,9 @@ typedef struct AmfContext { + int last_max_qp_i; + int last_min_qp_p; + int last_max_qp_p; ++ int last_rc_buffer_size; ++ int last_rc_initial_buffer_occupancy; ++ int64_t last_rc_max_rate; + + // HEVC - specific options + +-- +2.43.0.windows.1 + diff --git a/res/vcpkg/ffmpeg/portfile.cmake b/res/vcpkg/ffmpeg/portfile.cmake index 16cef8350..2df425071 100644 --- a/res/vcpkg/ffmpeg/portfile.cmake +++ b/res/vcpkg/ffmpeg/portfile.cmake @@ -28,6 +28,9 @@ vcpkg_from_github( patch/0010.disable-loading-DLLs-from-app-dir.patch patch/0011-android-mediacodec-encode-align-64.patch patch/0012-fix-macos-big-sur-CVBufferCopyAttachments.patch + patch/0013-nvenc-support-dynamic-constQP-reconfigure.patch + patch/0014-amfenc-support-dynamic-QP-and-qmin-qmax-reconfigure.patch + patch/0015-amfenc-support-dynamic-VBV-buffer-size-initial-fulln.patch ) if(SOURCE_PATH MATCHES " ") diff --git a/src/server/connection.rs b/src/server/connection.rs index 10b578042..43b22d2c2 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -930,7 +930,7 @@ impl Connection { let mut msg_out = Message::new(); msg_out.set_test_delay(TestDelay{ last_delay: conn.network_delay, - target_bitrate: video_service::VIDEO_QOS.lock().unwrap().bitrate(), + target_bitrate: video_service::VIDEO_QOS.lock().unwrap().bitrate_in_test_delay(), ..Default::default() }); conn.send(msg_out.into()).await; @@ -5346,9 +5346,8 @@ mod raii { } pub fn check_wake_lock_on_setting_changed() { - let current = config::Config::get_bool_option( - keys::OPTION_KEEP_AWAKE_DURING_INCOMING_SESSIONS, - ); + let current = + config::Config::get_bool_option(keys::OPTION_KEEP_AWAKE_DURING_INCOMING_SESSIONS); let cached = *WAKELOCK_KEEP_AWAKE_OPTION.lock().unwrap(); if cached != Some(current) { Self::check_wake_lock(); diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs index 948d31ebf..fbce7d66a 100644 --- a/src/server/video_qos.rs +++ b/src/server/video_qos.rs @@ -1,5 +1,5 @@ use super::*; -use scrap::codec::{Quality, BR_BALANCED, BR_BEST, BR_SPEED}; +use scrap::codec::{ratio_for_qp, Quality, RcState, BR_BALANCED, BR_BEST, BR_SPEED}; use std::{ collections::VecDeque, time::{Duration, Instant}, @@ -108,7 +108,7 @@ pub struct VideoQoS { ratio: f32, users: HashMap, displays: HashMap, - bitrate_store: u32, + rc_state: RcState, adjust_ratio_instant: Instant, abr_config: bool, new_user_instant: Instant, @@ -121,7 +121,7 @@ impl Default for VideoQoS { ratio: BR_BALANCED, users: Default::default(), displays: Default::default(), - bitrate_store: 0, + rc_state: Default::default(), adjust_ratio_instant: Instant::now(), abr_config: true, new_user_instant: Instant::now(), @@ -146,14 +146,19 @@ impl VideoQoS { } } - // Store bitrate for later use - pub fn store_bitrate(&mut self, bitrate: u32) { - self.bitrate_store = bitrate; + // Store rate control state for later use + pub fn store_rc_state(&mut self, rc_state: RcState) { + self.rc_state = rc_state; } // Get stored bitrate - pub fn bitrate(&self) -> u32 { - self.bitrate_store + pub fn bitrate_in_test_delay(&self) -> u32 { + if self.rc_state.qp_mode { + // For CQP mode, bitrate is not fixed, return 0 to indicate it's not applicable + 0 + } else { + self.rc_state.bitrate + } } // Get current bitrate ratio with bounds checking @@ -413,6 +418,16 @@ impl VideoQoS { .1 } + pub fn latest_image_quality(&self) -> ImageQuality { + let quality = self.latest_quality(); + match quality { + Quality::Best => ImageQuality::Best, + Quality::Balanced => ImageQuality::Balanced, + Quality::Low => ImageQuality::Low, + Quality::Custom(_) => ImageQuality::NotSet, // For custom quality, we don't handle this in rate control, change this if needed + } + } + // Adjust quality ratio based on network delay and screen changes fn adjust_ratio(&mut self, dynamic_screen: bool) { if !self.in_vbr_state() { @@ -427,14 +442,7 @@ impl VideoQoS { let target_quality = self.latest_quality(); let target_ratio = self.latest_quality().ratio(); let current_ratio = self.ratio; - let current_bitrate = self.bitrate(); - - // Calculate ratio for adding 150kbps bandwidth - let ratio_add_150kbps = if current_bitrate > 0 { - Some((current_bitrate + 150) as f32 * current_ratio / current_bitrate as f32) - } else { - None - }; + let rc_state = self.rc_state; // Set minimum ratio based on quality mode // Best(0.6) > Balanced(0.4) > Low(0.25) >= Custom(0.2~0.6) @@ -483,12 +491,23 @@ impl VideoQoS { } // Limit quality increase rate for better stability - if let Some(ratio_add_150kbps) = ratio_add_150kbps { - if v > ratio_add_150kbps - && ratio_add_150kbps > current_ratio - && current_ratio >= BR_SPEED - { - v = ratio_add_150kbps; + if v > current_ratio && current_ratio >= BR_SPEED { + if rc_state.qp_mode { + // For CQP mode, limit QP decrease to 1 step per adjustment period + let max_ratio = ratio_for_qp(rc_state.qp - 1, current_ratio, v); + if v > max_ratio { + v = max_ratio; + } + } else { + // For CBR/VBR mode, cap increase to equivalent of adding 150kbps bandwidth + let current_bitrate = rc_state.bitrate; + if current_bitrate > 0 { + let ratio_add_150kbps = + (current_bitrate + 150) as f32 * current_ratio / current_bitrate as f32; + if v > ratio_add_150kbps { + v = ratio_add_150kbps; + } + } } } diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 13a781c28..a500bae2e 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -576,11 +576,13 @@ fn run(vs: VideoService) -> ResultType<()> { &Config::get_option("allow-auto-record-incoming"), ); let client_record = video_qos.record(); + let image_quality = video_qos.latest_image_quality(); drop(video_qos); let (mut encoder, encoder_cfg, codec_format, use_i444, recorder) = match setup_encoder( &c, sp.name(), quality, + image_quality, client_record, record_incoming, last_portable_service_running, @@ -594,6 +596,7 @@ fn run(vs: VideoService) -> ResultType<()> { width: c.width as _, height: c.height as _, quality, + image_quality, codec: VpxVideoCodecId::VP9, keyframe_interval: None, })); @@ -601,6 +604,7 @@ fn run(vs: VideoService) -> ResultType<()> { &c, sp.name(), quality, + image_quality, client_record, record_incoming, last_portable_service_running, @@ -618,7 +622,7 @@ fn run(vs: VideoService) -> ResultType<()> { bail!(e); } } - VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate()); + VIDEO_QOS.lock().unwrap().store_rc_state(encoder.rc_state()); VIDEO_QOS .lock() .unwrap() @@ -658,6 +662,7 @@ fn run(vs: VideoService) -> ResultType<()> { check_qos( &mut encoder, &mut quality, + image_quality, &mut spf, client_record, &mut send_counter, @@ -926,6 +931,7 @@ fn setup_encoder( c: &CapturerInfo, name: String, quality: f32, + image_quality: ImageQuality, client_record: bool, record_incoming: bool, last_portable_service_running: bool, @@ -942,6 +948,7 @@ fn setup_encoder( &c, name.to_string(), quality, + image_quality, client_record || record_incoming, last_portable_service_running, source, @@ -958,6 +965,7 @@ fn get_encoder_config( c: &CapturerInfo, _name: String, quality: f32, + image_quality: ImageQuality, record: bool, _portable_service: bool, _source: VideoSource, @@ -981,6 +989,7 @@ fn get_encoder_config( width: c.width, height: c.height, quality, + image_quality, feature, keyframe_interval, }); @@ -993,6 +1002,7 @@ fn get_encoder_config( width: c.width, height: c.height, quality, + image_quality, keyframe_interval, }); } @@ -1000,6 +1010,7 @@ fn get_encoder_config( width: c.width as _, height: c.height as _, quality, + image_quality, codec: VpxVideoCodecId::VP9, keyframe_interval, }) @@ -1008,6 +1019,7 @@ fn get_encoder_config( width: c.width as _, height: c.height as _, quality, + image_quality, codec: if format == CodecFormat::VP8 { VpxVideoCodecId::VP8 } else { @@ -1025,6 +1037,7 @@ fn get_encoder_config( width: c.width as _, height: c.height as _, quality, + image_quality, codec: VpxVideoCodecId::VP9, keyframe_interval, }), @@ -1310,6 +1323,7 @@ pub fn make_display_changed_msg( fn check_qos( encoder: &mut Encoder, ratio: &mut f32, + image_quality: ImageQuality, spf: &mut Duration, client_record: bool, send_counter: &mut usize, @@ -1318,11 +1332,16 @@ fn check_qos( ) -> ResultType<()> { let mut video_qos = VIDEO_QOS.lock().unwrap(); *spf = video_qos.spf(); + let latest_image_quality = video_qos.latest_image_quality(); + if image_quality != latest_image_quality && encoder.rc_changed(latest_image_quality) { + log::info!("switch due to rate control changed"); + bail!("SWITCH"); + } if *ratio != video_qos.ratio() { *ratio = video_qos.ratio(); if encoder.support_changing_quality() { allow_err!(encoder.set_quality(*ratio)); - video_qos.store_bitrate(encoder.bitrate()); + video_qos.store_rc_state(encoder.rc_state()); } else { // Now only vaapi doesn't support changing quality if !video_qos.in_vbr_state() && !video_qos.latest_quality().is_custom() { diff --git a/src/ui/remote.tis b/src/ui/remote.tis index 0dd574af7..baa144a5f 100644 --- a/src/ui/remote.tis +++ b/src/ui/remote.tis @@ -522,6 +522,7 @@ class QualityMonitor: Reactor.Component } function render() { + var showBitrate = typeof qualityMonitorData[3] == "integer" && qualityMonitorData[3] >= 1; return
Speed: {qualityMonitorData[0]} @@ -532,9 +533,9 @@ class QualityMonitor: Reactor.Component
Delay: {qualityMonitorData[2]} ms
-
+ {showBitrate ?
Target Bitrate: {qualityMonitorData[3]}kb -
+
: undefined}
Codec: {qualityMonitorData[4]}