Cross-platform screen capture + H.264 encoder

- Implement unified Capturer trait with platform-specific stubs: DXGI (Win), CGDisplay (macOS), PipeWire (Linux)
- Implement SoftwareEncoder using ffmpeg-next (libx264) with RGBA→YUV420P conversion, low-latency preset
- Add create_capturer() factory function for platform auto-detection
- Register capture and encoder Tauri commands (start_capture, init_encoder, get_display_resolution)
This commit is contained in:
yuqianhe
2026-05-30 08:05:07 +00:00
parent e73824a48e
commit 083abe75a1
7 changed files with 299 additions and 19 deletions

View File

@@ -0,0 +1,33 @@
//! Linux screen capture (PipeWire / X11 stub)
//!
//! TODO: Implement using `pipewire` or `x11` + `xcb` crates.
use anyhow::{bail, Result};
use super::CapturedFrame;
pub struct LinuxCapturer {
width: u32,
height: u32,
fps: u32,
}
impl LinuxCapturer {
pub fn new(fps: u32) -> Result<Self> {
Ok(Self {
width: 1920,
height: 1080,
fps,
})
}
}
impl super::Capturer for LinuxCapturer {
fn capture_frame(&mut self) -> Result<CapturedFrame> {
// FIXME: capture using PipeWire or X11/Shm
bail!("Linux capture not yet implemented")
}
fn resolution(&self) -> (u32, u32) {
(self.width, self.height)
}
}

View File

@@ -0,0 +1,33 @@
//! macOS screen capture (CGDisplay stub)
//!
//! TODO: Implement using `core-graphics` or ScreenCaptureKit.
use anyhow::{bail, Result};
use super::CapturedFrame;
pub struct MacOsCapturer {
width: u32,
height: u32,
fps: u32,
}
impl MacOsCapturer {
pub fn new(fps: u32) -> Result<Self> {
Ok(Self {
width: 1920,
height: 1080,
fps,
})
}
}
impl super::Capturer for MacOsCapturer {
fn capture_frame(&mut self) -> Result<CapturedFrame> {
// FIXME: capture using CGDisplay or SCK
bail!("macOS capture not yet implemented")
}
fn resolution(&self) -> (u32, u32) {
(self.width, self.height)
}
}

View File

@@ -1,2 +1,6 @@
//! Screen capture (platform specific)
pub mod platform;
//! Cross-platform screen capture
//!
//! Provides a unified `Capturer` trait for capturing desktop frames.
mod platform;
pub use platform::*;

View File

@@ -1,14 +1,23 @@
//! Platform-specific screen capture
//! Platform-specific screen capture modules and the unified `Capturer` trait.
//!
//! Each platform module provides a `Capturer` that produces raw RGBA frames.
//! Each platform module implements the actual frame capture:
//! - Windows: DXGI Desktop Duplication API
//! - macOS: CGDisplay / ScreenCaptureKit
//! - Linux: PipeWire / X11 (xcb)
#[cfg(target_os = "windows")]
pub mod windows;
#[cfg(target_os = "macos")]
pub mod macos;
#[cfg(target_os = "linux")]
pub mod linux;
use anyhow::Result;
/// A raw RGBA frame captured from the desktop.
#[derive(Debug, Clone)]
pub struct CapturedFrame {
pub data: Vec<u8>,
pub width: u32,
@@ -16,6 +25,34 @@ pub struct CapturedFrame {
pub stride: u32,
}
pub trait Capturer {
fn capture_frame(&mut self) -> anyhow::Result<CapturedFrame>;
/// Unified screen capture trait.
pub trait Capturer: Send {
/// Capture the current desktop frame. Returns raw RGBA pixel data.
fn capture_frame(&mut self) -> Result<CapturedFrame>;
/// Get the current capture resolution.
fn resolution(&self) -> (u32, u32);
}
/// Create a platform-appropriate screen capturer.
pub fn create_capturer(fps: u32) -> Result<Box<dyn Capturer>> {
#[cfg(target_os = "windows")]
{
let capturer = windows::WindowsCapturer::new(fps)?;
Ok(Box::new(capturer))
}
#[cfg(target_os = "macos")]
{
let capturer = macos::MacOsCapturer::new(fps)?;
Ok(Box::new(capturer))
}
#[cfg(target_os = "linux")]
{
let capturer = linux::LinuxCapturer::new(fps)?;
Ok(Box::new(capturer))
}
#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))]
{
Err(anyhow::anyhow!("Unsupported platform for screen capture"))
}
}

View File

@@ -0,0 +1,34 @@
//! Windows screen capture (DXGI Desktop Duplication API stub)
//!
//! TODO: Implement using `dxgi` + `d3d11` crates or `captrs`.
use anyhow::{bail, Result};
use super::CapturedFrame;
pub struct WindowsCapturer {
width: u32,
height: u32,
fps: u32,
}
impl WindowsCapturer {
pub fn new(fps: u32) -> Result<Self> {
// FIXME: init DXGI duplication
Ok(Self {
width: 1920,
height: 1080,
fps,
})
}
}
impl super::Capturer for WindowsCapturer {
fn capture_frame(&mut self) -> Result<CapturedFrame> {
// FIXME: capture using DXGI OutputDuplication
bail!("Windows capture not yet implemented")
}
fn resolution(&self) -> (u32, u32) {
(self.width, self.height)
}
}

View File

@@ -1,21 +1,143 @@
//! Software H.264 encoder using ffmpeg-next (libx264)
//!
//! Converts raw RGBA frames to Annex-B H.264 NAL units.
use anyhow::Result;
use anyhow::{Context, Result};
use ffmpeg_next::codec;
use ffmpeg_next::encoder;
use ffmpeg_next::format::Pixel;
use ffmpeg_next::frame;
use ffmpeg_next::util::error::EAGAIN;
pub struct SoftwareEncoder {
encoder: encoder::Encoder,
width: u32,
height: u32,
fps: u32,
bitrate: u64,
pts: i64,
}
impl SoftwareEncoder {
pub fn new(width: u32, height: u32, fps: u32) -> Self {
Self { width, height, fps }
/// Create a new software H.264 encoder.
///
/// * `width`, `height` — frame dimensions
/// * `fps` — target frame rate
/// * `bitrate` — target bitrate in bps (e.g. 2_000_000 for 2 Mbps)
pub fn new(width: u32, height: u32, fps: u32, bitrate: u64) -> Result<Self> {
ffmpeg_next::init().context("ffmpeg init")?;
let codec = codec::encoder::find_by_name("libx264")
.or_else(|| codec::encoder::find(codec::Id::H264))
.context("H.264 encoder not found (install libx264)")?;
let mut encoder = encoder::Encoder::new(codec)?;
let ctx = encoder.codec();
ctx.set_width(width as i32);
ctx.set_height(height as i32);
ctx.set_pixel_format(Pixel::YUV420P);
ctx.set_time_base((1, fps as i32));
ctx.set_frame_rate(Some((fps as i32, 1)));
ctx.set_bit_rate(bitrate as i64);
ctx.set_gop_size(fps as i32 * 2); // 2-second GOP
ctx.set_max_b_frames(0); // No B-frames for low latency
// Set profile & tune for low-latency
ctx.set_profile(codec::Profile::H264::Main);
let opts = [
("preset", "veryfast"),
("tune", "zerolatency"),
("profile", "main"),
];
for (k, v) in &opts {
ctx.set_option(k, *v);
}
let encoder = encoder.open_as(codec)?;
Ok(Self {
encoder,
width,
height,
fps,
bitrate,
pts: 0,
})
}
/// Encode a raw RGBA frame to H.264 NAL unit(s)
pub fn encode(&mut self, _frame: &[u8]) -> Result<Vec<Vec<u8>>> {
// TODO: implement using ffmpeg-next
Ok(Vec::new())
/// Encode a raw RGBA frame into zero or more H.264 NAL units (Annex B).
///
/// Returns a list of NAL units (each prefixed with the Annex-B start code).
/// May return an empty vec when the encoder needs more data before producing output.
pub fn encode(&mut self, rgba_data: &[u8]) -> Result<Vec<Vec<u8>>> {
// Convert RGBA → YUV420P
let rgb_frame = self.rgba_to_yuv420(rgba_data)?;
// Send frame to encoder
self.encoder
.send_frame(&rgb_frame)
.context("send_frame failed")?;
// Receive packets
let mut nals = Vec::new();
let mut packet = ffmpeg_next::Packet::empty();
while self.encoder.receive_packet(&mut packet).is_ok() {
if !packet.is_empty() {
// Copy packet data (Annex B format already)
nals.push(packet.data().to_vec());
}
packet = ffmpeg_next::Packet::empty();
}
self.pts += 1;
Ok(nals)
}
/// Flush remaining frames from the encoder.
pub fn flush(&mut self) -> Result<Vec<Vec<u8>>> {
self.encoder.send_eof().context("send_eof failed")?;
let mut nals = Vec::new();
let mut packet = ffmpeg_next::Packet::empty();
while self.encoder.receive_packet(&mut packet).is_ok() {
if !packet.is_empty() {
nals.push(packet.data().to_vec());
}
packet = ffmpeg_next::Packet::empty();
}
Ok(nals)
}
pub fn resolution(&self) -> (u32, u32) {
(self.width, self.height)
}
/// Convert raw RGBA pixel data to an ffmpeg YUV420P frame.
fn rgba_to_yuv420(&self, rgba: &[u8]) -> Result<frame::Video> {
use ffmpeg_next::software::converter::Converter;
let expected = (self.width * self.height * 4) as usize;
anyhow::ensure!(
rgba.len() >= expected,
"RGBA data too short: {} < {}",
rgba.len(),
expected
);
// Create an RGBA frame
let mut src_frame = frame::Video::new(Pixel::RGBA, self.width as i32, self.height as i32);
src_frame.data_mut(0)[..rgba.len()].copy_from_slice(rgba);
src_frame.set_pts(Some(self.pts));
// Convert RGBA → YUV420P
let mut converter = Converter::new(
self.width as i32,
self.height as i32,
Pixel::RGBA,
Pixel::YUV420P,
)?;
let yuv_frame = converter.run(&src_frame)?;
Ok(yuv_frame)
}
}

View File

@@ -19,7 +19,6 @@ fn get_version() -> String {
#[tauri::command]
async fn discover_devices() -> Result<Vec<String>, String> {
let mut discovery = mdns::MdnsDiscovery::new().map_err(|e| e.to_string())?;
// mDNS browsing is blocking; run on blocking thread pool.
let devices = tokio::task::spawn_blocking(move || discovery.discover())
.await
.map_err(|e| e.to_string())?
@@ -38,11 +37,27 @@ async fn connect_adp(addr: String) -> Result<String, String> {
Ok(format!("Connected to {} via ADP", addr))
}
/// Send an H.264 NAL unit to the connected ADP server.
/// Start screen capture at specified FPS.
#[tauri::command]
async fn send_video_frame(addr: String, _nal_b64: String) -> Result<String, String> {
// TODO: implement sending H.264 NAL via AdpClient
Ok(format!("Frame sent to {}", addr))
async fn start_capture(fps: u32) -> Result<String, String> {
let mut capturer = capture::create_capturer(fps).map_err(|e| e.to_string())?;
let (w, h) = capturer.resolution();
Ok(format!("Capture started: {}x{} @ {}fps", w, h, fps))
}
/// Get current display resolution.
#[tauri::command]
async fn get_display_resolution() -> Result<(u32, u32), String> {
// TODO: query actual display resolution
Ok((1920, 1080))
}
/// Initialize the H.264 encoder with given params.
#[tauri::command]
async fn init_encoder(width: u32, height: u32, fps: u32, bitrate: u64) -> Result<String, String> {
let _enc = encoder::SoftwareEncoder::new(width, height, fps, bitrate)
.map_err(|e| format!("Encoder init failed: {}", e))?;
Ok(format!("Encoder initialized: {}x{} @ {}fps, {}bps", width, height, fps, bitrate))
}
pub fn run() {
@@ -55,7 +70,9 @@ pub fn run() {
get_version,
discover_devices,
connect_adp,
send_video_frame,
start_capture,
get_display_resolution,
init_encoder,
])
.setup(|app| {
let window = app.get_webview_window("main").unwrap();