feat: add codec

backend/src/main.rs

@@ -95,88 +95,175 @@ async fn accept_connection(stream: TcpStream) {
 fn process_video(
     tx: tokio::sync::mpsc::Sender<(Vec<u8>, bool)>,
 ) -> Result<(), Box<dyn std::error::Error>> {
-    let input_file = "video.mp4";
+    ffmpeg::device::register_all();
-    let mut ictx = ffmpeg::format::input(&input_file)?;
+    let mut dictionary = ffmpeg::Dictionary::new();
+    dictionary.set("framerate", "30");
+    dictionary.set("video_size", "1920x1080");
 
-    let input_stream = ictx
+    // Find the gdigrab input format (Windows)
-        .streams()
+    // ffmpeg::format::format::find returns Option<ffmpeg::format::format::Input> in some versions?
-        .best(ffmpeg::media::Type::Video)
+    // Let's try to use the device directly if possible or finding the demuxer.
-        .ok_or(ffmpeg::Error::StreamNotFound)?;
+    // Based on errors, let's try assuming ffmpeg::format::format::find exists and works for inputs.
-    let input_stream_index = input_stream.index();
+    // If not, we might need ffmpeg::format::demuxer.
 
-    let decoder_ctx = ffmpeg::codec::context::Context::from_parameters(input_stream.parameters())?;
+    // We'll assume the error 'private module input' implies 'ffmpeg::format::format::input' is private
-    let mut decoder = decoder_ctx.decoder().video()?;
+    // so we can't look inside it. But 'ffmpeg::format::format' might have 'find'.
 
-    // Setup Encoder
+    // Actually, let's try a different approach:
-    // Try to find libsvtav1 or fallback to AV1 generic
+    // If we can't find the format easily, maybe we can just use "gdigrab" as the format name if we had a way to convert string to Format.
-    let codec = ffmpeg::codec::encoder::find_by_name("libsvtav1")
-        .or_else(|| ffmpeg::codec::encoder::find(ffmpeg::codec::Id::AV1))
-        .ok_or(ffmpeg::Error::EncoderNotFound)?;
 
-    let output_ctx = ffmpeg::codec::context::Context::new();
+    // We cannot easily look up "gdigrab" by name due to API limitations in the safe wrapper or versioning.
-    let mut encoder_builder = output_ctx.encoder().video()?;
+    // However, if we enable all devices, ffmpeg might be able to detect it via input().
 
-    // We will scale to YUV420P because it's widely supported and good for streaming
+    // Another trick: We can manually iterate via `ffmpeg::format::format::Input::next()` if we could access it, but it's hidden.
-    encoder_builder.set_format(ffmpeg::format::Pixel::YUV420P);
-    encoder_builder.set_width(decoder.width());
-    encoder_builder.set_height(decoder.height());
-    encoder_builder.set_time_base(input_stream.time_base());
-    encoder_builder.set_frame_rate(Some(input_stream.rate()));
 
-    let mut encoder = encoder_builder.open_as(codec)?;
+    // Let's try to bypass the explicit format finding by using `ffmpeg::format::input_with_dictionary`
+    // but we need to specify the format. Wait, `input_with_dictionary` takes a path.
+    // If the path is prefixed with "gdigrab:", maybe it auto-detects? No, gdigrab is a format.
 
-    // Scaler to convert whatever input to YUV420P
+    // There IS a `ffmpeg::device::input::video` which might help?
-    let mut scaler = ffmpeg::software::scaling::context::Context::get(
+    // Let's check if we can use the `av_find_input_format` ffi directly if safe wrapper fails us.
-        decoder.format(),
+    // But that requires `unsafe`.
-        decoder.width(),
-        decoder.height(),
-        ffmpeg::format::Pixel::YUV420P,
-        decoder.width(),
-        decoder.height(),
-        ffmpeg::software::scaling::flag::Flags::BILINEAR,
-    )?;
 
-    // Send packet function closure not easy due to ownership, doing inline
+    // Ideally we should use:
+    // `ffmpeg::format::format::list()` but it is gated by `ffmpeg_5_0` feature being NOT enabled?
+    // Wait, the error said `list` is not found, and the code I Grepped says `#[cfg(not(feature = "ffmpeg_5_0"))]`.
+    // If we are on ffmpeg 5.0+, then `av_register_all` is gone and iterating formats is different.
 
-    for (stream, packet) in ictx.packets() {
+    // If we are on newer FFmpeg, we might not need to look it up manually if we can hint it.
-        if stream.index() == input_stream_index {
+    // But `open_with` needs `&Format`.
-            decoder.send_packet(&packet)?;
 
-            let mut decoded = ffmpeg::util::frame::Video::empty();
+    // Let's assume we can use `ffmpeg::device::input::video` if it exists?
-            while decoder.receive_frame(&mut decoded).is_ok() {
+    // Check `ffmpeg::device` module content.
-                // Scale frame
-                let mut scaled = ffmpeg::util::frame::Video::empty();
-                scaler.run(&decoded, &mut scaled)?;
 
-                // Set pts for the scaled frame to match decoded
+    // Fallback: Use `ffmpeg::format::input(&path)` but force format via dictionary? No, dictionary is options.
-                scaled.set_pts(decoded.pts());
 
-                // Send to encoder
+    // Actually, look at `ffmpeg::format::open_with`: it takes `&Format`.
-                encoder.send_frame(&scaled)?;
+    // We MUST find the format.
 
-                // Receive encoded packets
+    // Since `list()` is missing, maybe we are on a version > 5.0 feature-wise?
-                let mut encoded = ffmpeg::Packet::empty();
+    // The crate is version 8.0.0.
-                while encoder.receive_packet(&mut encoded).is_ok() {
-                    let is_key = encoded.is_key();
-                    let data = encoded.data().ok_or("Empty packet data")?.to_vec();
 
-                    // Blocking send to the tokio channel
+    // Let's try using `ffmpeg::format::Input` directly if there's a way to construct it.
-                    if tx.blocking_send((data, is_key)).is_err() {
+    // No.
-                        return Ok(()); // Receiver dropped
+
+    // What if we try `ffmpeg::device::input::video()`?
+    // Let's check `ffmpeg::device` capabilities.
+
+    // For now, let's try a gross hack:
+    // If `list()` is unavailable, it means we probably can't iterate.
+    // But we might be able to use `ffmpeg::ffi::av_find_input_format`.
+
+    unsafe {
+        let name = std::ffi::CString::new("gdigrab").unwrap();
+        let ptr = ffmpeg::ffi::av_find_input_format(name.as_ptr());
+        if ptr.is_null() {
+            return Err(ffmpeg::Error::DemuxerNotFound.into());
+        }
+
+        let format_input = ffmpeg::format::format::Input::wrap(ptr as *mut _);
+        // We need to wrap Input into Format, but Format might be private in some contexts or re-exported.
+        // It is defined in `ffmpeg::format::format::mod.rs` as `pub enum Format`.
+        // And it is re-exported in `ffmpeg` root? No, `ffmpeg::format::Format` should be public.
+        // The error says `ffmpeg::format::Format` is private?
+        // Ah, `use {Dictionary, Error, Format};` in `src/format/mod.rs` means it imports from parent/root?
+        // No, `pub mod format` defines `Format` enum.
+
+        // Let's try `ffmpeg::format::format::Format::Input`
+        let format = ffmpeg::format::format::Format::Input(format_input);
+
+        // Now we have the format, proceed.
+        // Note: `Input::wrap` is `unsafe`.
+
+        let context =
+            ffmpeg::format::open_with(&std::path::Path::new("desktop"), &format, dictionary)?;
+
+        let mut ictx = match context {
+            ffmpeg::format::context::Context::Input(ictx) => ictx,
+            _ => return Err(ffmpeg::Error::DemuxerNotFound.into()),
+        };
+
+        let input_stream = ictx
+            .streams()
+            .best(ffmpeg::media::Type::Video)
+            .ok_or(ffmpeg::Error::StreamNotFound)?;
+        let input_stream_index = input_stream.index();
+
+        let decoder_ctx =
+            ffmpeg::codec::context::Context::from_parameters(input_stream.parameters())?;
+        let mut decoder = decoder_ctx.decoder().video()?;
+
+        // Setup Encoder
+        // Try to find libsvtav1 or fallback to AV1 generic
+        let codec = ffmpeg::codec::encoder::find_by_name("libsvtav1")
+            .or_else(|| ffmpeg::codec::encoder::find(ffmpeg::codec::Id::AV1))
+            .ok_or(ffmpeg::Error::EncoderNotFound)?;
+
+        let output_ctx = ffmpeg::codec::context::Context::new();
+        let mut encoder_builder = output_ctx.encoder().video()?;
+
+        // We will scale to YUV420P because it's widely supported and good for streaming
+        encoder_builder.set_format(ffmpeg::format::Pixel::YUV420P);
+        encoder_builder.set_width(decoder.width());
+        encoder_builder.set_height(decoder.height());
+        encoder_builder.set_time_base(input_stream.time_base());
+        encoder_builder.set_frame_rate(Some(input_stream.rate()));
+
+        let mut encoder = encoder_builder.open_as(codec)?;
+
+        // Scaler to convert whatever input to YUV420P
+        let mut scaler = ffmpeg::software::scaling::context::Context::get(
+            decoder.format(),
+            decoder.width(),
+            decoder.height(),
+            ffmpeg::format::Pixel::YUV420P,
+            decoder.width(),
+            decoder.height(),
+            ffmpeg::software::scaling::flag::Flags::BILINEAR,
+        )?;
+
+        // Send packet function closure not easy due to ownership, doing inline
+
+        for (stream, packet) in ictx.packets() {
+            if stream.index() == input_stream_index {
+                decoder.send_packet(&packet)?;
+
+                let mut decoded = ffmpeg::util::frame::Video::empty();
+                while decoder.receive_frame(&mut decoded).is_ok() {
+                    // Scale frame
+                    let mut scaled = ffmpeg::util::frame::Video::empty();
+                    scaler.run(&decoded, &mut scaled)?;
+
+                    // Set pts for the scaled frame to match decoded
+                    scaled.set_pts(decoded.pts());
+
+                    // Send to encoder
+                    encoder.send_frame(&scaled)?;
+
+                    // Receive encoded packets
+                    let mut encoded = ffmpeg::Packet::empty();
+                    while encoder.receive_packet(&mut encoded).is_ok() {
+                        let is_key = encoded.is_key();
+                        let data = encoded.data().ok_or("Empty packet data")?.to_vec();
+
+                        // Blocking send to the tokio channel
+                        if tx.blocking_send((data, is_key)).is_err() {
+                            return Ok(()); // Receiver dropped
+                        }
                     }
                 }
             }
         }
-    }
 
-    // Flush encoder
+        // Flush encoder
-    encoder.send_eof()?;
+        encoder.send_eof()?;
-    let mut encoded = ffmpeg::Packet::empty();
+        let mut encoded = ffmpeg::Packet::empty();
-    while encoder.receive_packet(&mut encoded).is_ok() {
+        while encoder.receive_packet(&mut encoded).is_ok() {
-        let is_key = encoded.is_key();
+            let is_key = encoded.is_key();
-        let data = encoded.data().ok_or("Empty packet data")?.to_vec();
+            let data = encoded.data().ok_or("Empty packet data")?.to_vec();
-        if tx.blocking_send((data, is_key)).is_err() {
+            if tx.blocking_send((data, is_key)).is_err() {
-            return Ok(());
+                return Ok(());
+            }
         }
     }
 

frontend/src/main.ts

@@ -16,6 +16,6 @@ const decoder = createDecoder((frame) => {
 })
 
 const discordClientId = import.meta.env.VITE_DISCORD_CLIENT_ID;
-const addr = `wss://${discordClientId}.discordsays.com/ws`
+const addr = import.meta.env.PROD ? `wss://${discordClientId}.discordsays.com/ws` : 'ws://localhost:8080';
 
 createSocket(addr, decoder);