- Implement first draft of async streamed frame capturing for macos

- change the capturing to use `Recorder` that as a async stream api
- introduce `Frame` that holds the image and the `Instant` when captured
- change some args to work with `impl AsRef<>` in order to avoid tight type coupling

Author: sassman

Cargo.lock

@@ -27,6 +27,7 @@ env_logger = "0.9.0"
 simplerand = "1.3.0"
 humantime = "2.1.0"
 smol = "1.2.5"
+flume = { version = "0.10", default-features = false }
 
 [dependencies.clap]
 version = "3.1.9"

Cargo.toml

@@ -1,15 +1,17 @@
-use anyhow::{Context, Result};
-use image::save_buffer;
-use image::ColorType::Rgba8;
 use std::borrow::Borrow;
-use std::ops::{Add, Sub};
 use std::sync::mpsc::Receiver;
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result};
+use image::save_buffer;
+use image::ColorType::Rgba8;
+use smol::stream::block_on;
 use tempfile::TempDir;
 
+use crate::common::{Frame, Recorder};
 use crate::utils::{file_name_for, IMG_EXT};
-use crate::{ImageOnHeap, PlatformApi, WindowId};
+use crate::{Image, PlatformApi, WindowId};
 
 /// captures screenshots as file on disk
 /// collects also the timecodes when they have been captured
@@ -22,60 +24,49 @@ pub fn capture_thread(
     tempdir: Arc<Mutex<TempDir>>,
     force_natural: bool,
 ) -> Result<()> {
-    let duration = Duration::from_millis(250);
-    let start = Instant::now();
-    let mut idle_duration = Duration::from_millis(0);
-    let mut last_frame: Option<ImageOnHeap> = None;
+    let recorder = Recorder::new(api, win_id, 4);
+    let mut last_frame: Option<Frame> = None;
     let mut identical_frames = 0;
-    let mut last_now = Instant::now();
-    loop {
-        // blocks for a timeout
-        if rx.recv_timeout(duration).is_ok() {
-            break;
-        }
-        let now = Instant::now();
-        let effective_now = now.sub(idle_duration);
-        let tc = effective_now.saturating_duration_since(start).as_millis();
-        let image = api.capture_window_screenshot(win_id)?;
+    let start = Instant::now();
+
+    for frame in block_on(recorder) {
         if !force_natural {
-            if last_frame.is_some()
-                && image
-                    .samples
-                    .as_slice()
-                    .eq(last_frame.as_ref().unwrap().samples.as_slice())
-            {
-                identical_frames += 1;
-            } else {
-                identical_frames = 0;
+            let image: &Image = frame.as_ref();
+            if let Some(last_image) = last_frame.as_ref() {
+                let last_image: &Image = last_image.as_ref();
+                if image.samples.as_slice().eq(last_image.samples.as_slice()) {
+                    identical_frames += 1;
+                } else {
+                    identical_frames = 0;
+                }
             }
         }
 
-        if identical_frames > 0 {
-            // let's track now the duration as idle
-            idle_duration = idle_duration.add(now.duration_since(last_now));
-        } else {
-            if let Err(e) = save_frame(&image, tc, tempdir.lock().unwrap().borrow(), file_name_for)
-            {
-                eprintln!("{}", &e);
-                return Err(e);
-            }
+        if identical_frames == 0 {
+            let tc: &Instant = frame.as_ref();
+            let tc = tc.duration_since(start).as_millis();
+            save_image(&frame, tc, tempdir.lock().unwrap().borrow(), file_name_for)?;
             time_codes.lock().unwrap().push(tc);
-            last_frame = Some(image);
-            identical_frames = 0;
+            last_frame = Some(frame);
+        }
+
+        // when there is a message we should just stop
+        if rx.recv_timeout(Duration::from_millis(1)).is_ok() {
+            break;
         }
-        last_now = now;
     }
 
     Ok(())
 }
 
 /// saves a frame as a tga file
-pub fn save_frame(
-    image: &ImageOnHeap,
+pub fn save_image(
+    image: impl AsRef<Image>,
     time_code: u128,
     tempdir: &TempDir,
     file_name_for: fn(&u128, &str) -> String,
 ) -> Result<()> {
+    let image = image.as_ref();
     save_buffer(
         tempdir.path().join(file_name_for(&time_code, IMG_EXT)),
         &image.samples,

src/capture.rs

@@ -0,0 +1,49 @@
+use std::time::Instant;
+
+use image::flat::SampleLayout;
+use image::{ColorType, FlatSamples};
+
+use crate::common::image::{convert_bgra_to_rgba, crop, CropMut};
+use crate::{Image, Margin, Result};
+
+pub struct Frame {
+    image: Image,
+    timecode: Instant,
+}
+
+impl Frame {
+    pub fn from_bgra(raw_data: Vec<u8>, channels: u8, width: u32, height: u32) -> Self {
+        let timecode = Instant::now();
+        let mut raw_data = raw_data;
+        convert_bgra_to_rgba(&mut raw_data[..]);
+
+        let color = ColorType::Rgba8;
+        let image = FlatSamples {
+            samples: raw_data,
+            layout: SampleLayout::row_major_packed(channels, width, height),
+            color_hint: Some(color),
+        };
+
+        Self { image, timecode }
+    }
+}
+
+impl AsRef<Image> for Frame {
+    fn as_ref(&self) -> &Image {
+        &self.image
+    }
+}
+
+impl CropMut for Frame {
+    fn crop(&mut self, margin: &Margin) -> Result<()> {
+        self.image = crop(&self, margin)?;
+
+        Ok(())
+    }
+}
+
+impl AsRef<Instant> for Frame {
+    fn as_ref(&self) -> &Instant {
+        &self.timecode
+    }
+}

src/common/frame.rs

@@ -6,8 +6,8 @@ use image::{GenericImageView, Rgba};
 /// this helps to identify outer transparent regions
 /// since some backends provides transparency either from a compositor effect like drop shadow on ubuntu / GNOME
 /// or some strange right side strip on MacOS
-pub fn identify_transparency(image: Image) -> Result<Option<Margin>> {
-    let image: View<_, Rgba<u8>> = image.as_view()?;
+pub fn identify_transparency(image: impl AsRef<Image>) -> Result<Option<Margin>> {
+    let image: View<_, Rgba<u8>> = image.as_ref().as_view()?;
     let (width, height) = image.dimensions();
     let half_width = width / 2;
     let half_height = height / 2;

src/common/identify_transparency.rs

@@ -1,14 +1,19 @@
-use crate::common::Margin;
-use crate::{Image, ImageOnHeap, Result};
 use image::flat::View;
 use image::{imageops, GenericImageView, ImageBuffer, Rgba};
 
+use crate::common::Margin;
+use crate::{Image, ImageOnHeap, Result};
+
+pub trait CropMut {
+    fn crop(&mut self, margin: &Margin) -> Result<()>;
+}
+
 ///
 /// specialized version of crop for [`ImageOnHeap`] and [`Margin`]
 ///
 #[cfg_attr(not(macos), allow(dead_code))]
-pub fn crop(image: Image, margin: &Margin) -> Result<ImageOnHeap> {
-    let mut img2: View<_, Rgba<u8>> = image.as_view()?;
+pub fn crop(image: impl AsRef<Image>, margin: &Margin) -> Result<ImageOnHeap> {
+    let mut img2: View<_, Rgba<u8>> = image.as_ref().as_view()?;
     let (width, height) = (
         img2.width() - (margin.left + margin.right) as u32,
         img2.height() - (margin.top + margin.bottom) as u32,
@@ -28,7 +33,7 @@ pub fn crop(image: Image, margin: &Margin) -> Result<ImageOnHeap> {
         }
     }
 
-    Ok(Box::new(buf.into_flat_samples()))
+    Ok(buf.into_flat_samples())
 }
 
 ///
@@ -40,19 +45,20 @@ pub fn convert_bgra_to_rgba(buffer: &mut [u8]) {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use image::open;
 
+    use super::*;
+
     #[test]
     fn should_crop() -> Result<()> {
         // given
         let image_org = open("tests/frames/frame-macos-right-side-issue.tga")?;
         let image = image_org.into_rgba8();
-        let image_raw = ImageOnHeap::new(image.into_flat_samples());
+        let image_raw = image.into_flat_samples();
         let (width, height) = (image_raw.layout.width, image_raw.layout.height);
 
         // when
-        let cropped = crop(*image_raw, &Margin::new(1, 1, 1, 1))?;
+        let cropped = crop(image_raw, &Margin::new(1, 1, 1, 1))?;
 
         // then
         assert_eq!(cropped.layout.width, width - 2);

src/common/image.rs

@@ -2,10 +2,12 @@ pub mod identify_transparency;
 pub mod image;
 pub mod utils;
 
+mod frame;
 mod margin;
 mod platform_api;
 mod recorder;
 
+pub use frame::*;
 pub use margin::*;
 pub use platform_api::*;
 pub use recorder::*;

src/common/mod.rs

@@ -1,11 +1,12 @@
-use crate::{ImageOnHeap, Result, WindowId, WindowList};
+use crate::common::Frame;
+use crate::{Result, WindowId, WindowList};
 
 pub trait PlatformApi: Send + Unpin + Sized {
     /// 1. it does check for the screenshot
     /// 2. it checks for transparent margins and configures the api
     ///     to cut them away in further screenshots
     fn calibrate(&mut self, window_id: WindowId) -> Result<()>;
     fn window_list(&self) -> Result<WindowList>;
-    fn capture_window_screenshot(&self, window_id: WindowId) -> Result<ImageOnHeap>;
+    fn capture_window_screenshot(&self, window_id: WindowId) -> Result<Frame>;
     fn get_active_window(&self) -> Result<WindowId>;
 }

src/common/platform_api.rs

@@ -1,18 +1,18 @@
-use crate::{ImageOnHeap, PlatformApi, WindowId};
-
-use smol::future::FutureExt;
 use std::pin::Pin;
 use std::task::{Context, Poll};
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
+use smol::future::FutureExt;
 use smol::stream::Stream;
 use smol::Timer;
 
-struct Recorder<A: PlatformApi> {
+use crate::common::Frame;
+use crate::{PlatformApi, WindowId};
+
+pub struct Recorder<A: PlatformApi> {
     api: A,
     window_id: WindowId,
     timer: Timer,
-    last_frame_timestamp: Option<Instant>,
 }
 
 impl<A: PlatformApi> Recorder<A> {
@@ -22,48 +22,72 @@ impl<A: PlatformApi> Recorder<A> {
             api,
             window_id,
             timer: Timer::interval(fps),
-            last_frame_timestamp: None,
         }
     }
 }
 
 impl<A: PlatformApi> Stream for Recorder<A> {
-    type Item = ImageOnHeap;
+    type Item = Frame;
 
     fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
         let this = self.get_mut();
-        let now = Instant::now();
         match this.timer.poll(_cx) {
-            Poll::Ready(_) => {
-                let d = now.duration_since(this.last_frame_timestamp.unwrap_or(now));
-                dbg!(d);
-                this.last_frame_timestamp.replace(now);
-                Poll::Ready(this.api.capture_window_screenshot(this.window_id).ok())
-            }
+            Poll::Ready(_) => Poll::Ready(this.api.capture_window_screenshot(this.window_id).ok()),
             Poll::Pending => Poll::Pending,
         }
     }
 }
 
 #[cfg(test)]
 #[cfg(target_os = "macos")]
+#[cfg(feature = "e2e_tests")]
 mod tests {
-    use super::*;
-    use crate::macos::*;
     use smol::stream::block_on;
 
+    use crate::macos::*;
+
+    use super::*;
+
     #[test]
     fn should_record_not() {
-        let api = setup().unwrap();
-        let rec = Recorder::new(api, 682, 4);
+        let win = 9123;
+        let mut api = setup().unwrap();
+        api.calibrate(win).unwrap();
+        let rec = Recorder::new(api, win, 10);
 
         let mut i = 0;
         for _img in block_on(rec) {
             i += 1;
-            if i >= 4 {
+            if i >= 5 {
                 println!("Done with testing..");
                 break;
             }
         }
     }
+
+    // #[test]
+    // fn should_queue_frames_for_saving() {
+    //     let win = 9123;
+    //     let mut api = setup().unwrap();
+    //     api.calibrate(win).unwrap();
+    //     let mut rec = Recorder::new(api, win, 10);
+    //
+    //     let future = rec.next();
+    //     {
+    //         let (sender, receiver) = flume::unbounded();
+    //
+    //         // A function that schedules the task when it gets woken up.
+    //         let schedule = move |runnable| sender.send(runnable).unwrap();
+    //
+    //         // Construct a task.
+    //         let (runnable, task) = async_task::spawn(future, schedule);
+    //
+    //         // Push the task into the queue by invoking its schedule function.
+    //         runnable.schedule();
+    //
+    //         for runnable in receiver {
+    //             runnable.run();
+    //         }
+    //     }
+    // }
 }