gestura_core_tools/
screen_async.rs

1//! Async screen capture operations for pipeline integration
2//!
3//! Wraps the synchronous [`ScreenTools`] via
4//! `tokio::task::spawn_blocking` for use in async contexts (pipeline, GUI).
5
6use crate::error::{AppError, Result};
7use crate::screen::{CaptureRegion, ScreenTools, ScreenshotResult};
8use base64::Engine;
9use serde::Serialize;
10use std::path::Path;
11
12#[derive(Debug, Serialize)]
13struct ScreenshotOutput {
14    path: String,
15    width: Option<u32>,
16    height: Option<u32>,
17    format: String,
18    mime_type: String,
19    timestamp: chrono::DateTime<chrono::Utc>,
20    file_size_bytes: u64,
21
22    #[serde(default, skip_serializing_if = "Option::is_none")]
23    inline_base64: Option<String>,
24    #[serde(default, skip_serializing_if = "Option::is_none")]
25    inline_mime_type: Option<String>,
26    #[serde(default, skip_serializing_if = "Option::is_none")]
27    inline_kind: Option<String>,
28}
29
30#[derive(Debug, Serialize)]
31struct RecordingStartOutput {
32    recording_id: String,
33    output_path: String,
34    started_at: chrono::DateTime<chrono::Utc>,
35}
36
37#[derive(Debug, Serialize)]
38struct RecordingStopOutput {
39    recording_id: String,
40    path: String,
41    duration_secs: f64,
42    file_size_bytes: u64,
43    format: String,
44}
45
46/// How the screenshot tool should return its result.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum ScreenshotReturnMode {
49    /// Return metadata + file path only.
50    Path,
51    /// Return metadata + file path + a bounded inline base64 payload.
52    InlineBase64,
53}
54
55/// Options for bounded inline screenshot payloads.
56#[derive(Debug, Clone)]
57pub struct ScreenshotInlineOptions {
58    /// Max width for an inline thumbnail (pixels). If `None`, no resize is attempted.
59    pub max_width: Option<u32>,
60    /// Max height for an inline thumbnail (pixels). If `None`, no resize is attempted.
61    pub max_height: Option<u32>,
62    /// Maximum base64 character length for the inline payload.
63    pub max_base64_chars: usize,
64    /// Maximum serialized JSON length for the tool output (to avoid pipeline truncation).
65    pub max_result_chars: usize,
66}
67
68impl Default for ScreenshotInlineOptions {
69    fn default() -> Self {
70        Self {
71            max_width: Some(128),
72            max_height: Some(128),
73            max_base64_chars: 1400,
74            max_result_chars: 1800,
75        }
76    }
77}
78
79#[derive(Debug, Clone)]
80pub struct ScreenshotReturnOptions {
81    pub mode: ScreenshotReturnMode,
82    pub inline: ScreenshotInlineOptions,
83}
84
85impl Default for ScreenshotReturnOptions {
86    fn default() -> Self {
87        Self {
88            mode: ScreenshotReturnMode::Path,
89            inline: ScreenshotInlineOptions::default(),
90        }
91    }
92}
93
94fn mime_type_for_ext(ext: &str) -> String {
95    match ext.to_ascii_lowercase().as_str() {
96        "png" => "image/png".to_string(),
97        "jpg" | "jpeg" => "image/jpeg".to_string(),
98        "gif" => "image/gif".to_string(),
99        _ => "application/octet-stream".to_string(),
100    }
101}
102
103fn screenshot_output_json(
104    result: ScreenshotResult,
105    options: ScreenshotReturnOptions,
106) -> Result<String> {
107    let path = result.path.clone();
108    let ext = path
109        .extension()
110        .and_then(|e| e.to_str())
111        .unwrap_or("png")
112        .to_string();
113    let mime_type = mime_type_for_ext(&ext);
114
115    let mut out = ScreenshotOutput {
116        path: result.path.display().to_string(),
117        width: result.width,
118        height: result.height,
119        format: result.format,
120        mime_type,
121        timestamp: result.timestamp,
122        file_size_bytes: result.file_size_bytes,
123        inline_base64: None,
124        inline_mime_type: None,
125        inline_kind: None,
126    };
127
128    if options.mode == ScreenshotReturnMode::InlineBase64 {
129        let (b64, inline_mime, kind) = encode_inline_screenshot(&path, &options.inline)?;
130        out.inline_base64 = Some(b64);
131        out.inline_mime_type = Some(inline_mime);
132        out.inline_kind = Some(kind);
133
134        let max_result_chars = options.inline.max_result_chars.min(2000);
135        let json = serde_json::to_string(&out)?;
136        if json.len() > max_result_chars {
137            return Err(AppError::Io(std::io::Error::other(format!(
138                "Inline screenshot tool output is too large ({} chars; max {}). Reduce inline max_width/max_height/max_base64_chars.",
139                json.len(),
140                max_result_chars
141            ))));
142        }
143        return Ok(json);
144    }
145
146    Ok(serde_json::to_string_pretty(&out)?)
147}
148
149fn encode_inline_screenshot(
150    path: &Path,
151    inline: &ScreenshotInlineOptions,
152) -> Result<(String, String, String)> {
153    const HARD_MAX_BASE64_CHARS: usize = 1700;
154    const HARD_MAX_RESULT_CHARS: usize = 2000;
155    /// Minimum dimension we'll shrink to before giving up.
156    const MIN_THUMB_DIM: u32 = 16;
157
158    let max_base64_chars = inline.max_base64_chars.min(HARD_MAX_BASE64_CHARS);
159    let max_result_chars = inline.max_result_chars.min(HARD_MAX_RESULT_CHARS);
160    if max_base64_chars < 64 {
161        return Err(AppError::Io(std::io::Error::other(
162            "inline.max_base64_chars must be >= 64",
163        )));
164    }
165    if max_result_chars < 256 {
166        return Err(AppError::Io(std::io::Error::other(
167            "inline.max_result_chars must be >= 256",
168        )));
169    }
170
171    let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
172    let ext_lc = ext.to_ascii_lowercase();
173
174    // For image formats we can decode, iteratively resize + encode as JPEG
175    // (much smaller than PNG for photographic content like screenshots) until
176    // the base64 payload fits within the budget.
177    if matches!(ext_lc.as_str(), "png" | "jpg" | "jpeg")
178        && let Ok(img) = image::open(path)
179    {
180        let mut w = inline.max_width.unwrap_or(img.width());
181        let mut h = inline.max_height.unwrap_or(img.height());
182
183        loop {
184            let thumb = img.thumbnail(w, h);
185
186            // Encode as JPEG at quality 60 – much more compact than PNG for
187            // real-world screenshots.
188            let mut buf = Vec::new();
189            let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 60);
190            thumb.write_with_encoder(encoder).map_err(|e| {
191                AppError::Io(std::io::Error::other(format!(
192                    "Failed to encode inline JPEG thumbnail: {e}"
193                )))
194            })?;
195
196            let b64 = base64::engine::general_purpose::STANDARD.encode(&buf);
197            if b64.len() <= max_base64_chars {
198                return Ok((b64, "image/jpeg".to_string(), "thumbnail_jpeg".to_string()));
199            }
200
201            // Halve dimensions and retry.
202            w = (w / 2).max(MIN_THUMB_DIM);
203            h = (h / 2).max(MIN_THUMB_DIM);
204
205            if w <= MIN_THUMB_DIM && h <= MIN_THUMB_DIM {
206                // Even at minimum size it doesn't fit – give up.
207                return Err(AppError::Io(std::io::Error::other(format!(
208                    "Inline base64 thumbnail too large even at {}×{} ({} chars; max {}). \
209                     Increase max_base64_chars or use return.mode='path'.",
210                    w,
211                    h,
212                    b64.len(),
213                    max_base64_chars
214                ))));
215            }
216        }
217    }
218
219    // Fallback: base64 the raw file bytes (no resize).
220    let bytes = std::fs::read(path)?;
221    let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
222    if b64.len() > max_base64_chars {
223        return Err(AppError::Io(std::io::Error::other(format!(
224            "Inline base64 file payload too large ({} chars; max {}). \
225             Use a smaller capture region, save as PNG/JPG, or use return.mode='path'.",
226            b64.len(),
227            max_base64_chars
228        ))));
229    }
230
231    let mime = mime_type_for_ext(ext);
232    Ok((b64, mime, "raw_file".to_string()))
233}
234
235/// Capture a screenshot asynchronously.
236pub async fn screenshot(
237    output_path: &str,
238    region: Option<(u32, u32, u32, u32)>,
239    display: Option<u32>,
240) -> Result<String> {
241    screenshot_with_options(
242        output_path,
243        region,
244        display,
245        ScreenshotReturnOptions::default(),
246    )
247    .await
248}
249
250/// Capture a screenshot asynchronously with configurable return options.
251pub async fn screenshot_with_options(
252    output_path: &str,
253    region: Option<(u32, u32, u32, u32)>,
254    display: Option<u32>,
255    options: ScreenshotReturnOptions,
256) -> Result<String> {
257    let path = output_path.to_string();
258    let region_opt = region.map(|(x, y, w, h)| CaptureRegion {
259        x,
260        y,
261        width: w,
262        height: h,
263    });
264
265    tokio::task::spawn_blocking(move || {
266        let tools = ScreenTools::new();
267        let result = tools.screenshot(std::path::Path::new(&path), region_opt, display)?;
268
269        screenshot_output_json(result, options)
270    })
271    .await
272    .map_err(|e| AppError::Io(std::io::Error::other(format!("Task join error: {}", e))))?
273}
274
275/// Start screen recording asynchronously.
276pub async fn start_recording(
277    output_path: &str,
278    region: Option<(u32, u32, u32, u32)>,
279    display: Option<u32>,
280) -> Result<String> {
281    let path = output_path.to_string();
282    let region_opt = region.map(|(x, y, w, h)| CaptureRegion {
283        x,
284        y,
285        width: w,
286        height: h,
287    });
288
289    tokio::task::spawn_blocking(move || {
290        let tools = ScreenTools::new();
291        let result = tools.start_recording(std::path::Path::new(&path), region_opt, display)?;
292
293        let output = RecordingStartOutput {
294            recording_id: result.recording_id,
295            output_path: result.output_path.display().to_string(),
296            started_at: result.started_at,
297        };
298
299        Ok(serde_json::to_string_pretty(&output)?)
300    })
301    .await
302    .map_err(|e| AppError::Io(std::io::Error::other(format!("Task join error: {}", e))))?
303}
304
305/// Stop screen recording asynchronously.
306pub async fn stop_recording(recording_id: &str) -> Result<String> {
307    let id = recording_id.to_string();
308
309    tokio::task::spawn_blocking(move || {
310        let tools = ScreenTools::new();
311        let result = tools.stop_recording(&id)?;
312
313        let output = RecordingStopOutput {
314            recording_id: result.recording_id,
315            path: result.path.display().to_string(),
316            duration_secs: result.duration_secs,
317            file_size_bytes: result.file_size_bytes,
318            format: result.format,
319        };
320
321        Ok(serde_json::to_string_pretty(&output)?)
322    })
323    .await
324    .map_err(|e| AppError::Io(std::io::Error::other(format!("Task join error: {}", e))))?
325}
326
327#[cfg(test)]
328mod tests {
329    use super::*;
330
331    #[test]
332    fn inline_jpeg_thumbnail_is_bounded_and_decodable() {
333        let dir = tempfile::tempdir().unwrap();
334        let path = dir.path().join("tiny.png");
335
336        // Create a tiny PNG source image.
337        let img = image::RgbaImage::from_fn(32, 32, |x, y| {
338            image::Rgba([(x % 255) as u8, (y % 255) as u8, 0, 255])
339        });
340        img.save_with_format(&path, image::ImageFormat::Png)
341            .unwrap();
342
343        let opts = ScreenshotInlineOptions {
344            max_width: Some(16),
345            max_height: Some(16),
346            max_base64_chars: 1700,
347            max_result_chars: 2000,
348        };
349        let (b64, mime, kind) = encode_inline_screenshot(&path, &opts).unwrap();
350        // Now encodes as JPEG for compact inline thumbnails.
351        assert_eq!(mime, "image/jpeg");
352        assert_eq!(kind, "thumbnail_jpeg");
353        assert!(b64.len() <= opts.max_base64_chars);
354
355        let bytes = base64::engine::general_purpose::STANDARD
356            .decode(b64)
357            .unwrap();
358        // JPEG SOI marker
359        assert!(bytes.starts_with(&[0xFF, 0xD8]));
360    }
361
362    #[test]
363    fn inline_payload_too_small_errors() {
364        let dir = tempfile::tempdir().unwrap();
365        let path = dir.path().join("tiny.png");
366        let img = image::RgbaImage::from_fn(8, 8, |_x, _y| image::Rgba([0, 0, 0, 255]));
367        img.save_with_format(&path, image::ImageFormat::Png)
368            .unwrap();
369
370        let opts = ScreenshotInlineOptions {
371            max_width: Some(8),
372            max_height: Some(8),
373            max_base64_chars: 64,
374            max_result_chars: 2000,
375        };
376        // Force failure with an absurdly low limit.
377        let opts = ScreenshotInlineOptions {
378            max_base64_chars: 10,
379            ..opts
380        };
381        assert!(encode_inline_screenshot(&path, &opts).is_err());
382    }
383}
gestura_core_tools/screen_async.rs

gestura_core_tools/
screen_async.rs
