feat(clipboard): support readImage & writeImage (#845)

* feat(clipboard): support `read_image` & `write_image`

* fix plugin name

* platform specific bahavior

* remove unnecessary BufWriter

* improvement

* update example

* update example

* format

* header, fix change file

* use image from tauri

* fix ci

* update tauri, fix read

* image crate only on desktop [skip ci]

* Update plugins/authenticator/src/u2f_crate/protocol.rs [skip ci]

Co-authored-by: Amr Bashir <amr.bashir2015@gmail.com>

* Update plugins/authenticator/src/u2f_crate/protocol.rs [skip ci]

Co-authored-by: Amr Bashir <amr.bashir2015@gmail.com>

* update deps, address code review

* fix mobile [skip ci]

---------

Co-authored-by: Lucas Nogueira <lucas@tauri.studio>
Co-authored-by: Lucas Nogueira <lucas@tauri.app>
Co-authored-by: Amr Bashir <amr.bashir2015@gmail.com>

plugins/clipboard-manager/src/api-iife.js

@@ -1 +1 @@
-if("__TAURI__"in window){var __TAURI_PLUGIN_CLIPBOARDMANAGER__=function(n){"use strict";async function r(n,r={},a){return window.__TAURI_INTERNALS__.invoke(n,r,a)}return"function"==typeof SuppressedError&&SuppressedError,n.clear=async function(){await r("plugin:clipboard-manager|clear")},n.readText=async function(){return(await r("plugin:clipboard-manager|read")).plainText.text},n.writeHtml=async function(n,a){return r("plugin:clipboard-manager|write_html",{data:{html:{html:n,altHtml:a}}})},n.writeText=async function(n,a){return r("plugin:clipboard-manager|write",{data:{plainText:{label:a?.label,text:n}}})},n}({});Object.defineProperty(window.__TAURI__,"clipboardManager",{value:__TAURI_PLUGIN_CLIPBOARDMANAGER__})}
+if("__TAURI__"in window){var __TAURI_PLUGIN_CLIPBOARDMANAGER__=function(e){"use strict";var r;async function t(e,r={},t){return window.__TAURI_INTERNALS__.invoke(e,r,t)}"function"==typeof SuppressedError&&SuppressedError;class n{get rid(){return function(e,r,t,n){if("a"===t&&!n)throw new TypeError("Private accessor was defined without a getter");if("function"==typeof r?e!==r||!n:!r.has(e))throw new TypeError("Cannot read private member from an object whose class did not declare it");return"m"===t?n:"a"===t?n.call(e):n?n.value:r.get(e)}(this,r,"f")}constructor(e){r.set(this,void 0),function(e,r,t,n,a){if("m"===n)throw new TypeError("Private method is not writable");if("a"===n&&!a)throw new TypeError("Private accessor was defined without a setter");if("function"==typeof r?e!==r||!a:!r.has(e))throw new TypeError("Cannot write private member to an object whose class did not declare it");"a"===n?a.call(e,t):a?a.value=t:r.set(e,t)}(this,r,e,"f")}async close(){return t("plugin:resources|close",{rid:this.rid})}}r=new WeakMap;class a extends n{constructor(e){super(e)}static async new(e,r,n){return t("plugin:image|new",{rgba:i(e),width:r,height:n}).then((e=>new a(e)))}static async fromBytes(e){return t("plugin:image|from_bytes",{bytes:i(e)}).then((e=>new a(e)))}static async fromPath(e){return t("plugin:image|from_path",{path:e}).then((e=>new a(e)))}async rgba(){return t("plugin:image|rgba",{rid:this.rid}).then((e=>new Uint8Array(e)))}async size(){return t("plugin:image|size",{rid:this.rid})}}function i(e){return null==e?null:"string"==typeof e?e:e instanceof Uint8Array?Array.from(e):e instanceof ArrayBuffer?Array.from(new Uint8Array(e)):e instanceof a?e.rid:e}return e.clear=async function(){await t("plugin:clipboard-manager|clear")},e.readImage=async function(){return await t("plugin:clipboard-manager|read_image").then((e=>new a(e)))},e.readText=async function(){return(await t("plugin:clipboard-manager|read_text")).plainText.text},e.writeHtml=async function(e,r){return t("plugin:clipboard-manager|write_html",{data:{html:{html:e,altHtml:r}}})},e.writeImage=async function(e){return t("plugin:clipboard-manager|write_image",{data:{image:{image:i(e)}}})},e.writeText=async function(e,r){return t("plugin:clipboard-manager|write_text",{data:{plainText:{label:r?.label,text:e}}})},e}({});Object.defineProperty(window.__TAURI__,"clipboardManager",{value:__TAURI_PLUGIN_CLIPBOARDMANAGER__})}

plugins/clipboard-manager/src/commands.rs

@@ -2,25 +2,45 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-License-Identifier: MIT
 
-use tauri::{command, AppHandle, Runtime, State};
+use tauri::{command, AppHandle, Manager, ResourceId, Runtime, State};
 
 use crate::{ClipKind, Clipboard, ClipboardContents, Result};
 
 #[command]
-pub(crate) async fn write<R: Runtime>(
+pub(crate) async fn write_text<R: Runtime>(
     _app: AppHandle<R>,
     clipboard: State<'_, Clipboard<R>>,
     data: ClipKind,
 ) -> Result<()> {
-    clipboard.write(data)
+    clipboard.write_text(data)
 }
 
 #[command]
-pub(crate) async fn read<R: Runtime>(
+pub(crate) async fn write_image<R: Runtime>(
+    _app: AppHandle<R>,
+    clipboard: State<'_, Clipboard<R>>,
+    data: ClipKind,
+) -> Result<()> {
+    clipboard.write_image(data)
+}
+
+#[command]
+pub(crate) async fn read_text<R: Runtime>(
     _app: AppHandle<R>,
     clipboard: State<'_, Clipboard<R>>,
 ) -> Result<ClipboardContents> {
-    clipboard.read()
+    clipboard.read_text()
+}
+
+#[command]
+pub(crate) async fn read_image<R: Runtime>(
+    app: AppHandle<R>,
+    clipboard: State<'_, Clipboard<R>>,
+) -> Result<ResourceId> {
+    let image = clipboard.read_image()?.to_owned();
+    let mut resources_table = app.resources_table();
+    let rid = resources_table.add(image);
+    Ok(rid)
 }
 
 #[command]

plugins/clipboard-manager/src/desktop.rs

@@ -2,12 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-License-Identifier: MIT
 
+use arboard::ImageData;
+use image::ImageEncoder;
 use serde::de::DeserializeOwned;
-use tauri::{plugin::PluginApi, AppHandle, Runtime};
+use tauri::{image::Image, plugin::PluginApi, AppHandle, Runtime};
 
 use crate::models::*;
 
-use std::sync::Mutex;
+use std::{borrow::Cow, sync::Mutex};
 
 pub fn init<R: Runtime, C: DeserializeOwned>(
     app: &AppHandle<R>,
@@ -27,17 +29,38 @@ pub struct Clipboard<R: Runtime> {
 }
 
 impl<R: Runtime> Clipboard<R> {
-    pub fn write(&self, kind: ClipKind) -> crate::Result<()> {
+    pub fn write_text(&self, kind: ClipKind) -> crate::Result<()> {
         match kind {
             ClipKind::PlainText { text, .. } => match &self.clipboard {
                 Ok(clipboard) => clipboard.lock().unwrap().set_text(text).map_err(Into::into),
                 Err(e) => Err(crate::Error::Clipboard(e.to_string())),
             },
-            _ => Err(crate::Error::Clipboard("Invalid clip kind!".to_string())),
+            _ => Err(crate::Error::Clipboard("Invalid clip kind".to_string())),
         }
     }
 
-    pub fn read(&self) -> crate::Result<ClipboardContents> {
+    pub fn write_image(&self, kind: ClipKind) -> crate::Result<()> {
+        match kind {
+            ClipKind::Image { image, .. } => match &self.clipboard {
+                Ok(clipboard) => {
+                    let image = image.into_img(&self.app)?;
+                    clipboard
+                        .lock()
+                        .unwrap()
+                        .set_image(ImageData {
+                            bytes: Cow::Borrowed(image.rgba()),
+                            width: image.width() as usize,
+                            height: image.height() as usize,
+                        })
+                        .map_err(Into::into)
+                }
+                Err(e) => Err(crate::Error::Clipboard(e.to_string())),
+            },
+            _ => Err(crate::Error::Clipboard("Invalid clip kind".to_string())),
+        }
+    }
+
+    pub fn read_text(&self) -> crate::Result<ClipboardContents> {
         match &self.clipboard {
             Ok(clipboard) => {
                 let text = clipboard.lock().unwrap().get_text()?;
@@ -67,4 +90,24 @@ impl<R: Runtime> Clipboard<R> {
             Err(e) => Err(crate::Error::Clipboard(e.to_string())),
         }
     }
+
+    pub fn read_image(&self) -> crate::Result<Image<'_>> {
+        match &self.clipboard {
+            Ok(clipboard) => {
+                let image = clipboard.lock().unwrap().get_image()?;
+
+                let mut buffer: Vec<u8> = Vec::new();
+                image::codecs::png::PngEncoder::new(&mut buffer).write_image(
+                    &image.bytes,
+                    image.width as u32,
+                    image.height as u32,
+                    image::ColorType::Rgba8,
+                )?;
+
+                let image = Image::new_owned(buffer, image.width as u32, image.height as u32);
+                Ok(image)
+            }
+            Err(e) => Err(crate::Error::Clipboard(e.to_string())),
+        }
+    }
 }

plugins/clipboard-manager/src/error.rs

@@ -13,6 +13,12 @@ pub enum Error {
     PluginInvoke(#[from] tauri::plugin::mobile::PluginInvokeError),
     #[error("{0}")]
     Clipboard(String),
+    #[cfg(desktop)]
+    #[error(transparent)]
+    Tauri(#[from] tauri::Error),
+    #[cfg(desktop)]
+    #[error("invalid image: {0}")]
+    Image(#[from] image::ImageError),
 }
 
 impl Serialize for Error {

plugins/clipboard-manager/src/lib.rs

@@ -50,11 +50,11 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
     Builder::new("clipboard-manager")
         .js_init_script(include_str!("api-iife.js").to_string())
         .invoke_handler(tauri::generate_handler![
-            commands::write,
-            commands::read,
-            #[cfg(desktop)]
+            commands::write_text,
+            commands::read_text,
+            commands::read_image,
+            commands::write_image,
             commands::write_html,
-            #[cfg(desktop)]
             commands::clear
         ])
         .setup(|app, api| {

plugins/clipboard-manager/src/mobile.rs

@@ -4,6 +4,7 @@
 
 use serde::de::DeserializeOwned;
 use tauri::{
+    image::Image,
     plugin::{PluginApi, PluginHandle},
     AppHandle, Runtime,
 };
@@ -32,14 +33,26 @@ pub fn init<R: Runtime, C: DeserializeOwned>(
 pub struct Clipboard<R: Runtime>(PluginHandle<R>);
 
 impl<R: Runtime> Clipboard<R> {
-    pub fn write(&self, kind: ClipKind) -> crate::Result<()> {
+    pub fn write_text(&self, kind: ClipKind) -> crate::Result<()> {
         self.0.run_mobile_plugin("write", kind).map_err(Into::into)
     }
 
-    pub fn read(&self) -> crate::Result<ClipboardContents> {
+    pub fn write_image(&self, kind: ClipKind) -> crate::Result<()> {
+        Err(crate::Error::Clipboard(
+            "Unsupported on this platform".to_string(),
+        ))
+    }
+
+    pub fn read_text(&self) -> crate::Result<ClipboardContents> {
         self.0.run_mobile_plugin("read", ()).map_err(Into::into)
     }
 
+    pub fn read_image(&self) -> crate::Result<Image<'_>> {
+        Err(crate::Error::Clipboard(
+            "Unsupported on this platform".to_string(),
+        ))
+    }
+
     // Treat HTML as unsupported on mobile until tested
     pub fn write_html(&self, _kind: ClipKind) -> crate::Result<()> {
         Err(crate::Error::Clipboard(

plugins/clipboard-manager/src/models.rs

@@ -4,13 +4,18 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Deserialize)]
+#[cfg_attr(mobile, derive(Serialize))]
 #[serde(rename_all = "camelCase")]
 pub enum ClipKind {
     PlainText {
         label: Option<String>,
         text: String,
     },
+    #[cfg(desktop)]
+    Image {
+        image: tauri::image::JsImage,
+    },
     Html {
         html: String,
         alt_html: Option<String>,
@@ -20,5 +25,12 @@ pub enum ClipKind {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ClipboardContents {
-    PlainText { text: String },
+    PlainText {
+        text: String,
+    },
+    Image {
+        bytes: Vec<u8>,
+        width: usize,
+        height: usize,
+    },
 }