feat(fs): improve readTextFile and readTextFileLines performance (#1962)

This commit is contained in:
Amr Bashir authored and GitHub committed 2024-11-21 15:43:51 +02:00
1 parent 5092ea5e89
commit ed981027dd
4 files changed
+116 -65

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
---
"fs": "patch"
"fs-js": "patch"
---
Improve performance of `readTextFile` and `readTextFileLines` APIs
+1 -1
View File
@@ -1 +1 @@
if("__TAURI__"in window){var __TAURI_PLUGIN_FS__=function(t){"use strict";function e(t,e,n,i){if("a"===n&&!i)throw new TypeError("Private accessor was defined without a getter");if("function"==typeof e?t!==e||!i:!e.has(t))throw new TypeError("Cannot read private member from an object whose class did not declare it");return"m"===n?i:"a"===n?i.call(t):i?i.value:e.get(t)}function n(t,e,n,i,o){if("function"==typeof e?t!==e||!o:!e.has(t))throw new TypeError("Cannot write private member to an object whose class did not declare it");return e.set(t,n),n}var i,o,r,a;"function"==typeof SuppressedError&&SuppressedError;const s="__TAURI_TO_IPC_KEY__";class c{constructor(){this.__TAURI_CHANNEL_MARKER__=!0,i.set(this,(()=>{})),o.set(this,0),r.set(this,{}),this.id=function(t,e=!1){return window.__TAURI_INTERNALS__.transformCallback(t,e)}((({message:t,id:a})=>{if(a===e(this,o,"f")){n(this,o,a+1),e(this,i,"f").call(this,t);const s=Object.keys(e(this,r,"f"));if(s.length>0){let t=a+1;for(const n of s.sort()){if(parseInt(n)!==t)break;{const o=e(this,r,"f")[n];delete e(this,r,"f")[n],e(this,i,"f").call(this,o),t+=1}}n(this,o,t)}}else e(this,r,"f")[a.toString()]=t}))}set onmessage(t){n(this,i,t)}get onmessage(){return e(this,i,"f")}[(i=new WeakMap,o=new WeakMap,r=new WeakMap,s)](){return`__CHANNEL__:${this.id}`}toJSON(){return this[s]()}}async function f(t,e={},n){return window.__TAURI_INTERNALS__.invoke(t,e,n)}class l{get rid(){return e(this,a,"f")}constructor(t){a.set(this,void 0),n(this,a,t)}async close(){return f("plugin:resources|close",{rid:this.rid})}}var u,p;function w(t){return{isFile:t.isFile,isDirectory:t.isDirectory,isSymlink:t.isSymlink,size:t.size,mtime:null!==t.mtime?new Date(t.mtime):null,atime:null!==t.atime?new Date(t.atime):null,birthtime:null!==t.birthtime?new Date(t.birthtime):null,readonly:t.readonly,fileAttributes:t.fileAttributes,dev:t.dev,ino:t.ino,mode:t.mode,nlink:t.nlink,uid:t.uid,gid:t.gid,rdev:t.rdev,blksize:t.blksize,blocks:t.blocks}}a=new WeakMap,t.BaseDirectory=void 0,(u=t.BaseDirectory||(t.BaseDirectory={}))[u.Audio=1]="Audio",u[u.Cache=2]="Cache",u[u.Config=3]="Config",u[u.Data=4]="Data",u[u.LocalData=5]="LocalData",u[u.Document=6]="Document",u[u.Download=7]="Download",u[u.Picture=8]="Picture",u[u.Public=9]="Public",u[u.Video=10]="Video",u[u.Resource=11]="Resource",u[u.Temp=12]="Temp",u[u.AppConfig=13]="AppConfig",u[u.AppData=14]="AppData",u[u.AppLocalData=15]="AppLocalData",u[u.AppCache=16]="AppCache",u[u.AppLog=17]="AppLog",u[u.Desktop=18]="Desktop",u[u.Executable=19]="Executable",u[u.Font=20]="Font",u[u.Home=21]="Home",u[u.Runtime=22]="Runtime",u[u.Template=23]="Template",t.SeekMode=void 0,(p=t.SeekMode||(t.SeekMode={}))[p.Start=0]="Start",p[p.Current=1]="Current",p[p.End=2]="End";class h extends l{async read(t){if(0===t.byteLength)return 0;const e=await f("plugin:fs|read",{rid:this.rid,len:t.byteLength}),n=function(t){const e=new Uint8ClampedArray(t),n=e.byteLength;let i=0;for(let t=0;t<n;t++)i*=256,i+=e[t];return i}(e.slice(-8)),i=e instanceof ArrayBuffer?new Uint8Array(e):e;return t.set(i.slice(0,i.length-8)),0===n?null:n}async seek(t,e){return await f("plugin:fs|seek",{rid:this.rid,offset:t,whence:e})}async stat(){return w(await f("plugin:fs|fstat",{rid:this.rid}))}async truncate(t){await f("plugin:fs|ftruncate",{rid:this.rid,len:t})}async write(t){return await f("plugin:fs|write",{rid:this.rid,data:t})}}async function d(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");const n=await f("plugin:fs|open",{path:t instanceof URL?t.toString():t,options:e});return new h(n)}async function y(t){await f("plugin:fs|unwatch",{rid:t})}return t.FileHandle=h,t.copyFile=async function(t,e,n){if(t instanceof URL&&"file:"!==t.protocol||e instanceof URL&&"file:"!==e.protocol)throw new TypeError("Must be a file URL.");await f("plugin:fs|copy_file",{fromPath:t instanceof URL?t.toString():t,toPath:e instanceof URL?e.toString():e,options:n})},t.create=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");const n=await f("plugin:fs|create",{path:t instanceof URL?t.toString():t,options:e});return new h(n)},t.exists=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");return await f("plugin:fs|exists",{path:t instanceof URL?t.toString():t,options:e})},t.lstat=async function(t,e){return w(await f("plugin:fs|lstat",{path:t instanceof URL?t.toString():t,options:e}))},t.mkdir=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");await f("plugin:fs|mkdir",{path:t instanceof URL?t.toString():t,options:e})},t.open=d,t.readDir=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");return await f("plugin:fs|read_dir",{path:t instanceof URL?t.toString():t,options:e})},t.readFile=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.")Line truncated
if("__TAURI__"in window){var __TAURI_PLUGIN_FS__=function(t){"use strict";function e(t,e,n,i){if("a"===n&&!i)throw new TypeError("Private accessor was defined without a getter");if("function"==typeof e?t!==e||!i:!e.has(t))throw new TypeError("Cannot read private member from an object whose class did not declare it");return"m"===n?i:"a"===n?i.call(t):i?i.value:e.get(t)}function n(t,e,n,i,o){if("function"==typeof e?t!==e||!o:!e.has(t))throw new TypeError("Cannot write private member to an object whose class did not declare it");return e.set(t,n),n}var i,o,r,a;"function"==typeof SuppressedError&&SuppressedError;const s="__TAURI_TO_IPC_KEY__";class c{constructor(){this.__TAURI_CHANNEL_MARKER__=!0,i.set(this,(()=>{})),o.set(this,0),r.set(this,{}),this.id=function(t,e=!1){return window.__TAURI_INTERNALS__.transformCallback(t,e)}((({message:t,id:a})=>{if(a===e(this,o,"f")){n(this,o,a+1),e(this,i,"f").call(this,t);const s=Object.keys(e(this,r,"f"));if(s.length>0){let t=a+1;for(const n of s.sort()){if(parseInt(n)!==t)break;{const o=e(this,r,"f")[n];delete e(this,r,"f")[n],e(this,i,"f").call(this,o),t+=1}}n(this,o,t)}}else e(this,r,"f")[a.toString()]=t}))}set onmessage(t){n(this,i,t)}get onmessage(){return e(this,i,"f")}[(i=new WeakMap,o=new WeakMap,r=new WeakMap,s)](){return`__CHANNEL__:${this.id}`}toJSON(){return this[s]()}}async function f(t,e={},n){return window.__TAURI_INTERNALS__.invoke(t,e,n)}class l{get rid(){return e(this,a,"f")}constructor(t){a.set(this,void 0),n(this,a,t)}async close(){return f("plugin:resources|close",{rid:this.rid})}}var u,p;function w(t){return{isFile:t.isFile,isDirectory:t.isDirectory,isSymlink:t.isSymlink,size:t.size,mtime:null!==t.mtime?new Date(t.mtime):null,atime:null!==t.atime?new Date(t.atime):null,birthtime:null!==t.birthtime?new Date(t.birthtime):null,readonly:t.readonly,fileAttributes:t.fileAttributes,dev:t.dev,ino:t.ino,mode:t.mode,nlink:t.nlink,uid:t.uid,gid:t.gid,rdev:t.rdev,blksize:t.blksize,blocks:t.blocks}}a=new WeakMap,t.BaseDirectory=void 0,(u=t.BaseDirectory||(t.BaseDirectory={}))[u.Audio=1]="Audio",u[u.Cache=2]="Cache",u[u.Config=3]="Config",u[u.Data=4]="Data",u[u.LocalData=5]="LocalData",u[u.Document=6]="Document",u[u.Download=7]="Download",u[u.Picture=8]="Picture",u[u.Public=9]="Public",u[u.Video=10]="Video",u[u.Resource=11]="Resource",u[u.Temp=12]="Temp",u[u.AppConfig=13]="AppConfig",u[u.AppData=14]="AppData",u[u.AppLocalData=15]="AppLocalData",u[u.AppCache=16]="AppCache",u[u.AppLog=17]="AppLog",u[u.Desktop=18]="Desktop",u[u.Executable=19]="Executable",u[u.Font=20]="Font",u[u.Home=21]="Home",u[u.Runtime=22]="Runtime",u[u.Template=23]="Template",t.SeekMode=void 0,(p=t.SeekMode||(t.SeekMode={}))[p.Start=0]="Start",p[p.Current=1]="Current",p[p.End=2]="End";class d extends l{async read(t){if(0===t.byteLength)return 0;const e=await f("plugin:fs|read",{rid:this.rid,len:t.byteLength}),n=function(t){const e=new Uint8ClampedArray(t),n=e.byteLength;let i=0;for(let t=0;t<n;t++)i*=256,i+=e[t];return i}(e.slice(-8)),i=e instanceof ArrayBuffer?new Uint8Array(e):e;return t.set(i.slice(0,i.length-8)),0===n?null:n}async seek(t,e){return await f("plugin:fs|seek",{rid:this.rid,offset:t,whence:e})}async stat(){return w(await f("plugin:fs|fstat",{rid:this.rid}))}async truncate(t){await f("plugin:fs|ftruncate",{rid:this.rid,len:t})}async write(t){return await f("plugin:fs|write",{rid:this.rid,data:t})}}async function h(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");const n=await f("plugin:fs|open",{path:t instanceof URL?t.toString():t,options:e});return new d(n)}async function y(t){await f("plugin:fs|unwatch",{rid:t})}return t.FileHandle=d,t.copyFile=async function(t,e,n){if(t instanceof URL&&"file:"!==t.protocol||e instanceof URL&&"file:"!==e.protocol)throw new TypeError("Must be a file URL.");await f("plugin:fs|copy_file",{fromPath:t instanceof URL?t.toString():t,toPath:e instanceof URL?e.toString():e,options:n})},t.create=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");const n=await f("plugin:fs|create",{path:t instanceof URL?t.toString():t,options:e});return new d(n)},t.exists=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");return await f("plugin:fs|exists",{path:t instanceof URL?t.toString():t,options:e})},t.lstat=async function(t,e){return w(await f("plugin:fs|lstat",{path:t instanceof URL?t.toString():t,options:e}))},t.mkdir=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");await f("plugin:fs|mkdir",{path:t instanceof URL?t.toString():t,options:e})},t.open=h,t.readDir=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.");return await f("plugin:fs|read_dir",{path:t instanceof URL?t.toString():t,options:e})},t.readFile=async function(t,e){if(t instanceof URL&&"file:"!==t.protocol)throw new TypeError("Must be a file URL.")Line truncated
+26 -5
View File
@@ -770,10 +770,14 @@ async function readTextFile(
throw new TypeError('Must be a file URL.')
}
return await invoke<string>('plugin:fs|read_text_file', {
const arr = await invoke<ArrayBuffer | number[]>('plugin:fs|read_text_file', {
path: path instanceof URL ? path.toString() : path,
options
})
const bytes = arr instanceof ArrayBuffer ? arr : Uint8Array.from(arr)
return new TextDecoder().decode(bytes)
}
/**
@@ -804,6 +808,7 @@ async function readTextFileLines(
return await Promise.resolve({
path: pathStr,
rid: null as number | null,
async next(): Promise<IteratorResult<string>> {
if (this.rid === null) {
this.rid = await invoke<number>('plugin:fs|read_text_file_lines', {
@@ -812,19 +817,35 @@ async function readTextFileLines(
})
}
const [line, done] = await invoke<[string | null, boolean]>(
const arr = await invoke<ArrayBuffer | number[]>(
'plugin:fs|read_text_file_lines_next',
{ rid: this.rid }
)
// an iteration is over, reset rid for next iteration
if (done) this.rid = null
const bytes =
arr instanceof ArrayBuffer ? new Uint8Array(arr) : Uint8Array.from(arr)
// Rust side will never return an empty array for this command and
// ensure there is at least one elements there.
//
// This is an optimization to include whether we finished iteration or not (1 or 0)
// at the end of returned array to avoid serialization overhead of separate values.
const done = bytes[bytes.byteLength - 1] === 1
if (done) {
// a full iteration is over, reset rid for next iteration
this.rid = null
return { value: null, done }
}
const line = new TextDecoder().decode(bytes.slice(0, bytes.byteLength))
return {
value: done ? '' : line!,
value: line,
done
}
},
[Symbol.asyncIterator](): AsyncIterableIterator<string> {
return this
}
+83 -59
View File
@@ -15,7 +15,7 @@ use tauri::{
use std::{
borrow::Cow,
fs::File,
io::{BufReader, Lines, Read, Write},
io::{BufRead, BufReader, Read, Write},
path::PathBuf,
str::FromStr,
sync::Mutex,
@@ -372,6 +372,7 @@ pub async fn read_file<R: Runtime>(
Ok(tauri::ipc::Response::new(contents))
}
// TODO, remove in v3, rely on `read_file` command instead
#[tauri::command]
pub async fn read_text_file<R: Runtime>(
webview: Webview<R>,
@@ -379,33 +380,8 @@ pub async fn read_text_file<R: Runtime>(
command_scope: CommandScope<Entry>,
path: SafeFilePath,
options: Option<BaseOptions>,
) -> CommandResult<String> {
let (mut file, path) = resolve_file(
&webview,
&global_scope,
&command_scope,
path,
OpenOptions {
base: BaseOptions {
base_dir: options.as_ref().and_then(|o| o.base_dir),
},
options: crate::OpenOptions {
read: true,
..Default::default()
},
},
)?;
let mut contents = String::new();
file.read_to_string(&mut contents).map_err(|e| {
format!(
"failed to read file as text at path: {} with error: {e}",
path.display()
)
})?;
Ok(contents)
) -> CommandResult<tauri::ipc::Response> {
read_file(webview, global_scope, command_scope, path, options).await
}
#[tauri::command]
@@ -416,8 +392,6 @@ pub fn read_text_file_lines<R: Runtime>(
path: SafeFilePath,
options: Option<BaseOptions>,
) -> CommandResult<ResourceId> {
use std::io::BufRead;
let resolved_path = resolve_path(
&webview,
&global_scope,
@@ -433,7 +407,7 @@ pub fn read_text_file_lines<R: Runtime>(
)
})?;
let lines = BufReader::new(file).lines();
let lines = BufReader::new(file);
let rid = webview.resources_table().add(StdLinesResource::new(lines));
Ok(rid)
@@ -443,18 +417,28 @@ pub fn read_text_file_lines<R: Runtime>(
pub async fn read_text_file_lines_next<R: Runtime>(
webview: Webview<R>,
rid: ResourceId,
) -> CommandResult<(Option<String>, bool)> {
) -> CommandResult<tauri::ipc::Response> {
let mut resource_table = webview.resources_table();
let lines = resource_table.get::<StdLinesResource>(rid)?;
let ret = StdLinesResource::with_lock(&lines, |lines| {
lines.next().map(|a| (a.ok(), false)).unwrap_or_else(|| {
let _ = resource_table.close(rid);
(None, true)
})
let ret = StdLinesResource::with_lock(&lines, |lines| -> CommandResult<Vec<u8>> {
// This is an optimization to include wether we finished iteration or not (1 or 0)
// at the end of returned vector so we can use `tauri::ipc::Response`
// and avoid serialization overhead of separate values.
match lines.next() {
Some(Ok(mut bytes)) => {
bytes.push(false as u8);
Ok(bytes)
}
Some(Err(_)) => Ok(vec![false as u8]),
None => {
resource_table.close(rid)?;
Ok(vec![true as u8])
}
}
});
Ok(ret)
ret.map(tauri::ipc::Response::new)
}
#[derive(Debug, Clone, Deserialize)]
@@ -805,10 +789,11 @@ fn default_create_value() -> bool {
true
}
fn write_file_inner<R: Runtime>(
#[tauri::command]
pub async fn write_file<R: Runtime>(
webview: Webview<R>,
global_scope: &GlobalScope<Entry>,
command_scope: &CommandScope<Entry>,
global_scope: GlobalScope<Entry>,
command_scope: CommandScope<Entry>,
request: tauri::ipc::Request<'_>,
) -> CommandResult<()> {
let data = match request.body() {
@@ -839,8 +824,8 @@ fn write_file_inner<R: Runtime>(
let (mut file, path) = resolve_file(
&webview,
global_scope,
command_scope,
&global_scope,
&command_scope,
path,
if let Some(opts) = options {
OpenOptions {
@@ -883,17 +868,7 @@ fn write_file_inner<R: Runtime>(
.map_err(Into::into)
}
#[tauri::command]
pub async fn write_file<R: Runtime>(
webview: Webview<R>,
global_scope: GlobalScope<Entry>,
command_scope: CommandScope<Entry>,
request: tauri::ipc::Request<'_>,
) -> CommandResult<()> {
write_file_inner(webview, &global_scope, &command_scope, request)
}
// TODO, in v3, remove this command and rely on `write_file` command only
// TODO, remove in v3, rely on `write_file` command instead
#[tauri::command]
pub async fn write_text_file<R: Runtime>(
webview: Webview<R>,
@@ -901,7 +876,7 @@ pub async fn write_text_file<R: Runtime>(
command_scope: CommandScope<Entry>,
request: tauri::ipc::Request<'_>,
) -> CommandResult<()> {
write_file_inner(webview, &global_scope, &command_scope, request)
write_file(webview, global_scope, command_scope, request).await
}
#[tauri::command]
@@ -1048,14 +1023,38 @@ impl StdFileResource {
impl Resource for StdFileResource {}
struct StdLinesResource(Mutex<Lines<BufReader<File>>>);
/// Same as [std::io::Lines] but with bytes
struct LinesBytes<T: BufRead>(T);
impl<B: BufRead> Iterator for LinesBytes<B> {
type Item = std::io::Result<Vec<u8>>;
fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> {
let mut buf = Vec::new();
match self.0.read_until(b'\n', &mut buf) {
Ok(0) => None,
Ok(_n) => {
if buf.last() == Some(&b'\n') {
buf.pop();
if buf.last() == Some(&b'\r') {
buf.pop();
}
}
Some(Ok(buf))
}
Err(e) => Some(Err(e)),
}
}
}
struct StdLinesResource(Mutex<LinesBytes<BufReader<File>>>);
impl StdLinesResource {
fn new(lines: Lines<BufReader<File>>) -> Self {
Self(Mutex::new(lines))
fn new(lines: BufReader<File>) -> Self {
Self(Mutex::new(LinesBytes(lines)))
}
fn with_lock<R, F: FnMut(&mut Lines<BufReader<File>>) -> R>(&self, mut f: F) -> R {
fn with_lock<R, F: FnMut(&mut LinesBytes<BufReader<File>>) -> R>(&self, mut f: F) -> R {
let mut lines = self.0.lock().unwrap();
f(&mut lines)
}
@@ -1154,7 +1153,12 @@ fn get_stat(metadata: std::fs::Metadata) -> FileInfo {
}
}
#[cfg(test)]
mod test {
use std::io::{BufRead, BufReader};
use super::LinesBytes;
#[test]
fn safe_file_path_parse() {
use super::SafeFilePath;
@@ -1168,4 +1172,24 @@ mod test {
Ok(SafeFilePath::Url(_))
));
}
#[test]
fn test_lines_bytes() {
let base = String::from("line 1\nline2\nline 3\nline 4");
let bytes = base.as_bytes();
let string1 = base.lines().collect::<String>();
let string2 = BufReader::new(bytes)
.lines()
.map_while(Result::ok)
.collect::<String>();
let string3 = LinesBytes(BufReader::new(bytes))
.flatten()
.flat_map(String::from_utf8)
.collect::<String>();
assert_eq!(string1, string2);
assert_eq!(string1, string3);
assert_eq!(string2, string3);
}
}