Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: introduce URN to speed up large directory file updates #1652

Merged
merged 2 commits into from
Sep 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions yazi-core/src/manager/commands/bulk_rename.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use tokio::{fs::{self, OpenOptions}, io::{stdin, AsyncReadExt, AsyncWriteExt}};
use yazi_config::{OPEN, PREVIEW};
use yazi_dds::Pubsub;
use yazi_proxy::{AppProxy, TasksProxy, HIDER, WATCHER};
use yazi_shared::{fs::{max_common_root, maybe_exists, paths_to_same_file, File, FilesOp, Url}, terminal_clear};
use yazi_shared::{fs::{max_common_root, maybe_exists, paths_to_same_file, File, Url}, terminal_clear};

use crate::manager::Manager;

Expand Down Expand Up @@ -98,7 +98,8 @@ impl Manager {
// FIXME: consider old and new in the different directories
if !succeeded.is_empty() {
Pubsub::pub_from_bulk(succeeded.iter().map(|(u, f)| (u, f.url())).collect());
FilesOp::Upserting(cwd, succeeded).emit();
// FIXME 0
// FilesOp::Upserting(cwd, succeeded).emit();
}
drop(permit);

Expand Down
10 changes: 5 additions & 5 deletions yazi-core/src/manager/commands/create.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};

use anyhow::Result;
use tokio::fs;
use yazi_config::popup::{ConfirmCfg, InputCfg};
use yazi_proxy::{ConfirmProxy, InputProxy, TabProxy, WATCHER};
use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, symlink_realpath, File, FilesOp, Url}};
use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, realname, File, FilesOp, Url, UrnBuf}};

use crate::manager::Manager;

Expand Down Expand Up @@ -48,9 +48,9 @@ impl Manager {

if dir {
fs::create_dir_all(&new).await?;
} else if let Ok(real) = symlink_realpath(&new).await {
} else if let Some(real) = realname(&new).await {
ok_or_not_found(fs::remove_file(&new).await)?;
FilesOp::Deleting(parent.clone(), vec![Url::from(real)]).emit();
FilesOp::Deleting(parent.clone(), HashSet::from_iter([UrnBuf::_from(real)])).emit();
fs::File::create(&new).await?;
} else {
fs::create_dir_all(&parent).await.ok();
Expand All @@ -59,7 +59,7 @@ impl Manager {
}

if let Ok(f) = File::from(new.clone()).await {
FilesOp::Upserting(parent, HashMap::from_iter([(f.url_owned(), f)])).emit();
FilesOp::Upserting(parent, HashMap::from_iter([(f.urn_owned(), f)])).emit();
TabProxy::reveal(&new)
}
Ok(())
Expand Down
25 changes: 15 additions & 10 deletions yazi-core/src/manager/commands/rename.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};

use anyhow::Result;
use tokio::fs;
use yazi_config::popup::{ConfirmCfg, InputCfg};
use yazi_dds::Pubsub;
use yazi_proxy::{ConfirmProxy, InputProxy, TabProxy, WATCHER};
use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, paths_to_same_file, symlink_realpath, File, FilesOp, Url}};
use yazi_shared::{event::Cmd, fs::{maybe_exists, ok_or_not_found, paths_to_same_file, realname, File, FilesOp, Url, UrnBuf}};

use crate::manager::Manager;

Expand Down Expand Up @@ -74,22 +74,27 @@ impl Manager {
}

async fn rename_do(tab: usize, old: Url, new: Url) -> Result<()> {
let Some(p_old) = old.parent_url() else { return Ok(()) };
let Some(p_new) = new.parent_url() else { return Ok(()) };
let Some((p_old, n_old)) = old.pair() else { return Ok(()) };
let Some((p_new, n_new)) = new.pair() else { return Ok(()) };
let _permit = WATCHER.acquire().await.unwrap();

let overwritten = symlink_realpath(&new).await;
let overwritten = realname(&new).await;
fs::rename(&old, &new).await?;

if let Ok(o) = overwritten {
ok_or_not_found(fs::rename(&o, &new).await)?;
FilesOp::Deleting(p_new.clone(), vec![Url::from(o)]).emit();
if let Some(o) = overwritten {
ok_or_not_found(fs::rename(p_new.join(&o), &new).await)?;
FilesOp::Deleting(p_new.clone(), HashSet::from_iter([UrnBuf::_from(o)])).emit();
}
Pubsub::pub_from_rename(tab, &old, &new);

let file = File::from(new.clone()).await?;
FilesOp::Deleting(p_old, vec![old]).emit();
FilesOp::Upserting(p_new, HashMap::from_iter([(new.clone(), file)])).emit();
if p_new == p_old {
FilesOp::Upserting(p_old, HashMap::from_iter([(n_old, file)])).emit();
} else {
FilesOp::Deleting(p_old, HashSet::from_iter([n_old])).emit();
FilesOp::Upserting(p_new, HashMap::from_iter([(n_new, file)])).emit();
}

Ok(TabProxy::reveal(&new))
}

Expand Down
11 changes: 4 additions & 7 deletions yazi-core/src/manager/commands/update_files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ impl Manager {

let mut ops = vec![opt.op];
for u in LINKED.read().from_dir(ops[0].cwd()) {
ops.push(ops[0].chroot(u));
ops.push(ops[0].rebase(u));
}

for op in ops {
Expand Down Expand Up @@ -60,10 +60,7 @@ impl Manager {

fn update_parent(tab: &mut Tab, op: Cow<FilesOp>) {
let urn = tab.cwd().urn_owned();
// FIXME
let leave = false;
// let leave = matches!(*op, FilesOp::Deleting(_, ref urls) if
// urls.contains(&urn));
let leave = matches!(*op, FilesOp::Deleting(_, ref urns) if urns.contains(&urn));

if let Some(f) = tab.parent.as_mut() {
render!(f.update(op.into_owned()));
Expand Down Expand Up @@ -111,8 +108,8 @@ impl Manager {
}

fn update_history(tab: &mut Tab, op: Cow<FilesOp>) {
let leave = tab.parent.as_ref().and_then(|f| f.loc.parent_url().map(|p| (&f.loc, p))).is_some_and(
|(p, pp)| matches!(*op, FilesOp::Deleting(ref parent, ref urls) if *parent == pp && urls.contains(p)),
let leave = tab.parent.as_ref().and_then(|f| f.loc.parent_url().map(|p| (p, f.loc.urn()))).is_some_and(
|(p, n)| matches!(*op, FilesOp::Deleting(ref parent, ref urns) if *parent == p && urns.contains(n)),
);

let folder = tab.history.entry(op.cwd().clone()).or_insert_with(|| Folder::from(op.cwd()));
Expand Down
21 changes: 10 additions & 11 deletions yazi-core/src/manager/watcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use tracing::error;
use yazi_fs::{Files, Folder};
use yazi_plugin::isolate;
use yazi_proxy::WATCHER;
use yazi_shared::{fs::{symlink_realname, Cha, File, FilesOp, Url}, RoCell};
use yazi_shared::{fs::{realname_unchecked, Cha, File, FilesOp, Url}, RoCell};

use super::Linked;

Expand Down Expand Up @@ -124,27 +124,26 @@ impl Watcher {
let _permit = WATCHER.acquire().await.unwrap();
let mut reload = Vec::with_capacity(urls.len());

for url in urls {
let Some(name) = url.file_name() else { continue };
let Some(parent) = url.parent_url() else { continue };

let Ok(file) = File::from(url.clone()).await else {
FilesOp::Deleting(parent, vec![url]).emit();
for u in urls {
let Some((parent, urn)) = u.pair() else { continue };
let Ok(file) = File::from(u).await else {
FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit();
continue;
};

let eq = (!file.is_link() && fs::canonicalize(&url).await.is_ok_and(|p| p == *url))
|| symlink_realname(&url, &mut cached).await.is_ok_and(|s| s == name);
let u = file.url();
let eq = (!file.is_link() && fs::canonicalize(u).await.is_ok_and(|p| p == **u))
|| realname_unchecked(u, &mut cached).await.is_ok_and(|s| s == urn._deref()._as_path());

if !eq {
FilesOp::Deleting(parent, vec![url]).emit();
FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit();
continue;
}

if !file.is_dir() {
reload.push(file.clone());
}
FilesOp::Upserting(parent, HashMap::from_iter([(url, file)])).emit();
FilesOp::Upserting(parent, HashMap::from_iter([(urn, file)])).emit();
}

if reload.is_empty() {
Expand Down
11 changes: 2 additions & 9 deletions yazi-core/src/manager/yanked.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,10 @@ impl Yanked {
}

pub fn apply_op(&mut self, op: &FilesOp) {
let (removal, addition) = match op {
FilesOp::Deleting(_, urls) => (urls.iter().collect(), vec![]),
FilesOp::Updating(_, urls) | FilesOp::Upserting(_, urls) => {
urls.iter().filter(|(u, _)| self.contains(u)).map(|(u, f)| (u, f.url_owned())).unzip()
}
_ => (vec![], vec![]),
};

let (removal, addition) = op.diff_recoverable(|u| self.contains(u));
if !removal.is_empty() {
let old = self.urls.len();
self.urls.retain(|u| !removal.contains(&u));
self.urls.retain(|u| !removal.contains(u));
self.revision += (old != self.urls.len()) as u64;
}

Expand Down
24 changes: 9 additions & 15 deletions yazi-core/src/tab/selected.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,24 @@ impl Selected {
#[inline]
pub fn add(&mut self, url: &Url) -> bool { self.add_same(&[url]) == 1 }

pub fn add_many(&mut self, urls: &[&Url], same: bool) -> usize {
pub fn add_many(&mut self, urls: &[impl AsRef<Url>], same: bool) -> usize {
if same {
return self.add_same(urls);
}

let mut grouped: HashMap<_, Vec<_>> = Default::default();
for &u in urls {
if let Some(p) = u.parent_url() {
for u in urls {
if let Some(p) = u.as_ref().parent_url() {
grouped.entry(p).or_default().push(u);
}
}
grouped.into_values().map(|v| self.add_same(&v)).sum()
}

fn add_same(&mut self, urls: &[&Url]) -> usize {
fn add_same(&mut self, urls: &[impl AsRef<Url>]) -> usize {
// If it has appeared as a parent
let urls: Vec<_> = urls.iter().filter(|&&u| !self.parents.contains_key(u)).collect();
let urls: Vec<_> =
urls.iter().map(|u| u.as_ref()).filter(|&u| !self.parents.contains_key(u)).collect();
if urls.is_empty() {
return 0;
}
Expand All @@ -52,7 +53,7 @@ impl Selected {
}

let (now, len) = (timestamp_us(), self.inner.len());
self.inner.extend(urls.iter().enumerate().map(|(i, &&u)| (u.clone(), now + i as u64)));
self.inner.extend(urls.iter().enumerate().map(|(i, &u)| (u.clone(), now + i as u64)));

for u in parents {
*self.parents.entry(u).or_insert(0) += self.inner.len() - len;
Expand Down Expand Up @@ -103,14 +104,7 @@ impl Selected {
}

pub fn apply_op(&mut self, op: &FilesOp) {
let (removal, addition) = match op {
FilesOp::Deleting(_, urls) => (urls.iter().collect(), vec![]),
FilesOp::Updating(_, urls) | FilesOp::Upserting(_, urls) => {
urls.iter().filter(|&(u, _)| self.contains_key(u)).map(|(u, f)| (u, f.url())).unzip()
}
_ => (vec![], vec![]),
};

let (removal, addition) = op.diff_recoverable(|u| self.contains_key(u));
if !removal.is_empty() {
self.remove_many(&removal, !op.cwd().is_search());
}
Expand Down Expand Up @@ -196,7 +190,7 @@ mod tests {
fn insert_many_empty_urls_list() {
let mut s = Selected::default();

assert_eq!(0, s.add_same(&[]));
assert_eq!(0, s.add_same(&[] as &[&Url]));
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion yazi-core/src/tasks/preload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ impl Tasks {
targets
.iter()
.filter(|f| {
f.is_dir() && !targets.sizes.contains_key(f.url()) && !loading.contains(f.url())
f.is_dir() && !targets.sizes.contains_key(f.urn()) && !loading.contains(f.url())
})
.map(|f| f.url())
.collect()
Expand Down
2 changes: 1 addition & 1 deletion yazi-fm/src/lives/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ impl File {

reg.add_field_method_get("idx", |_, me| Ok(me.idx + 1));
reg.add_method("size", |_, me, ()| {
Ok(if me.is_dir() { me.folder().files.sizes.get(me.url()).copied() } else { Some(me.len) })
Ok(if me.is_dir() { me.folder().files.sizes.get(me.urn()).copied() } else { Some(me.len) })
});
reg.add_method("mime", |lua, me, ()| {
let cx = lua.named_registry_value::<CtxRef>("cx")?;
Expand Down
Loading