optimize file daemon for large number of files

Cargo.toml

@@ -33,16 +33,17 @@ ignore = "0.4.20"
 include_dir = "0.7.3" 
 lexical-sort = "0.3.1" 
 memchr = "2.5.0" 
-rustix = "0.38.34"
 notify = "6.0.0" 
 num-traits = "0.2.15" 
 once_cell = "1.17.1" 
 opener = "0.7.0"
 rayon = "1.7.0" 
 ropey = "1.5.1" 
+rustix = "0.38.34"
 serde = "1.0.152" 
 serde_json = "1.0.115" 
 slab = "0.4.8" 
+sorted-vec = "0.8.3"
 sublime_fuzzy = "0.7.0" 
 subprocess = "0.2.9" 
 tempdir = "0.3.7" 

crates/ferrite-core/Cargo.toml

@@ -34,6 +34,7 @@ ropey = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true } 
 slab = { workspace = true } 
+sorted-vec = { workspace = true }
 sublime_fuzzy = { workspace = true } 
 subprocess = { workspace = true } 
 toml = { workspace = true, features = ["parse"] } 

crates/ferrite-core/src/engine.rs

@@ -3,7 +3,7 @@ use std::{
     env, io,
     num::NonZeroUsize,
     path::{Path, PathBuf},
-    sync::mpsc,
+    sync::{mpsc, Arc, RwLock},
     time::{Duration, Instant},
 };
 
@@ -884,7 +884,7 @@ impl Engine {
             .collect();
 
         self.buffer_finder = Some(SearchBuffer::new(
-            BufferFindProvider(buffers.into()),
+            BufferFindProvider(Arc::new(RwLock::new(buffers))),
             self.proxy.dup(),
             self.try_get_current_buffer_path(),
         ));

crates/ferrite-core/src/pubsub.rs

@@ -4,31 +4,39 @@ use flume::{Receiver, RecvError, SendError, Sender};
 
 pub struct Publisher<T> {
     sender: Sender<()>,
-    data: Arc<RwLock<Arc<T>>>,
+    data: Arc<RwLock<T>>,
 }
 
 impl<T> Publisher<T> {
-    pub fn publish(&self, value: T) -> Result<(), SendError<()>> {
-        *self.data.write().unwrap() = Arc::new(value);
+    pub fn modify(&self, f: impl FnOnce(&mut T)) {
+        let mut mut_ref = self.data.write().unwrap();
+        (f)(&mut *mut_ref);
+    }
+
+    pub fn publish(&self) -> Result<(), SendError<()>> {
         self.sender.send(())
     }
 }
 
 pub struct Subscriber<T> {
-    data: Arc<RwLock<Arc<T>>>,
+    data: Arc<RwLock<T>>,
     reciver: Receiver<()>,
     has_recived: bool,
 }
 
 impl<T> Subscriber<T> {
-    pub fn recive(&mut self) -> Result<Arc<T>, RecvError> {
+    pub fn recive(&mut self) -> Result<Arc<RwLock<T>>, RecvError> {
         if !self.has_recived {
             self.has_recived = true;
-            return Ok(self.data.read().unwrap().clone());
+            return Ok(self.data.clone());
         }
 
         self.reciver.recv()?;
-        Ok(self.data.read().unwrap().clone())
+        Ok(self.data.clone())
+    }
+
+    pub fn get(&self) -> Arc<RwLock<T>> {
+        self.data.clone()
     }
 }
 
@@ -44,7 +52,7 @@ impl<T> Clone for Subscriber<T> {
 
 pub fn create<T>(value: T) -> (Publisher<T>, Subscriber<T>) {
     let (sender, reciver) = flume::unbounded::<()>();
-    let data = Arc::new(RwLock::new(Arc::new(value)));
+    let data = Arc::new(RwLock::new(value));
     (
         Publisher {
             sender,

crates/ferrite-core/src/search_buffer.rs

@@ -1,4 +1,9 @@
-use std::{borrow::Cow, path::PathBuf, sync::Arc, thread};
+use std::{
+    borrow::Cow,
+    path::PathBuf,
+    sync::{Arc, RwLock},
+    thread,
+};
 
 use cb::select;
 use ferrite_utility::{graphemes::RopeGraphemeExt, line_ending::LineEnding};
@@ -40,10 +45,10 @@ where
         search_field.set_view_lines(1);
 
         let (search_tx, search_rx): (_, cb::Receiver<String>) = cb::unbounded();
-        let (result_tx, result_rx): (_, cb::Receiver<SearchResult<_>>) = cb::unbounded();
+        let (result_tx, result_rx): (_, cb::Receiver<SearchResult<M>>) = cb::unbounded();
 
         thread::spawn(move || {
-            let mut options = Arc::new(Vec::new());
+            let mut options = Arc::new(RwLock::new(Vec::new()));
             let mut query = String::new();
             let options_recv = option_provder.get_options_reciver();
 
@@ -78,13 +83,17 @@ where
                     continue;
                 }
 
-                let output = fuzzy_match::fuzzy_match(&query, (*options).clone(), path.as_deref());
-                let result = SearchResult {
-                    matches: output,
-                    total: options.len(),
-                };
-                if result_tx.send(result).is_err() {
-                    break;
+                {
+                    let options = options.read().unwrap();
+                    let options = &*options;
+                    let output = fuzzy_match::fuzzy_match::<M>(&query, options, path.as_deref());
+                    let result = SearchResult {
+                        matches: output,
+                        total: options.len(),
+                    };
+                    if result_tx.send(result).is_err() {
+                        break;
+                    }
                 }
 
                 proxy.request_render();
@@ -185,7 +194,7 @@ pub trait Matchable: Clone {
 
 pub trait SearchOptionProvider {
     type Matchable: Matchable;
-    fn get_options_reciver(&self) -> cb::Receiver<Arc<Vec<Self::Matchable>>>;
+    fn get_options_reciver(&self) -> cb::Receiver<Arc<RwLock<Vec<Self::Matchable>>>>;
 }
 
 impl Matchable for String {

crates/ferrite-core/src/search_buffer/buffer_find.rs

@@ -1,13 +1,16 @@
-use std::{borrow::Cow, sync::Arc};
+use std::{
+    borrow::Cow,
+    sync::{Arc, RwLock},
+};
 
 use super::{Matchable, SearchOptionProvider};
 
-pub struct BufferFindProvider(pub Arc<Vec<BufferItem>>);
+pub struct BufferFindProvider(pub Arc<RwLock<Vec<BufferItem>>>);
 
 impl SearchOptionProvider for BufferFindProvider {
     type Matchable = BufferItem;
 
-    fn get_options_reciver(&self) -> cb::Receiver<Arc<Vec<Self::Matchable>>> {
+    fn get_options_reciver(&self) -> cb::Receiver<Arc<RwLock<Vec<Self::Matchable>>>> {
         let (tx, rx) = cb::bounded(1);
         let _ = tx.send(self.0.clone());
         rx

crates/ferrite-core/src/search_buffer/file_daemon.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{HashMap, HashSet},
+    collections::HashMap,
     fs::File,
     io::Read,
     path::{Path, PathBuf},
@@ -10,13 +10,13 @@ use std::{
 
 use cb::select;
 use ignore::gitignore::{Gitignore, GitignoreBuilder};
-use lexical_sort::StringSort;
 use notify::{RecursiveMode, Watcher};
 use rayon::prelude::*;
+use sorted_vec::SortedSet;
 
 use crate::{
     config::Config,
-    pubsub::{self, Subscriber},
+    pubsub::{self, Publisher, Subscriber},
 };
 
 fn get_text_file_path(path: PathBuf) -> Option<PathBuf> {
@@ -48,16 +48,45 @@ fn trim_path(start: &str, path: &Path) -> String {
         .to_string()
 }
 
+#[repr(transparent)]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LexicallySortedString(pub String);
+
+impl From<String> for LexicallySortedString {
+    fn from(value: String) -> Self {
+        Self(value)
+    }
+}
+
+impl From<LexicallySortedString> for String {
+    fn from(value: LexicallySortedString) -> Self {
+        value.0
+    }
+}
+
+impl PartialOrd for LexicallySortedString {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for LexicallySortedString {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        lexical_sort::natural_lexical_cmp(&self.0, &other.0)
+    }
+}
+
 pub struct FileDaemon {
-    subscriber: Subscriber<Vec<String>>,
+    subscriber: Subscriber<SortedSet<LexicallySortedString>>,
     change_detector: Subscriber<()>,
     exit_tx: cb::Sender<()>,
 }
 
 impl FileDaemon {
     pub fn new(path: PathBuf, config: &Config) -> anyhow::Result<Self> {
         let (exit_tx, exit_rx) = cb::bounded::<()>(1);
-        let (publisher, subscriber) = pubsub::create(Vec::new());
+        let (publisher, subscriber): (Publisher<SortedSet<LexicallySortedString>>, _) =
+            pubsub::create(SortedSet::new());
         let (change_broadcaster, change_detector) = pubsub::create(());
         let path_to_search = path.clone();
         let picker_config = config.picker;
@@ -108,8 +137,6 @@ impl FileDaemon {
                 watcher_thread.join().unwrap();
             });
 
-            let mut tracked_files = HashSet::new();
-
             let path: PathBuf = path_to_search;
             let path_str = path.to_string_lossy().into_owned();
 
@@ -124,6 +151,7 @@ impl FileDaemon {
 
             {
                 loop {
+                    let mut tracked_files = Vec::new();
                     let start = Instant::now();
                     let entries: Vec<_> = iterator.by_ref().take(1000).collect();
 
@@ -144,12 +172,15 @@ impl FileDaemon {
                             }),
                     );
 
-                    let mut files: Vec<_> = tracked_files
-                        .iter()
-                        .map(|path| trim_path(&path_str, path))
-                        .collect();
-                    files.string_sort(lexical_sort::natural_lexical_cmp);
-                    if publisher.publish(files).is_err() {
+                    publisher.modify(|published_files| {
+                        for file in tracked_files
+                            .iter()
+                            .map(|path| trim_path(&path_str, path).into())
+                        {
+                            published_files.replace(file);
+                        }
+                    });
+                    if publisher.publish().is_err() {
                         return;
                     }
 
@@ -166,7 +197,6 @@ impl FileDaemon {
             let mut gitignore_cache: HashMap<PathBuf, Gitignore> = HashMap::new();
             let mut last_clear = Instant::now();
 
-            let mut updated = false;
             loop {
                 {
                     let now = Instant::now();
@@ -213,7 +243,9 @@ impl FileDaemon {
                                         match gitignore_cache.get(&path) {
                                             Some(ignore) => {
                                                 if ignore.matched(&path, false).is_ignore() {
-                                                    updated |= tracked_files.insert(path);
+                                                    publisher.modify(|files| {
+                                                        files.replace(trim_path(&path_str, &path).into());
+                                                    });
                                                 }
                                             }
                                             None => {
@@ -239,21 +271,27 @@ impl FileDaemon {
                                                             .matched_path_or_any_parents(&path, false)
                                                             .is_ignore()
                                                         {
-                                                            updated |= tracked_files.insert(path.clone());
+                                                            publisher.modify(|files| {
+                                                                files.replace(trim_path(&path_str, &path).into());
+                                                            });
                                                         }
                                                         if let Some(parent) = path.parent() {
                                                             gitignore_cache
                                                                 .insert(parent.to_path_buf(), ignore);
                                                         }
                                                     }
                                                     Err(_) => {
-                                                        updated |= tracked_files.insert(path);
+                                                        publisher.modify(|files| {
+                                                            files.replace(trim_path(&path_str, &path).into());
+                                                        });
                                                     }
                                                 }
                                             }
                                         }
                                     } else {
-                                        updated |= tracked_files.remove(&path);
+                                        publisher.modify(|files| {
+                                            files.replace(trim_path(&path_str, &path).into());
+                                        });
                                     };
                                 }
                             }
@@ -266,17 +304,11 @@ impl FileDaemon {
                     }
                 }
 
-                if update_rx.is_empty() && updated {
-                    updated = false;
-                    let mut files: Vec<_> = tracked_files
-                        .iter()
-                        .map(|path| trim_path(&path_str, path))
-                        .collect();
-                    files.string_sort(lexical_sort::natural_lexical_cmp);
-                    if publisher.publish(files).is_err() {
+                if update_rx.is_empty() {
+                    if publisher.publish().is_err() {
                         return;
                     }
-                    if change_broadcaster.publish(()).is_err() {
+                    if change_broadcaster.publish().is_err() {
                         return;
                     }
                 }
@@ -290,7 +322,7 @@ impl FileDaemon {
         })
     }
 
-    pub fn subscribe(&self) -> Subscriber<Vec<String>> {
+    pub fn subscribe(&self) -> Subscriber<SortedSet<LexicallySortedString>> {
         self.subscriber.clone()
     }