diff --git a/dvs.yaml b/dvs.yaml deleted file mode 100644 index d0c8343..0000000 --- a/dvs.yaml +++ /dev/null @@ -1,3 +0,0 @@ -storage_dir: /cluster-data/user-homes/jenna/Projects/devious/src/test_directory_storage -permissions: 511 -group: datascience diff --git a/src/helpers/hash.rs b/src/helpers/hash.rs index 59e37ef..9a7ef5b 100644 --- a/src/helpers/hash.rs +++ b/src/helpers/hash.rs @@ -4,7 +4,7 @@ use std::fs::File; use std::io::Result; use std::io::{self, Read}; -pub fn hash_file_with_blake3(file_path: &PathBuf) -> io::Result { +pub fn hash_file_with_blake3(file_path: &PathBuf) -> io::Result> { let file = File::open(file_path)?; let mmap = match maybe_memmap_file(&file) { @@ -17,10 +17,10 @@ pub fn hash_file_with_blake3(file_path: &PathBuf) -> io::Result { }; let mut hasher = Hasher::new(); hasher.update_rayon(&mmap); - Ok(hasher.finalize().to_string()) + Ok(Some(hasher.finalize().to_string())) } -fn hash_file_with_blake3_direct(file_path: &PathBuf) -> io::Result { +fn hash_file_with_blake3_direct(file_path: &PathBuf) -> io::Result> { let mut file = File::open(file_path)?; let mut hasher = Hasher::new(); @@ -35,7 +35,7 @@ fn hash_file_with_blake3_direct(file_path: &PathBuf) -> io::Result { } let hash_result = hasher.finalize(); - Ok(hash_result.to_string()) + Ok(Some(hash_result.to_string())) } // Mmap a file, if it looks like a good idea. Return None in cases where we @@ -70,17 +70,17 @@ fn maybe_memmap_file(file: &File) -> Result> { }) } -pub fn get_file_hash(path: &PathBuf) -> String { +pub fn get_file_hash(path: &PathBuf) -> Option { // TODO: get cache if possible let hash =match hash_file_with_blake3(&path) { Ok(hash) => hash, - Err(_) => String::from(""), // swallowing error - return empty string if not hashable + Err(_) => None, }; // TODO: cache bytes - return hash.to_string(); + return hash; } pub fn get_storage_path(storage_dir: &PathBuf, file_hash: &String) -> PathBuf { diff --git a/src/library/add.rs b/src/library/add.rs index 27e2443..35981a0 100644 --- a/src/library/add.rs +++ b/src/library/add.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; use std::os::unix::fs::PermissionsExt; +use serde::Serialize; use file_owner::{Group, PathExt}; use std::{fs, u32}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use crate::helpers::hash; use crate::helpers::copy; use crate::helpers::file; @@ -10,12 +11,28 @@ use crate::helpers::ignore; use crate::helpers::config; use crate::helpers::repo; -pub fn dvs_add(files: &Vec, message: &String) -> Result<()> { +#[derive(Clone, PartialEq, Serialize)] +enum Outcome { + Success, + AlreadyPresent, + Error +} + +#[derive(Clone, PartialEq, Serialize)] +pub struct AddedFile { + path: PathBuf, + hash: Option, + outcome: Outcome, + error: Option, + size: Option, +} + +pub fn dvs_add(files: &Vec, message: &String) -> Result> { // Get git root let git_dir = repo::get_nearest_repo_dir(&PathBuf::from(".")).with_context(|| "could not find git repo root - make sure you're in an active git repository")?; // load the config - let conf = config::read(&git_dir)?; + let conf = config::read(&git_dir).with_context(|| "could not load configuration file - no dvs.yaml in directory - be sure to initiate devious")?; let mut queued_paths: Vec = Vec::new(); @@ -48,65 +65,167 @@ pub fn dvs_add(files: &Vec, message: &String) -> Result<()> { queued_paths.push(file); } // for - // add each file in queued_paths to storage - for file in &queued_paths { - add(file, &conf, &message)?; - } + let added_files = queued_paths.into_iter().map(|file| { + add(&file, &conf, &message) + }).collect::>(); - if queued_paths.is_empty() { - // json warning: no files were queued - } - - Ok(()) + return Ok(added_files) } // run_add_cmd -fn add(local_path: &PathBuf, conf: &config::Config, message: &String) -> Result { +fn add(local_path: &PathBuf, conf: &config::Config, message: &String) -> AddedFile { + // set error to None by default + let mut error: Option = None; + // get file hash - let file_hash = hash::hash_file_with_blake3(local_path).with_context(|| format!("could not hash file"))?; + let file_hash = hash::get_file_hash(&local_path); + if file_hash.is_none() && error.is_none() { + error = Some(String::from("could not hash file")); + } + + // get file size + let file_size = get_file_size(&local_path); + if file_size.is_none() && error.is_none() { + error = Some(String::from("unable to get size of file")); + } + + // get user name + let user_name = get_user_name(&local_path); + if user_name.is_none() && error.is_none() { + error = Some(String::from("could not get name of file owner")); + } + + // check group + let group: Option = match Group::from_name(&conf.group) { + Ok(group) => Some(group), + Err(_) => { + if error.is_none() {error = Some(String::from("group not found"));} + None + } + }; + + // now see if file can be added + let storage_dir_abs: Option = match conf.storage_dir.canonicalize() { + Ok(path) => Some(path), + Err(_) => { + if error.is_none() {error = Some(String::from("could not find storage directory"))} + None + } + }; - // get storage path - let storage_dir_abs = conf.storage_dir.canonicalize().with_context(|| format!("could not find storage directory: {}", conf.storage_dir.display()))?; - let dest_path = hash::get_storage_path(&storage_dir_abs, &file_hash); + if error.is_some() { + return AddedFile{ + path: local_path.clone(), + hash: file_hash, + outcome: Outcome::Error, + error: error, + size: file_size + }; + } - // check if group exists again - let group = Group::from_name(&conf.group).with_context(|| format!("group not found: {}", conf.group))?; + // can safely unwrap storage_dir_abs and file_hash + let file_hash_value = file_hash.clone().unwrap(); + let dest_path = hash::get_storage_path(&storage_dir_abs.unwrap(), &file_hash_value); // Copy the file to the storage directory if it's not already there + let mut outcome: Outcome = Outcome::Success; if !dest_path.exists() { // copy - copy::copy(&local_path, &dest_path).with_context(|| format!("could not copy {} to storage directory: {}", local_path.display(), dest_path.display()))?; + copy_file_to_storage_directory(local_path, &dest_path, &conf.permissions, &group.unwrap()); } else { - println!("{} already exists in storage directory", local_path.display()) + outcome = Outcome::AlreadyPresent; } - // set permissions - set_permissions(&conf.permissions, &dest_path)?; - - // set group ownership - dest_path.set_group(group).with_context(|| format!("unable to set group: {}", group))?; - - // get file size - let file_size = get_file_size(&local_path)?; - - // get user name - let user_name = get_user_name(&local_path)?; - - // create + write metadata file + // create metadata let metadata = file::Metadata{ - file_hash: file_hash.clone(), - file_size, + file_hash: file_hash_value, + file_size: file_size.unwrap(), time_stamp: chrono::offset::Local::now().to_string(), message: message.clone(), - saved_by: user_name + saved_by: user_name.unwrap() + }; + + // write metadata file + match file::save(&metadata, &local_path) { + Ok(_) => {}, + Err(_) => if error.is_none() {error = Some(String::from("could not save metadata file"))} }; - file::save(&metadata, &local_path).with_context(|| format!("could not save metadata into file"))?; // Add file to gitignore - ignore::add_gitignore_entry(local_path).with_context(|| format!("could not add .gitignore entry"))?; + match ignore::add_gitignore_entry(local_path) { + Ok(_) => {}, + Err(_) => { + if error.is_none() {error = Some(String::from("could not add .gitignore entry"))} + } + }; - return Ok(file_hash); + if error.is_some() {outcome = Outcome::Error} + + return AddedFile { + path: local_path.clone(), + hash: file_hash.clone(), + outcome, + error, + size: file_size + } +} + + +fn get_file_size(local_path: &PathBuf) -> Option { + match local_path.metadata() { + Ok(data) => return Some(data.len()), + Err(_) => return None, + }; +} + + +fn get_user_name(local_path: &PathBuf) -> Option { + let owner = match local_path.owner().with_context(|| format!("")) { + Ok(owner) => owner, + Err(_) => return None, + }; + match owner.name() { + Ok(name) => return Some(name.unwrap()), + Err(_) => return None, + }; +} + + +fn copy_file_to_storage_directory(local_path: &PathBuf, dest_path: &PathBuf, mode: &u32, group: &Group) -> Option { + let mut error = None; + match copy::copy(&local_path, &dest_path) { + Ok(_) => { + // set permissions + match set_permissions(&mode, &dest_path) { + Ok(_) => {}, + Err(_) => { + // set error + if error.is_none() {error = Some(String::from("could not set file permissions"))} + // delete copied file + fs::remove_file(&dest_path) + .expect(format!("could not set permissions after copying {} to {}: error deleting copied file. Delete {} manually.", local_path.display(), dest_path.display(), dest_path.display()).as_str()); + } + }; + + // set group ownership + match dest_path.set_group(group.clone()) { + Ok(_) => {}, + Err(_) => { + // set error + if error.is_none() {error = Some(String::from("could not set file group ownership"))} + // delete copied file + fs::remove_file(&dest_path) + .expect(format!("could not set group ownership after copying {} to {}: error deleting copied file. Delete {} manually.", local_path.display(), dest_path.display(), dest_path.display()).as_str()); + + } + }; + } // Ok, could copy + Err(_) => { + if error.is_none() {error = Some(String::from("could not copy file to storage directory"))} + } + }; + return error } fn set_permissions(mode: &u32, dest_path: &PathBuf) -> Result<()> { @@ -115,17 +234,4 @@ fn set_permissions(mode: &u32, dest_path: &PathBuf) -> Result<()> { let new_permissions = fs::Permissions::from_mode(*mode); fs::set_permissions(&dest_path, new_permissions).with_context(|| format!("unable to set permissions: {}", mode))?; Ok(()) -} - -fn get_file_size(local_path: &PathBuf) -> Result { - let local_path_data = local_path.metadata().with_context(|| format!("unable to get size of file: {}", local_path.display()))?; - return Ok(local_path_data.len()); -} - -fn get_user_name(local_path: &PathBuf) -> Result { - let owner = local_path.owner().with_context(|| format!(""))?; - match owner.name() { - Ok(name) => return Ok(name.unwrap()), - Err(e) => return Err(anyhow!("could not get name of file owner: {e}")), - }; } \ No newline at end of file diff --git a/src/library/get.rs b/src/library/get.rs index 57f4a50..b20a8cb 100644 --- a/src/library/get.rs +++ b/src/library/get.rs @@ -42,7 +42,7 @@ pub fn get(local_path: &PathBuf, storage_dir: &PathBuf) -> Result<()> { let metadata_hash = metadata.file_hash; // check if up-to-date file is already present locally - if !local_path.exists() || local_hash == String::from("") || metadata_hash == String::from("") || local_hash != metadata_hash { + if !local_path.exists() || local_hash.is_none() || metadata_hash == String::from("") || local_hash.unwrap() != metadata_hash { copy::copy(&storage_path, &local_path).with_context(|| format!("could not copy {} from storage directory: {}", local_path.display(), storage_dir.display()))?; } else { diff --git a/src/library/status.rs b/src/library/status.rs index a8b38f7..74cee23 100644 --- a/src/library/status.rs +++ b/src/library/status.rs @@ -56,11 +56,14 @@ pub fn dvs_status(files: &Vec) -> Result> { if !path.exists() {status = String::from("not-present")} else { // get whether file was hashable and file hash - let file_hash = hash::get_file_hash(&path); // will return "" if not hashable, which won't match metadata.file_hash - if file_hash == metadata.file_hash { - status = String::from("up-to-date") - } - // else, the file exists, but the hash isn't up to date, so still with default: out-of-date + match hash::get_file_hash(&path) { + Some(file_hash) => { + if file_hash == metadata.file_hash { + status = String::from("up-to-date") + } + } + None => (), + }; } // assemble info into JsonFileResult diff --git a/src/main.rs b/src/main.rs index a9b3592..b7067f7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,10 +30,6 @@ fn main() -> Result<()> { test2.metadata().unwrap().permissions().set_mode(0o777); test3.metadata().unwrap().permissions().set_mode(0o777); - // let _test1_mode = test1.metadata().unwrap().permissions().mode() & 0o777; - // let _test2_mode = test2.metadata().unwrap().permissions().mode() & 0o777; - // let _test3_mode = test3.metadata().unwrap().permissions().mode() & 0o777; - // dvs init src/test_directory_storage --mode=0o764 --group=datascience let storage_dir = PathBuf::from(r"src/test_directory_storage"); let new_mode = 0o776; @@ -43,18 +39,9 @@ fn main() -> Result<()> { // dvs add src/test_directory/test1.txt src/test_directory/test2.txt src/test_directory/test3.txt "assembled data" let files: Vec = vec![test1_path.clone(), test2_path.clone(), test3_path.clone()]; let message = String::from("assembled data"); - add::dvs_add(&files, &message)?; - - // TODO: check permissions and group - // let test1_storage = PathBuf::from("src/test_directory_storage/7f/08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe"); - // let test2_storage = PathBuf::from("src/test_directory_storage/d3/9dfa7e18189f9d4cacabdaffc941191508ffac753e9eafa28155c154d76d5d"); - // let test3_storage = PathBuf::from("src/test_directory_storage/8e/287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4"); - // let test1_mode_new = test1_storage.metadata().unwrap().permissions().mode() & 0o777; - // let test2_mode_new = test2_storage.metadata().unwrap().permissions().mode() & 0o777; - // let test3_mode_new = test3_storage.metadata().unwrap().permissions().mode() & 0o777; - // assert_eq!(0o777, test1_mode_new); - // assert_eq!(0o777, test2_mode_new); - // assert_eq!(0o777, test3_mode_new); + let added_files = add::dvs_add(&files, &message)?; + let added_files_string = serde_json::to_string_pretty(&added_files).unwrap(); + println!("new add:\n{added_files_string}"); // remove one of the files fs::remove_file(&test1_path)?; @@ -77,15 +64,32 @@ fn main() -> Result<()> { // dvs add rc/test_directory/test2.txt "assembled data again" let message = String::from("assembled data again"); - add::dvs_add(&vec![test2_path.clone()], &message)?; + let added_files = add::dvs_add(&files, &message)?; + let added_files_string = serde_json::to_string_pretty(&added_files).unwrap(); + println!("new add:\n{added_files_string}"); // dvs status src/test_directory/test1.txt src/test_directory/test2.txt src/test_directory/test3.txt let status = status::dvs_status(&vec![test1_path.clone(), test2_path.clone(), test3_path.clone()])?; let status_string = serde_json::to_string_pretty(&status).unwrap(); println!("new status:\n{status_string}"); + + + + // delete everything fs::remove_file(&test1_path)?; fs::remove_file(&test2_path)?; fs::remove_file(&test3_path)?; + fs::remove_file(PathBuf::from("src/test_directory_storage/7f/08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe"))?; + fs::remove_file(PathBuf::from("src/test_directory_storage/8e/287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4"))?; + fs::remove_file(PathBuf::from("src/test_directory_storage/20/fa09e17c49b68fd9606fa736b03b5115059cbbbf3090d89eb50e4da044f028"))?; + fs::remove_file(PathBuf::from("src/test_directory_storage/d3/9dfa7e18189f9d4cacabdaffc941191508ffac753e9eafa28155c154d76d5d"))?; + fs:: remove_dir(PathBuf::from("src/test_directory_storage/7f"))?; + fs:: remove_dir(PathBuf::from("src/test_directory_storage/8e"))?; + fs:: remove_dir(PathBuf::from("src/test_directory_storage/20"))?; + fs:: remove_dir(PathBuf::from("src/test_directory_storage/d3"))?; + fs::remove_file(PathBuf::from("src/test_directory/.gitignore"))?; + fs::remove_file(PathBuf::from("dvs.yaml"))?; + Ok(()) } \ No newline at end of file diff --git a/src/test_directory/.gitignore b/src/test_directory/.gitignore deleted file mode 100644 index a018ab5..0000000 --- a/src/test_directory/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ - - -# Devious entry -/test1.txt - - -# Devious entry -/test2.txt - - -# Devious entry -/test3.txt diff --git a/src/test_directory/test1.txt.dvsmeta b/src/test_directory/test1.txt.dvsmeta index af7eb83..898e2c3 100644 --- a/src/test_directory/test1.txt.dvsmeta +++ b/src/test_directory/test1.txt.dvsmeta @@ -1,7 +1,7 @@ { "file_hash": "7f08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe", "file_size": 13, - "time_stamp": "2024-03-29 16:42:17.409693598 -04:00", - "message": "assembled data", + "time_stamp": "2024-04-01 13:09:35.115181573 -04:00", + "message": "assembled data again", "saved_by": "jenna" } \ No newline at end of file diff --git a/src/test_directory/test2.txt.dvsmeta b/src/test_directory/test2.txt.dvsmeta index 178da1a..378608b 100644 --- a/src/test_directory/test2.txt.dvsmeta +++ b/src/test_directory/test2.txt.dvsmeta @@ -1,7 +1,7 @@ { "file_hash": "20fa09e17c49b68fd9606fa736b03b5115059cbbbf3090d89eb50e4da044f028", "file_size": 26, - "time_stamp": "2024-03-29 16:42:17.416385039 -04:00", + "time_stamp": "2024-04-01 13:09:35.115493138 -04:00", "message": "assembled data again", "saved_by": "jenna" } \ No newline at end of file diff --git a/src/test_directory/test3.txt.dvsmeta b/src/test_directory/test3.txt.dvsmeta index f35f412..f54aa1d 100644 --- a/src/test_directory/test3.txt.dvsmeta +++ b/src/test_directory/test3.txt.dvsmeta @@ -1,7 +1,7 @@ { "file_hash": "8e287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4", "file_size": 13, - "time_stamp": "2024-03-29 16:42:17.410281271 -04:00", - "message": "assembled data", + "time_stamp": "2024-04-01 13:09:35.115711679 -04:00", + "message": "assembled data again", "saved_by": "jenna" } \ No newline at end of file diff --git a/src/test_directory_storage/20/fa09e17c49b68fd9606fa736b03b5115059cbbbf3090d89eb50e4da044f028 b/src/test_directory_storage/20/fa09e17c49b68fd9606fa736b03b5115059cbbbf3090d89eb50e4da044f028 deleted file mode 100755 index e426817..0000000 --- a/src/test_directory_storage/20/fa09e17c49b68fd9606fa736b03b5115059cbbbf3090d89eb50e4da044f028 +++ /dev/null @@ -1,2 +0,0 @@ -Hello, test2! -added a line \ No newline at end of file diff --git a/src/test_directory_storage/7f/08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe b/src/test_directory_storage/7f/08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe deleted file mode 100755 index 1873396..0000000 --- a/src/test_directory_storage/7f/08b8682ee8258389605201d65ed6a9104eed809c000d7975186bc4cd8a3efe +++ /dev/null @@ -1 +0,0 @@ -Hello, test1! \ No newline at end of file diff --git a/src/test_directory_storage/8e/287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4 b/src/test_directory_storage/8e/287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4 deleted file mode 100755 index 59fe671..0000000 --- a/src/test_directory_storage/8e/287466df9f3e8b1a2bd177ba15efe111aae572ea8859bb24557cfb4418a5b4 +++ /dev/null @@ -1 +0,0 @@ -Hello, test3! \ No newline at end of file diff --git a/src/test_directory_storage/d3/9dfa7e18189f9d4cacabdaffc941191508ffac753e9eafa28155c154d76d5d b/src/test_directory_storage/d3/9dfa7e18189f9d4cacabdaffc941191508ffac753e9eafa28155c154d76d5d deleted file mode 100755 index 0223a44..0000000 --- a/src/test_directory_storage/d3/9dfa7e18189f9d4cacabdaffc941191508ffac753e9eafa28155c154d76d5d +++ /dev/null @@ -1 +0,0 @@ -Hello, test2! \ No newline at end of file