diff --git a/src/config.rs b/src/config.rs index 5187a29..9e2ed32 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,7 +4,7 @@ use crate::filter::{Filter, NoFilter}; use crate::plugin::console::ConsoleAppender; use crate::plugin::file::FileAppender; use crate::plugin::file_loop::FileLoopAppender; -use crate::plugin::file_split::{FileSplitAppender, Packer, RawFile, RollingType, SplitFile}; +use crate::plugin::file_split::{FileSplitAppender, Packer, RawFile, Rolling, SplitFile}; use crate::FastLogFormat; use dark_std::sync::SyncVec; use log::LevelFilter; @@ -90,16 +90,15 @@ impl Config { self } /// add a FileSplitAppender - pub fn file_split( + pub fn file_split( self, file_path: &str, temp_size: LogSize, - rolling_type: RollingType, + rolling_type: R, packer: P, ) -> Self { self.appends.push(Mutex::new(Box::new( - FileSplitAppender::::new(file_path, temp_size, rolling_type, Box::new(packer)) - .unwrap(), + FileSplitAppender::::new(file_path, temp_size, rolling_type, packer).unwrap(), ))); self } @@ -118,16 +117,19 @@ impl Config { // LogPacker {}, // ), // ); - pub fn split( + pub fn split< + F: SplitFile + 'static, + R: Rolling + Sync + 'static, + P: Packer + Sync + 'static, + >( self, file_path: &str, temp_size: LogSize, - rolling_type: RollingType, + rolling_type: R, packer: P, ) -> Self { self.appends.push(Mutex::new(Box::new( - FileSplitAppender::::new(file_path, temp_size, rolling_type, Box::new(packer)) - .unwrap(), + FileSplitAppender::::new(file_path, temp_size, rolling_type, packer).unwrap(), ))); self } diff --git a/src/fast_log.rs b/src/fast_log.rs index 1e6f2a2..265e441 100644 --- a/src/fast_log.rs +++ b/src/fast_log.rs @@ -113,19 +113,19 @@ pub fn init(config: Config) -> Result<&'static Logger, LogError> { sender_vec.push(s); recever_vec.push((r, a)); } - for (recever, appender) in recever_vec { + for (receiver, appender) in recever_vec { spawn(move || { let mut exit = false; loop { let mut remain = vec![]; - if recever.len() == 0 { - if let Ok(msg) = recever.recv() { + if receiver.len() == 0 { + if let Ok(msg) = receiver.recv() { remain.push(msg); } } //recv all loop { - match recever.try_recv() { + match receiver.try_recv() { Ok(v) => { remain.push(v); } diff --git a/src/plugin/file_loop.rs b/src/plugin/file_loop.rs index 8dd65c3..3054d06 100644 --- a/src/plugin/file_loop.rs +++ b/src/plugin/file_loop.rs @@ -1,7 +1,7 @@ use crate::appender::{FastLogRecord, LogAppender}; use crate::consts::LogSize; use crate::error::LogError; -use crate::plugin::file_split::{FileSplitAppender, RollingType, SplitFile}; +use crate::plugin::file_split::{FileSplitAppender, RollingNum, SplitFile}; use crate::plugin::packer::LogPacker; /// Single logs are stored in rolling mode by capacity @@ -15,8 +15,8 @@ impl FileLoopAppender { file: FileSplitAppender::::new( log_file_path, size, - RollingType::KeepNum(1), - Box::new(LogPacker {}), + RollingNum { num: 1 }, + LogPacker {}, )?, }) } diff --git a/src/plugin/file_split.rs b/src/plugin/file_split.rs index 89eb305..43b7a0f 100644 --- a/src/plugin/file_split.rs +++ b/src/plugin/file_split.rs @@ -6,8 +6,10 @@ use fastdate::DateTime; use std::cell::RefCell; use std::fs::{DirEntry, File, OpenOptions}; use std::io::{Seek, SeekFrom, Write}; +use std::ops::Deref; use std::path::Path; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use std::time::Duration; pub trait SplitFile: Send { @@ -100,22 +102,21 @@ pub trait Packer: Send { /// split log file allow pack compress log /// Memory space swop running time , reduces the number of repeated queries for IO pub struct FileSplitAppender { - dir_path: String, file: F, + dir_path: String, sender: Sender, temp_size: LogSize, - rolling_type: RollingType, //cache data temp_bytes: AtomicUsize, temp_name: String, } impl FileSplitAppender { - pub fn new( + pub fn new( file_path: &str, temp_size: LogSize, - rolling_type: RollingType, - packer: Box, + rolling: R, + packer: P, ) -> Result, LogError> { let temp_name = { let buf = Path::new(&file_path); @@ -154,7 +155,7 @@ impl FileSplitAppender { temp_bytes.store(offset, Ordering::Relaxed); file.seek(SeekFrom::Start(temp_bytes.load(Ordering::Relaxed) as u64)); let (sender, receiver) = chan(None); - spawn_saver(temp_name.clone(), receiver, packer); + spawn_saver(temp_name.clone(), receiver, rolling, packer); Ok(Self { temp_bytes, dir_path: dir_path.to_string(), @@ -162,7 +163,6 @@ impl FileSplitAppender { sender, temp_size, temp_name, - rolling_type, }) } /// send data make an pack,and truncate data when finish. @@ -208,7 +208,6 @@ impl FileSplitAppender { std::fs::copy(&first_file_path, &new_log_name); self.sender.send(LogPack { dir: self.dir_path.clone(), - rolling: self.rolling_type.clone(), new_log_name: new_log_name, }); self.truncate(); @@ -224,25 +223,42 @@ impl FileSplitAppender { ///log data pack pub struct LogPack { pub dir: String, - pub rolling: RollingType, pub new_log_name: String, } -///rolling keep type -#[derive(Copy, Clone, Debug)] -pub enum RollingType { - /// keep All of log packs - All, - /// keep by Time Duration, - /// for example: - /// // keep one day log pack - /// (Duration::from_secs(24 * 3600)) - KeepTime(Duration), - /// keep log pack num(.log,.zip.lz4...more) - KeepNum(i64), +impl LogPack { + /// write an Pack to zip file + pub fn do_pack(mut self, packer: &P) -> Result { + let log_file_path = self.new_log_name.as_str(); + if log_file_path.is_empty() { + return Err(self); + } + let log_file = OpenOptions::new().read(true).open(log_file_path); + if log_file.is_err() { + return Err(self); + } + //make + let r = packer.do_pack(log_file.unwrap(), log_file_path); + if r.is_err() && packer.retry() > 0 { + let mut retry = 1; + while let Err(packs) = self.do_pack(packer) { + self = packs; + retry += 1; + if retry > packer.retry() { + break; + } + } + } + if let Ok(b) = r { + return Ok(b); + } + return Ok(false); + } } -impl RollingType { +pub trait Rolling: Send { + /// return removed + fn do_rolling(&self, dir: &str, temp_name: &str) -> i64; fn read_paths(&self, dir: &str, temp_name: &str) -> Vec { let base_name = get_base_name(&Path::new(temp_name)); let paths = std::fs::read_dir(dir); @@ -267,15 +283,101 @@ impl RollingType { } } paths_vec.sort_by(|a, b| b.file_name().cmp(&a.file_name())); - // if let Some(v) = temp_file { - // paths_vec.push(v); - // } return paths_vec; } return vec![]; } - pub fn do_rolling(&self, temp_name: &str, dir: &str) -> i64 { + /// parse `temp2023-07-20T10-13-17.452247.log` + fn file_name_parse_time(name: &str, temp_name: &str) -> Option + where + Self: Sized, + { + let base_name = get_base_name(&Path::new(temp_name)); + if name.starts_with(&base_name) { + let mut time_str = name.trim_start_matches(&base_name).to_string(); + if let Some(v) = time_str.rfind(".") { + time_str = time_str[0..v].to_string(); + } + let time = DateTime::parse("YYYY-MM-DDThh:mm:ss.000000", &time_str); + if let Ok(time) = time { + return Some(time); + } + } + return None; + } +} + +///rolling keep type +#[deprecated(note = "use RollingAll,RollingNum,RollingTime replace this")] +#[derive(Copy, Clone, Debug)] +pub enum RollingType { + /// keep All of log packs + #[deprecated(note = "use RollingAll,RollingNum,RollingTime replace this")] + All, + /// keep by Time Duration, + /// for example: + /// // keep one day log pack + /// (Duration::from_secs(24 * 3600)) + #[deprecated(note = "use RollingAll,RollingNum,RollingTime replace this")] + KeepTime(Duration), + /// keep log pack num(.log,.zip.lz4...more) + #[deprecated(note = "use RollingAll,RollingNum,RollingTime replace this")] + KeepNum(i64), +} + +pub struct RollingAll {} +impl Rolling for RollingAll { + fn do_rolling(&self, dir: &str, temp_name: &str) -> i64 { + 0 + } +} + +pub struct RollingNum { + pub num: i64, +} + +impl Rolling for RollingNum { + fn do_rolling(&self, dir: &str, temp_name: &str) -> i64 { + let mut removed = 0; + let paths_vec = self.read_paths(dir, temp_name); + for index in 0..paths_vec.len() { + if index >= (self.num) as usize { + let item = &paths_vec[index]; + std::fs::remove_file(item.path()); + removed += 1; + } + } + removed + } +} + +pub struct RollingTime { + pub duration: Duration, +} + +impl Rolling for RollingTime { + fn do_rolling(&self, dir: &str, temp_name: &str) -> i64 { + let mut removed = 0; + let paths_vec = self.read_paths(dir, temp_name); + let now = DateTime::now(); + for index in 0..paths_vec.len() { + let item = &paths_vec[index]; + let file_name = item.file_name(); + let name = file_name.to_str().unwrap_or("").to_string(); + if let Some(time) = Self::file_name_parse_time(&name, temp_name) { + if now.clone().sub(self.duration.clone()) > time { + std::fs::remove_file(item.path()); + removed += 1; + } + } + } + removed + } +} + +impl Rolling for RollingType { + fn do_rolling(&self, temp_name: &str, dir: &str) -> i64 { let mut removed = 0; match self { RollingType::KeepNum(n) => { @@ -307,22 +409,6 @@ impl RollingType { } removed } - - /// parse `temp2023-07-20T10-13-17.452247.log` - pub fn file_name_parse_time(name: &str, temp_name: &str) -> Option { - let base_name = get_base_name(&Path::new(temp_name)); - if name.starts_with(&base_name) { - let mut time_str = name.trim_start_matches(&base_name).to_string(); - if let Some(v) = time_str.rfind(".") { - time_str = time_str[0..v].to_string(); - } - let time = DateTime::parse("YYYY-MM-DDThh:mm:ss.000000", &time_str); - if let Ok(time) = time { - return Some(time); - } - } - return None; - } } impl LogAppender for FileSplitAppender { @@ -376,15 +462,20 @@ impl LogAppender for FileSplitAppender { } ///spawn an saver thread to save log file or zip file -fn spawn_saver(temp_name: String, r: Receiver, packer: Box) { +fn spawn_saver( + temp_name: String, + r: Receiver, + rolling: R, + packer: P, +) { std::thread::spawn(move || { loop { if let Ok(pack) = r.recv() { //do rolling - pack.rolling.do_rolling(&temp_name, &pack.dir); + rolling.do_rolling(&pack.dir, &temp_name); let log_file_path = pack.new_log_name.clone(); //do save pack - let remove = do_pack(&packer, pack); + let remove = pack.do_pack(&packer); if let Ok(remove) = remove { if remove { std::fs::remove_file(log_file_path); @@ -397,34 +488,6 @@ fn spawn_saver(temp_name: String, r: Receiver, packer: Box) }); } -/// write an Pack to zip file -pub fn do_pack(packer: &Box, mut pack: LogPack) -> Result { - let log_file_path = pack.new_log_name.as_str(); - if log_file_path.is_empty() { - return Err(pack); - } - let log_file = OpenOptions::new().read(true).open(log_file_path); - if log_file.is_err() { - return Err(pack); - } - //make - let r = packer.do_pack(log_file.unwrap(), log_file_path); - if r.is_err() && packer.retry() > 0 { - let mut retry = 1; - while let Err(packs) = do_pack(packer, pack) { - pack = packs; - retry += 1; - if retry > packer.retry() { - break; - } - } - } - if let Ok(b) = r { - return Ok(b); - } - return Ok(false); -} - fn get_base_name(path: &Path) -> String { let file_name = path .file_name() diff --git a/src/plugin/packer.rs b/src/plugin/packer.rs index f7c0b62..2490c73 100644 --- a/src/plugin/packer.rs +++ b/src/plugin/packer.rs @@ -3,6 +3,7 @@ use crate::plugin::file_split::Packer; use std::fs::File; /// keep temp{date}.log +#[derive(Clone)] pub struct LogPacker {} impl Packer for LogPacker { fn pack_name(&self) -> &'static str { diff --git a/tests/split_test.rs b/tests/split_test.rs index e7662d9..c1e6b9b 100644 --- a/tests/split_test.rs +++ b/tests/split_test.rs @@ -2,7 +2,9 @@ mod test { use fast_log::appender::{Command, FastLogRecord, LogAppender}; use fast_log::consts::LogSize; - use fast_log::plugin::file_split::{FileSplitAppender, RawFile, RollingType}; + use fast_log::plugin::file_split::{ + FileSplitAppender, RawFile, Rolling, RollingAll, RollingNum, + }; use fast_log::plugin::packer::LogPacker; use log::Level; use std::fs::remove_dir_all; @@ -15,8 +17,8 @@ mod test { let appender = FileSplitAppender::::new( "target/test/", LogSize::MB(1), - RollingType::All, - Box::new(LogPacker {}), + RollingAll {}, + LogPacker {}, ) .unwrap(); appender.do_logs(&[FastLogRecord { @@ -32,14 +34,14 @@ mod test { }]); appender.send_pack(); sleep(Duration::from_secs(1)); - let rolling_num = RollingType::KeepNum(0).do_rolling("temp.log", "target/test/"); + let rolling_num = RollingNum { num: 0 }.do_rolling("target/test/", "temp.log"); assert_eq!(rolling_num, 1); let _ = remove_dir_all("target/test/"); } #[test] fn test_file_name_parse_time() { - let t = RollingType::file_name_parse_time("temp2023-07-20T10-13-17.452247.log", "temp.log") + let t = RollingAll::file_name_parse_time("temp2023-07-20T10-13-17.452247.log", "temp.log") .unwrap(); assert_eq!(t.to_string(), "2023-07-20 10:13:17.452247"); }