diff --git a/.gitignore b/.gitignore index 8be72874..721f993e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ Cargo.lock .idea/ *.iml .vscode/ +log4rs.code-workspace \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 177a8de1..224707a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ json_format = ["serde_json"] toml_format = ["toml"] xml_format = ["serde-xml-rs"] -console_appender = ["console_writer", "simple_writer", "pattern_encoder"] +console_appender = ["parking_lot","console_writer", "simple_writer", "pattern_encoder"] file_appender = ["parking_lot", "simple_writer", "pattern_encoder"] rolling_file_appender = ["parking_lot", "simple_writer", "pattern_encoder"] compound_policy = [] @@ -32,7 +32,7 @@ console_writer = ["ansi_writer", "libc", "winapi"] simple_writer = [] threshold_filter = [] background_rotation = [] - +dedup=[] all_components = [ "console_appender", "file_appender", diff --git a/log4rs.code-workspace b/log4rs.code-workspace new file mode 100644 index 00000000..876a1499 --- /dev/null +++ b/log4rs.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/src/append/console.rs b/src/append/console.rs index 2292e9b7..85de9903 100644 --- a/src/append/console.rs +++ b/src/append/console.rs @@ -2,15 +2,8 @@ //! //! Requires the `console_appender` feature. -use log::Record; -#[cfg(feature = "file")] -use serde_derive::Deserialize; -use std::{ - error::Error, - fmt, - io::{self, Write}, -}; - +#[cfg(feature = "dedup")] +use crate::append::dedup::*; #[cfg(feature = "file")] use crate::encode::EncoderConfig; #[cfg(feature = "file")] @@ -28,7 +21,17 @@ use crate::{ }, priv_io::{StdWriter, StdWriterLock}, }; +use log::Record; +#[cfg(feature = "dedup")] +use parking_lot::Mutex; +#[cfg(feature = "file")] +use serde_derive::Deserialize; +use std::{ + error::Error, + fmt, + io::{self, Write}, +}; /// The console appender's configuration. #[cfg(feature = "file")] #[derive(Deserialize)] @@ -36,6 +39,8 @@ use crate::{ pub struct ConsoleAppenderConfig { target: Option, encoder: Option, + #[cfg(feature = "dedup")] + dedup: Option, } #[cfg(feature = "file")] @@ -112,6 +117,8 @@ impl<'a> encode::Write for WriterLock<'a> { pub struct ConsoleAppender { writer: Writer, encoder: Box, + #[cfg(feature = "dedup")] + deduper: Option>, } impl fmt::Debug for ConsoleAppender { @@ -125,6 +132,14 @@ impl fmt::Debug for ConsoleAppender { impl Append for ConsoleAppender { fn append(&self, record: &Record) -> Result<(), Box> { let mut writer = self.writer.lock(); + #[cfg(feature = "dedup")] + let _ = { + if let Some(dd) = &self.deduper { + if dd.lock().dedup(&mut writer, &*self.encoder, record)? == DedupResult::Skip { + return Ok(()); + } + } + }; self.encoder.encode(&mut writer, record)?; writer.flush()?; Ok(()) @@ -139,6 +154,8 @@ impl ConsoleAppender { ConsoleAppenderBuilder { encoder: None, target: Target::Stdout, + #[cfg(feature = "dedup")] + dedup: false, } } } @@ -146,7 +163,10 @@ impl ConsoleAppender { /// A builder for `ConsoleAppender`s. pub struct ConsoleAppenderBuilder { encoder: Option>, + target: Target, + #[cfg(feature = "dedup")] + dedup: bool, } impl ConsoleAppenderBuilder { @@ -163,6 +183,14 @@ impl ConsoleAppenderBuilder { self.target = target; self } + /// Determines if the appender will reject and count duplicate messages. + /// + /// Defaults to `false`. + #[cfg(feature = "dedup")] + pub fn dedup(mut self, dedup: bool) -> ConsoleAppenderBuilder { + self.dedup = dedup; + self + } /// Consumes the `ConsoleAppenderBuilder`, producing a `ConsoleAppender`. pub fn build(self) -> ConsoleAppender { @@ -176,12 +204,22 @@ impl ConsoleAppenderBuilder { None => Writer::Raw(StdWriter::stdout()), }, }; + #[cfg(feature = "dedup")] + let deduper = { + if self.dedup { + Some(Mutex::new(DeDuper::default())) + } else { + None + } + }; ConsoleAppender { writer, encoder: self .encoder .unwrap_or_else(|| Box::new(PatternEncoder::default())), + #[cfg(feature = "dedup")] + deduper, } } } @@ -233,6 +271,12 @@ impl Deserialize for ConsoleAppenderDeserializer { if let Some(encoder) = config.encoder { appender = appender.encoder(deserializers.deserialize(&encoder.kind, encoder.config)?); } + #[cfg(feature = "dedup")] + let _ = { + if let Some(dedup) = config.dedup { + appender = appender.dedup(dedup); + } + }; Ok(Box::new(appender.build())) } } diff --git a/src/append/dedup.rs b/src/append/dedup.rs new file mode 100644 index 00000000..cff9b55c --- /dev/null +++ b/src/append/dedup.rs @@ -0,0 +1,94 @@ +//! The dedup common handler +//! +use crate::append::Error; +use crate::encode::{Encode, Write}; +use log::Record; + +const REPEAT_COUNT: i32 = 1000; +/// dedup object to be used by deduping appender. +/// internals are private to dedup +#[derive(Default)] +pub struct DeDuper { + count: i32, + last: String, +} +#[derive(PartialEq)] +/// Used by an appender that uses dedup. +/// Indicates whether or not the current message should be output. +/// +/// sample use from console appender +/// if let Some(dd) = &self.deduper { +/// if dd.lock().dedup(&mut *file, &*self.encoder, record)? == DedupResult::Skip { +/// return Ok(()); +/// } +/// ... output the message +pub enum DedupResult { + /// skip + Skip, + /// write + Write, +} +impl DeDuper { + // emits the extra line saying 'last line repeated n times' + fn write( + w: &mut dyn Write, + encoder: &dyn Encode, + record: &Record, + n: i32, + ) -> Result<(), Box> { + if n == 1 { + encoder.encode( + w, + &Record::builder() + .args(format_args!("last message repeated, suppressing dups")) + .level(record.level()) + .target(record.target()) + .module_path_static(None) + .file_static(None) + .line(None) + .build(), + ) + } else { + encoder.encode( + w, + &Record::builder() + .args(format_args!("last message repeated {} times", n)) + .level(record.level()) + .target(record.target()) + .module_path_static(None) + .file_static(None) + .line(None) + .build(), + ) + } + } + + /// Appender calls this. + /// If it returns Skip then appender should not write + /// If it returns Write then the appender should write as per normal + pub fn dedup( + &mut self, + w: &mut dyn Write, + encoder: &dyn Encode, + record: &Record, + ) -> Result> { + let msg = format!("{}", *record.args()); + if msg == self.last { + self.count += 1; + + // every now and then keep saying we saw lots of dups + if self.count % REPEAT_COUNT == 0 || self.count == 1 { + Self::write(w, encoder, record, self.count)?; + } + Ok(DedupResult::Skip) + } else { + self.last = msg; + let svct = self.count; + self.count = 0; + if svct > 1 { + Self::write(w, encoder, record, svct)?; + } + Ok(DedupResult::Write) + } + } +} diff --git a/src/append/file.rs b/src/append/file.rs index 6ceae436..ed2b5aa0 100644 --- a/src/append/file.rs +++ b/src/append/file.rs @@ -14,6 +14,8 @@ use std::{ path::{Path, PathBuf}, }; +#[cfg(feature = "dedup")] +use crate::append::dedup::*; #[cfg(feature = "file")] use crate::encode::EncoderConfig; #[cfg(feature = "file")] @@ -31,6 +33,8 @@ pub struct FileAppenderConfig { path: String, encoder: Option, append: Option, + #[cfg(feature = "dedup")] + dedup: Option, } /// An appender which logs to a file. @@ -38,6 +42,8 @@ pub struct FileAppender { path: PathBuf, file: Mutex>>, encoder: Box, + #[cfg(feature = "dedup")] + deduper: Option>, } impl fmt::Debug for FileAppender { @@ -52,6 +58,14 @@ impl fmt::Debug for FileAppender { impl Append for FileAppender { fn append(&self, record: &Record) -> Result<(), Box> { let mut file = self.file.lock(); + #[cfg(feature = "dedup")] + let _ = { + if let Some(dd) = &self.deduper { + if dd.lock().dedup(&mut *file, &*self.encoder, record)? == DedupResult::Skip { + return Ok(()); + } + } + }; self.encoder.encode(&mut *file, record)?; file.flush()?; Ok(()) @@ -66,6 +80,8 @@ impl FileAppender { FileAppenderBuilder { encoder: None, append: true, + #[cfg(feature = "dedup")] + dedup: false, } } } @@ -74,6 +90,8 @@ impl FileAppender { pub struct FileAppenderBuilder { encoder: Option>, append: bool, + #[cfg(feature = "dedup")] + dedup: bool, } impl FileAppenderBuilder { @@ -91,6 +109,15 @@ impl FileAppenderBuilder { self } + /// Determines if the appender will reject and count duplicate messages. + /// + /// Defaults to `false`. + #[cfg(feature = "dedup")] + pub fn dedup(mut self, dedup: bool) -> FileAppenderBuilder { + self.dedup = dedup; + self + } + /// Consumes the `FileAppenderBuilder`, producing a `FileAppender`. pub fn build>(self, path: P) -> io::Result { let path = path.as_ref().to_owned(); @@ -104,9 +131,19 @@ impl FileAppenderBuilder { .create(true) .open(&path)?; + #[cfg(feature = "dedup")] + let deduper = { + if self.dedup { + Some(Mutex::new(DeDuper::default())) + } else { + None + } + }; Ok(FileAppender { path, file: Mutex::new(SimpleWriter(BufWriter::with_capacity(1024, file))), + #[cfg(feature = "dedup")] + deduper, encoder: self .encoder .unwrap_or_else(|| Box::new(PatternEncoder::default())), @@ -150,6 +187,12 @@ impl Deserialize for FileAppenderDeserializer { if let Some(append) = config.append { appender = appender.append(append); } + #[cfg(feature = "dedup")] + let _ = { + if let Some(dedup) = config.dedup { + appender = appender.dedup(dedup); + } + }; if let Some(encoder) = config.encoder { appender = appender.encoder(deserializers.deserialize(&encoder.kind, encoder.config)?); } diff --git a/src/append/mod.rs b/src/append/mod.rs index a5207c41..afd06ddc 100644 --- a/src/append/mod.rs +++ b/src/append/mod.rs @@ -16,6 +16,8 @@ use crate::filter::FilterConfig; #[cfg(feature = "console_appender")] pub mod console; +#[cfg(feature = "dedup")] +pub mod dedup; #[cfg(feature = "file_appender")] pub mod file; #[cfg(feature = "rolling_file_appender")]