Add file rotation based on file age to file output plugin

plugins/outputs/file/file.go

@@ -11,9 +11,10 @@ import (
 )
 
 type File struct {
-	Files []string
+	Files        []string
+	RotateMaxAge string
 
-	writers []io.Writer
+	writer  io.Writer
 	closers []io.Closer
 
 	serializer serializers.Serializer
@@ -23,6 +24,9 @@ var sampleConfig = `
   ## Files to write to, "stdout" is a specially handled file.
   files = ["stdout", "/tmp/metrics.out"]
 
+  ## If this is defined, files will be rotated by the time.Duration specified
+  #rotate_max_age = "1m"
+
   ## Data format to output.
   ## Each data format has its own unique set of configuration options, read
   ## more about them here:
@@ -35,29 +39,35 @@ func (f *File) SetSerializer(serializer serializers.Serializer) {
 }
 
 func (f *File) Connect() error {
+	writers := []io.Writer{}
+
 	if len(f.Files) == 0 {
 		f.Files = []string{"stdout"}
 	}
 
 	for _, file := range f.Files {
 		if file == "stdout" {
-			f.writers = append(f.writers, os.Stdout)
+			writers = append(writers, os.Stdout)
 		} else {
-			var of *os.File
+			var of io.WriteCloser
 			var err error
-			if _, err := os.Stat(file); os.IsNotExist(err) {
-				of, err = os.Create(file)
+			if f.RotateMaxAge != "" {
+				of, err = NewRotatingWriter(file, f.RotateMaxAge)
 			} else {
+				if _, err := os.Stat(file); os.IsNotExist(err) {
+					of, err = os.Create(file)
+				}
 				of, err = os.OpenFile(file, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
 			}
 
 			if err != nil {
 				return err
 			}
-			f.writers = append(f.writers, of)
+			writers = append(writers, of)
 			f.closers = append(f.closers, of)
 		}
 	}
+	f.writer = io.MultiWriter(writers...)
 	return nil
 }
 
@@ -84,19 +94,19 @@ func (f *File) Description() string {
 
 func (f *File) Write(metrics []telegraf.Metric) error {
 	var writeErr error = nil
+
 	for _, metric := range metrics {
 		b, err := f.serializer.Serialize(metric)
 		if err != nil {
 			return fmt.Errorf("failed to serialize message: %s", err)
 		}
 
-		for _, writer := range f.writers {
-			_, err = writer.Write(b)
-			if err != nil && writer != os.Stdout {
-				writeErr = fmt.Errorf("E! failed to write message: %s, %s", b, err)
-			}
+		_, err = f.writer.Write(b)
+		if err != nil && f.writer != os.Stdout {
+			writeErr = fmt.Errorf("E! failed to write message: %s, %s", b, err)
 		}
 	}
+
 	return writeErr
 }
 

plugins/outputs/file/rotatingwriter.go

@@ -0,0 +1,94 @@
+package file
+
+// Rotating things
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+)
+
+// FilePerm defines the permissions that Writer will use for all
+// the files it creates.
+var FilePerm = os.FileMode(0644)
+
+// Writer implements the io.Writer interface and writes to the
+// filename specified.  When current file age exceeds max, it is
+// renamed and a new file is created.
+type Writer struct {
+	filename   string
+	current    *os.File
+	expireTime time.Time
+	max        time.Duration
+	sync.Mutex
+}
+
+// New creates a new Writer.
+func NewRotatingWriter(filename, maxAgeInput string) (*Writer, error) {
+	maxAge, err := time.ParseDuration(maxAgeInput)
+	if err != nil {
+		return nil, err
+	}
+	l := &Writer{filename: filename, max: maxAge}
+	if err := l.openCurrent(); err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+// Write writes p to the current file, then checks to see if
+// rotation is necessary.
+func (r *Writer) Write(p []byte) (n int, err error) {
+	r.Lock()
+	defer r.Unlock()
+	n, err = r.current.Write(p)
+	if err != nil {
+		return n, err
+	}
+	if time.Now().After(r.expireTime) {
+		if err := r.rotate(); err != nil {
+			return n, err
+		}
+	}
+	return n, nil
+}
+
+// Close closes the current file.  Writer is unusable after this
+// is called.
+func (r *Writer) Close() error {
+	r.Lock()
+	defer r.Unlock()
+
+	// Rotate before closing
+	if err := r.rotate(); err != nil {
+		return err
+	}
+
+	if err := r.current.Close(); err != nil {
+		return err
+	}
+	r.current = nil
+	return nil
+}
+
+func (r *Writer) openCurrent() error {
+	var err error
+	r.current, err = os.OpenFile(r.filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, FilePerm)
+	r.expireTime = time.Now().Add(r.max)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *Writer) rotate() error {
+	if err := r.current.Close(); err != nil {
+		return err
+	}
+	rotatedFilename := fmt.Sprintf("%s-%d", r.filename, time.Now().UnixNano()) // UnixNano should be unique enough for this (up until a point)
+	if err := os.Rename(r.filename, rotatedFilename); err != nil {
+		return err
+	}
+	return r.openCurrent()
+}