Submodules

.circleci/config.yml

@@ -98,7 +98,7 @@ jobs:
           name: Run Unit Tests
           command: make test
       - codecov/upload:
-          file: ./coverage.txt
+          file: '**/coverage.txt'
 
   test-macos:
     executor: mac
@@ -138,7 +138,31 @@ jobs:
       - run:
           name: Unit Tests
           shell: powershell.exe
-          command: go test -race -coverprofile coverage.txt -coverpkg ./... ./...
+          command: >
+            go test -race ./...; `
+              Push-Location; Set-Location ./cmd/stanza; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/file; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/journald; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/windows; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/k8sevent; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/tcp; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/input/udp; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/output/elastic; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/output/file; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/output/googlecloud; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/output/stdout; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/parser/json; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/parser/regex; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/parser/severity; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/parser/syslog; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/parser/time; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/filter; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/hostmetadata; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/k8smetadata; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/metadata; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/ratelimit; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/restructure; go test -race ./...; Pop-Location; `
+              Push-Location; Set-Location ./operator/builtin/transformer/router; go test -race ./...; Pop-Location; 
 
   run-benchmark:
     parameters:
@@ -178,7 +202,7 @@ jobs:
           command: mkdir {bin,out,tmp}
       - run:
           name: Build Stanza Agent
-          command: GOPROXY=direct go build -v -o ./bin/stanza ./
+          command: (cd ./cmd/stanza && GO111MODULE=on GOPROXY=direct go build -v -o ../../bin/stanza ./)
       - run:
           name: Build Log Bench
           command: GOPROXY=direct go get github.com/observiq/amazon-log-agent-benchmark-tool/cmd/logbench/ &&

.gitignore

@@ -1,7 +1,8 @@
 dev/*
 .DS_Store
 tmp/*
-coverage.txt
-coverage.html
+local/*
+**/coverage.txt
+**/coverage.html
 artifacts/*
 .vscode/*

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Split buffers into buffers and flushers for better modularity
 - New memory buffer design for a uniform interface between disk and memory buffers
+- Most operators are now dedicated modules, so that they may be imported individually
 
 ## [0.9.14] - 2020-08-31
 ### Fixed

Makefile

@@ -3,6 +3,10 @@ GOARCH=$(shell go env GOARCH)
 
 GIT_SHA=$(shell git rev-parse --short HEAD)
 
+PROJECT_ROOT = $(shell pwd)
+ARTIFACTS = ${PROJECT_ROOT}/artifacts
+ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort )
+
 BUILD_INFO_IMPORT_PATH=github.com/observiq/stanza/internal/version
 BUILD_X1=-X $(BUILD_INFO_IMPORT_PATH).GitHash=$(GIT_SHA)
 ifdef VERSION
@@ -18,12 +22,26 @@ install-tools:
 
 .PHONY: test
 test:
-	go test -race -coverprofile coverage.txt -coverpkg ./... ./...
-	go tool cover -html=coverage.txt -o coverage.html
+	$(MAKE) for-all CMD="go test -race -coverprofile coverage.txt -coverpkg ./... ./..."
 
 .PHONY: bench
 bench:
-	go test -run=NONE -bench '.*' ./... -benchmem
+	$(MAKE) for-all CMD="go test -run=NONE -bench '.*' ./... -benchmem"
+
+.PHONY: clean
+clean:
+	rm -fr ./artifacts
+	$(MAKE) for-all CMD="rm -f coverage.txt coverage.html"
+
+.PHONY: tidy
+tidy:
+	$(MAKE) for-all CMD="go mod tidy"
+
+.PHONY: listmod
+listmod:
+	@set -e; for dir in $(ALL_MODULES); do \
+		(echo "$${dir}"); \
+	done
 
 .PHONY: lint
 lint:
@@ -35,11 +53,11 @@ generate:
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 go build -o ./artifacts/stanza_$(GOOS)_$(GOARCH) $(BUILD_INFO) .
+	(cd ./cmd/stanza && CGO_ENABLED=0 go build -o ../../artifacts/stanza_$(GOOS)_$(GOARCH) $(BUILD_INFO) .)
 
 .PHONY: install
 install:
-	CGO_ENABLED=0 go install $(BUILD_INFO) .
+	(cd ./cmd/stanza && CGO_ENABLED=0 go install $(BUILD_INFO) .)
 
 .PHONY: build-all
 build-all: build-darwin-amd64 build-linux-amd64 build-windows-amd64
@@ -55,3 +73,10 @@ build-linux-amd64:
 .PHONY: build-windows-amd64
 build-windows-amd64:
 	@GOOS=windows GOARCH=amd64 $(MAKE) build
+
+.PHONY: for-all
+for-all:
+	@$${CMD}
+	@set -e; for dir in $(ALL_MODULES); do \
+	  (cd "$${dir}" && $${CMD} ); \
+	done

agent/agent.go

@@ -1,134 +1,47 @@
 package agent
 
 import (
-	"fmt"
-	"os"
-	"path/filepath"
 	"sync"
-	"time"
 
-	"github.com/observiq/stanza/errors"
-	"github.com/observiq/stanza/operator"
-	_ "github.com/observiq/stanza/operator/builtin" // register operators
+	"github.com/observiq/stanza/database"
 	"github.com/observiq/stanza/pipeline"
-	"go.etcd.io/bbolt"
 	"go.uber.org/zap"
 )
 
 // LogAgent is an entity that handles log monitoring.
 type LogAgent struct {
-	database operator.Database
-	pipeline *pipeline.Pipeline
+	database database.Database
+	pipeline pipeline.Pipeline
 
 	startOnce sync.Once
 	stopOnce  sync.Once
 
 	*zap.SugaredLogger
 }
 
-// Start will start the log monitoring process.
+// Start will start the log monitoring process
 func (a *LogAgent) Start() (err error) {
 	a.startOnce.Do(func() {
 		err = a.pipeline.Start()
 		if err != nil {
 			return
 		}
-		a.Info("Agent started")
 	})
 	return
 }
 
-// Stop will stop the log monitoring process.
-func (a *LogAgent) Stop() {
+// Stop will stop the log monitoring process
+func (a *LogAgent) Stop() (err error) {
 	a.stopOnce.Do(func() {
-		a.pipeline.Stop()
-		a.database.Close()
-		a.Info("Agent stopped")
-	})
-}
-
-// OpenDatabase will open and create a database.
-func OpenDatabase(file string) (operator.Database, error) {
-	if file == "" {
-		return operator.NewStubDatabase(), nil
-	}
-
-	if _, err := os.Stat(filepath.Dir(file)); err != nil {
-		if os.IsNotExist(err) {
-			err := os.MkdirAll(filepath.Dir(file), 0755)
-			if err != nil {
-				return nil, fmt.Errorf("creating database directory: %s", err)
-			}
-		} else {
-			return nil, err
+		err = a.pipeline.Stop()
+		if err != nil {
+			return
 		}
-	}
-
-	options := &bbolt.Options{Timeout: 1 * time.Second}
-	return bbolt.Open(file, 0666, options)
-}
-
-// LogAgentBuilder is a construct used to build a log agent
-type LogAgentBuilder struct {
-	cfg           *Config
-	logger        *zap.SugaredLogger
-	pluginDir     string
-	databaseFile  string
-	defaultOutput operator.Operator
-}
-
-// NewBuilder creates a new LogAgentBuilder
-func NewBuilder(cfg *Config, logger *zap.SugaredLogger) *LogAgentBuilder {
-	return &LogAgentBuilder{
-		cfg:    cfg,
-		logger: logger,
-	}
-}
-
-// WithPluginDir adds the specified plugin directory when building a log agent
-func (b *LogAgentBuilder) WithPluginDir(pluginDir string) *LogAgentBuilder {
-	b.pluginDir = pluginDir
-	return b
-}
 
-// WithDatabaseFile adds the specified database file when building a log agent
-func (b *LogAgentBuilder) WithDatabaseFile(databaseFile string) *LogAgentBuilder {
-	b.databaseFile = databaseFile
-	return b
-}
-
-// WithDefaultOutput adds a default output when building a log agent
-func (b *LogAgentBuilder) WithDefaultOutput(defaultOutput operator.Operator) *LogAgentBuilder {
-	b.defaultOutput = defaultOutput
-	return b
-}
-
-// Build will build a new log agent using the values defined on the builder
-func (b *LogAgentBuilder) Build() (*LogAgent, error) {
-	database, err := OpenDatabase(b.databaseFile)
-	if err != nil {
-		return nil, errors.Wrap(err, "open database")
-	}
-
-	registry, err := operator.NewPluginRegistry(b.pluginDir)
-	if err != nil {
-		return nil, errors.Wrap(err, "load plugin registry")
-	}
-
-	buildContext := operator.BuildContext{
-		Logger:         b.logger,
-		PluginRegistry: registry,
-		Database:       database,
-	}
-
-	pipeline, err := b.cfg.Pipeline.BuildPipeline(buildContext, b.defaultOutput)
-	if err != nil {
-		return nil, err
-	}
-
-	return &LogAgent{
-		pipeline:      pipeline,
-		database:      database,
-		SugaredLogger: b.logger,
-	}, nil
+		err = a.database.Close()
+		if err != nil {
+			return
+		}
+	})
+	return
 }