Introduce service layer for holding erporter specific logics

- Add service test
- Update collector test
- Make collector send available metrics
instead of skipping them in case of errors

Make service FS and Object stat methods not return error

.talismanrc

@@ -1,6 +1,3 @@
-
 fileignoreconfig:
-- filename: pkg/internal/mocks/ssh_key.go
-  checksum: 79a7d1dfe0d71923bf55d108093ab8b841783a5dce3f5a73d6f246dc9a9aa990
 - filename: pkg/utils/utils_test.go
-  checksum: b58ec705d049b542883f77ef0ad56e4070103e16648f4c85ec5e2ad2f3c3667f
+  checksum: 6b68e750c9df29c3ef6fab8d34be92cadefc9985ddcbcf1fef2ba1c23f93ac0c

Makefile

@@ -36,7 +36,7 @@ test: ## run unit tests
 run: ## run the app
 	go run main.go
 
-check: install-deps fmt lint test ## runs fmt, lint, test
+check: install-deps fmt lint mocks test ## runs fmt, lint, test
 
 install-mockgen: ## install mockgen
 	go get github.com/golang/mock/mockgen@$(MOCKGEN_VERSION)

README.md

@@ -1,9 +1,5 @@
 # sftp-exporter (W.I.P)
 
-## TODO
+## Grafana
 
-- [x] Create grafana dashbiard and test the exporter
-- [ ] Add missing tests
-- [ ] Publish dashboard
-- [ ] Update readme
-- [ ] Setup CI/CD
+[Grafana Dashoard](https://grafana.com/grafana/dashboards/12828)

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/golang/mock v1.4.3
+	github.com/kr/fs v0.1.0
 	github.com/pkg/sftp v1.11.0
 	github.com/prometheus/client_golang v0.9.3
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90

main.go

@@ -5,6 +5,7 @@ import "github.com/arunvelsriram/sftp-exporter/cmd"
 //go:generate mkdir -p pkg/internal/mocks
 //go:generate mockgen -source pkg/config/config.go -destination pkg/internal/mocks/config.go -package mocks
 //go:generate mockgen -source pkg/client/sftp_client.go -destination pkg/internal/mocks/sftp_client.go -package mocks
+//go:generate mockgen -source pkg/service/sftp_service.go -destination pkg/internal/mocks/sftp_service.go -package mocks
 func main() {
 	cmd.Execute()
 }

pkg/client/sftp_client.go

@@ -3,124 +3,72 @@ package client
 import (
 	"fmt"
 
-	"github.com/arunvelsriram/sftp-exporter/pkg/utils"
-
-	log "github.com/sirupsen/logrus"
-
 	"github.com/arunvelsriram/sftp-exporter/pkg/config"
-	"github.com/arunvelsriram/sftp-exporter/pkg/model"
+	"github.com/arunvelsriram/sftp-exporter/pkg/utils"
+	"github.com/kr/fs"
 	"github.com/pkg/sftp"
+	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
 )
 
 type (
 	SFTPClient interface {
-		Close()
-		FSStats() (model.FSStats, error)
-		ObjectStats() (model.ObjectStats, error)
+		Connect() error
+		Close() error
+		StatVFS(path string) (*sftp.StatVFS, error)
+		Walk(root string) *fs.Walker
 	}
 
-	defaultSFTPClient struct {
-		sshClient  *ssh.Client
-		sftpClient *sftp.Client
-		config     config.Config
+	sftpClient struct {
+		*sftp.Client
+		sshClient *ssh.Client
+		config    config.Config
 	}
 )
 
-func (d defaultSFTPClient) Close() {
-	if err := d.sftpClient.Close(); err != nil {
+func (s *sftpClient) Close() error {
+	if err := s.Client.Close(); err != nil {
 		log.WithFields(log.Fields{
 			"event": "closing SFTP connection"},
 		).Error(err)
 	}
-	if err := d.sshClient.Close(); err != nil {
+	if err := s.sshClient.Close(); err != nil {
 		log.WithFields(log.Fields{
 			"event": "closing SSH connection"},
 		).Error(err)
 	}
+	return nil
 }
 
-func (d defaultSFTPClient) FSStats() (model.FSStats, error) {
-	paths := d.config.GetSFTPPaths()
-	fsStats := make([]model.FSStat, len(paths))
-	for i, path := range paths {
-		statVFS, err := d.sftpClient.StatVFS(path)
-		if err != nil {
-			return nil, err
-		}
-		fsStats[i] = model.FSStat{
-			Path:       path,
-			TotalSpace: float64(statVFS.TotalSpace()),
-			FreeSpace:  float64(statVFS.FreeSpace()),
-		}
-	}
-	return fsStats, nil
-}
-
-func (d defaultSFTPClient) ObjectStats() (model.ObjectStats, error) {
-	paths := d.config.GetSFTPPaths()
-	objectStats := make([]model.ObjectStat, len(paths))
-	for i, path := range paths {
-		walker := d.sftpClient.Walk(path)
-		var size int64
-		count := 0
-		for walker.Step() {
-			if err := walker.Err(); err != nil {
-				log.WithFields(log.Fields{
-					"event": "collecting object stats",
-					"path":  path,
-				}).Error(err)
-				continue
-			}
-
-			if walker.Stat().IsDir() {
-				continue
-			}
-			size += walker.Stat().Size()
-			count++
-		}
-
-		objectStats[i] = model.ObjectStat{
-			Path:        path,
-			ObjectCount: float64(count),
-			ObjectSize:  float64(size),
-		}
-	}
-
-	return objectStats, nil
-}
-
-func NewSFTPClient(cfg config.Config) (SFTPClient, error) {
-	addr := fmt.Sprintf("%s:%d", cfg.GetSFTPHost(), cfg.GetSFTPPort())
-	auth, err := utils.SSHAuthMethods(cfg.GetSFTPPass(), cfg.GetSFTPKey(), cfg.GetSFTPKeyPassphrase())
+func (s *sftpClient) Connect() (err error) {
+	addr := fmt.Sprintf("%s:%d", s.config.GetSFTPHost(), s.config.GetSFTPPort())
+	auth, err := utils.SSHAuthMethods(s.config.GetSFTPPass(), s.config.GetSFTPKey(), s.config.GetSFTPKeyPassphrase())
 	if err != nil {
 		log.Error("unable to get SSH auth methods")
-		return nil, err
+		return err
 	}
 	clientConfig := &ssh.ClientConfig{
-		User:            cfg.GetSFTPUser(),
+		User:            s.config.GetSFTPUser(),
 		Auth:            auth,
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
-
-	sshClient, err := ssh.Dial("tcp", addr, clientConfig)
+	s.sshClient, err = ssh.Dial("tcp", addr, clientConfig)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	sftpClient, err := sftp.NewClient(sshClient)
+	s.Client, err = sftp.NewClient(s.sshClient)
 	if err != nil {
-		if err := sshClient.Close(); err != nil {
+		if err := s.sshClient.Close(); err != nil {
 			log.WithFields(log.Fields{
-				"event": "closing SFTP connection"},
+				"event": "closing SSH connection"},
 			).Error(err)
 		}
-		return nil, err
+		return err
 	}
+	return nil
+}
 
-	return defaultSFTPClient{
-		sshClient:  sshClient,
-		sftpClient: sftpClient,
-		config:     cfg,
-	}, nil
+func NewSFTPClient(cfg config.Config) SFTPClient {
+	return &sftpClient{config: cfg}
 }

pkg/collector/sftp_collector.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"github.com/arunvelsriram/sftp-exporter/pkg/service"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/arunvelsriram/sftp-exporter/pkg/client"
@@ -49,8 +50,8 @@ var (
 type CreateClientFn func(config.Config) (client.SFTPClient, error)
 
 type SFTPCollector struct {
-	config         config.Config
-	createClientFn CreateClientFn
+	config      config.Config
+	sftpService service.SFTPService
 }
 
 func (s SFTPCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -62,36 +63,30 @@ func (s SFTPCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (s SFTPCollector) Collect(ch chan<- prometheus.Metric) {
-	sftpClient, err := s.createClientFn(s.config)
-	if err != nil {
+	if err := s.sftpService.Connect(); err != nil {
 		ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, 0)
-		log.WithFields(log.Fields{"event": "creating SFTP sftpClient"}).Error(err)
+		log.WithFields(log.Fields{"event": "creating SFTP connection"}).Error(err)
 		return
 	}
-	defer sftpClient.Close()
+	defer s.sftpService.Close()
 	ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, 1)
 
-	fsStats, err := sftpClient.FSStats()
-	if err != nil {
-		log.WithFields(log.Fields{"event": "getting FS stats"}).Error(err)
-	} else {
-		for _, stat := range fsStats {
-			ch <- prometheus.MustNewConstMetric(fsTotalSpace, prometheus.GaugeValue, stat.TotalSpace, stat.Path)
-			ch <- prometheus.MustNewConstMetric(fsFreeSpace, prometheus.GaugeValue, stat.FreeSpace, stat.Path)
-		}
+	fsStats := s.sftpService.FSStats()
+	for _, stat := range fsStats {
+		ch <- prometheus.MustNewConstMetric(fsTotalSpace, prometheus.GaugeValue, stat.TotalSpace, stat.Path)
+		ch <- prometheus.MustNewConstMetric(fsFreeSpace, prometheus.GaugeValue, stat.FreeSpace, stat.Path)
 	}
 
-	objectStats, err := sftpClient.ObjectStats()
-	if err != nil {
-		log.WithFields(log.Fields{"event": "getting object stats"}).Error(err)
-	} else {
-		for _, stat := range objectStats {
-			ch <- prometheus.MustNewConstMetric(objectCount, prometheus.GaugeValue, stat.ObjectCount, stat.Path)
-			ch <- prometheus.MustNewConstMetric(objectSize, prometheus.GaugeValue, stat.ObjectSize, stat.Path)
-		}
+	objectStats := s.sftpService.ObjectStats()
+	for _, stat := range objectStats {
+		ch <- prometheus.MustNewConstMetric(objectCount, prometheus.GaugeValue, stat.ObjectCount, stat.Path)
+		ch <- prometheus.MustNewConstMetric(objectSize, prometheus.GaugeValue, stat.ObjectSize, stat.Path)
 	}
 }
 
-func NewSFTPCollector(cfg config.Config, fn CreateClientFn) prometheus.Collector {
-	return SFTPCollector{config: cfg, createClientFn: fn}
+func NewSFTPCollector(cfg config.Config, s service.SFTPService) prometheus.Collector {
+	return SFTPCollector{
+		config:      cfg,
+		sftpService: s,
+	}
 }