xlget: added store status feature

Dockerfile.xlget

@@ -10,7 +10,8 @@ RUN apk update && apk add --no-cache git make ca-certificates && update-ca-certi
 
 # create user for service
 RUN adduser -D -g 'luids' luxlist \
-    && mkdir -p /var/lib/luids/xlist \
+    && mkdir -p /var/lib/luids/xlist/status \
+    && mkdir -p /var/lib/luids/xlist/local \
     && mkdir -p /var/cache/luids/xlist \
     && touch /var/lib/luids/xlist/.keep \
     && touch /var/cache/luids/xlist/.keep \

Dockerfile.xlistd

@@ -9,7 +9,11 @@ ARG TARGETVARIANT
 RUN apk update && apk add --no-cache git make ca-certificates && update-ca-certificates
 
 # create user for service
-RUN adduser -D -g 'luids' luxlist
+RUN adduser -D -g 'luids' luxlist \
+    && mkdir -p /var/lib/luids/xlist/status \
+    && mkdir -p /var/lib/luids/xlist/local \
+    && touch /var/lib/luids/xlist/.keep \
+    && chown -R luxlist /var/lib/luids/xlist
 
 WORKDIR /app
 
@@ -33,6 +37,7 @@ COPY --from=build-env /etc/passwd /etc/passwd
 
 COPY --from=build-env /app/bin/xlist? /bin/
 COPY --from=build-env /app/configs/docker/xlistd/*  /etc/luids/xlist/
+COPY --from=build-env /var/lib/luids /var/lib/luids
 
 USER luxlist
 

Makefile

@@ -43,8 +43,8 @@ clean:
 
 docker:
 	@echo "$(WHALE) $@"
-	DOCKER_BUILDKIT=1 docker build -t xlistd -f Dockerfile.xlistd .
-	DOCKER_BUILDKIT=1 docker build -t xlget -f Dockerfile.xlget .
+	DOCKER_BUILDKIT=1 docker build --progress=plain -t xlistd -f Dockerfile.xlistd .
+	DOCKER_BUILDKIT=1 docker build --progress=plain -t xlget -f Dockerfile.xlget .
 
 ## Targets for Makefile.release
 .PHONY: release

deployments/installer/installer_linux.sh

@@ -337,6 +337,20 @@ create_data_dir() {
 		log "$VAR_DIR/$NAME already exists"
 	fi
 
+	if [ ! -d $VAR_DIR/$NAME/status ]; then
+		do_create_dir $VAR_DIR/$NAME/status $SVC_GROUP 1770
+		[ $? -ne 0 ] && step_err && return 1
+	else
+		log "$VAR_DIR/$NAME/status already exists"
+	fi
+
+	if [ ! -d $VAR_DIR/$NAME/local ]; then
+		do_create_dir $VAR_DIR/$NAME/local $SVC_GROUP 1770
+		[ $? -ne 0 ] && step_err && return 1
+	else
+		log "$VAR_DIR/$NAME/local already exists"
+	fi
+
 	step_ok
 }
 
@@ -419,7 +433,7 @@ EOF
 
 	if [ ! -f $ETC_DIR/$NAME/services.json ]; then
 		log "creating $ETC_DIR/$NAME/services.json"
-		echo '[{"id":"root","class":"mock"}]' > $ETC_DIR/$NAME/services.json 2>>$LOG_FILE
+		echo '[{"id":"root","class":"mem"}]' > $ETC_DIR/$NAME/services.json 2>>$LOG_FILE
 		[ $? -ne 0 ] && step_err && return 1
 	else
 		log "$ETC_DIR/$NAME/services.json already exists"
@@ -431,6 +445,7 @@ EOF
 [xlget]
 outputdir = "${VAR_DIR}/${NAME}"
 cachedir  = "${CACHE_DIR}/${NAME}"
+statusdir = "${VAR_DIR}/${NAME}/status"
 
 [xlget.source]
 files     = [ "${ETC_DIR}/${NAME}/sources.json" ]

internal/config/xlget.go

@@ -19,6 +19,7 @@ type XLGetCfg struct {
 	SourceFiles []string
 	OutputDir   string
 	CacheDir    string
+	StatusDir   string
 }
 
 // SetPFlags setups posix flags for commandline configuration
@@ -36,6 +37,7 @@ func (cfg *XLGetCfg) SetPFlags(short bool, prefix string) {
 	}
 	pflag.StringVar(&cfg.OutputDir, aprefix+"outputdir", cfg.OutputDir, "Output dir.")
 	pflag.StringVar(&cfg.CacheDir, aprefix+"cachedir", cfg.CacheDir, "Cache dir.")
+	pflag.StringVar(&cfg.CacheDir, aprefix+"statusdir", cfg.CacheDir, "Status dir.")
 }
 
 // BindViper setups posix flags for commandline configuration and bind to viper
@@ -48,6 +50,7 @@ func (cfg *XLGetCfg) BindViper(v *viper.Viper, prefix string) {
 	util.BindViper(v, aprefix+"source.files")
 	util.BindViper(v, aprefix+"outputdir")
 	util.BindViper(v, aprefix+"cachedir")
+	util.BindViper(v, aprefix+"statusdir")
 }
 
 // FromViper fill values from viper
@@ -60,6 +63,7 @@ func (cfg *XLGetCfg) FromViper(v *viper.Viper, prefix string) {
 	cfg.SourceFiles = v.GetStringSlice(aprefix + "source.files")
 	cfg.OutputDir = v.GetString(aprefix + "outputdir")
 	cfg.CacheDir = v.GetString(aprefix + "cachedir")
+	cfg.StatusDir = v.GetString(aprefix + "statusdir")
 }
 
 // Empty returns true if configuration is empty
@@ -101,6 +105,11 @@ func (cfg XLGetCfg) Validate() error {
 			return errors.New("cache dir doesn't exists")
 		}
 	}
+	if cfg.StatusDir != "" {
+		if !util.DirExists(cfg.StatusDir) {
+			return errors.New("status dir doesn't exists")
+		}
+	}
 	return nil
 }
 

internal/factory/xlget.go

@@ -25,7 +25,7 @@ func XLGet(cfg *config.XLGetCfg, logger yalogi.Logger) (*xlget.Manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("loading dbfiles: %v", err)
 	}
-	manager, err := xlget.NewManager(cfg.OutputDir, cfg.CacheDir, xlget.SetLogger(logger))
+	manager, err := xlget.NewManager(cfg.OutputDir, cfg.CacheDir, cfg.StatusDir, xlget.SetLogger(logger))
 	if err != nil {
 		return nil, fmt.Errorf("creating manager: %v", err)
 	}

pkg/xlget/entries.go

@@ -282,3 +282,80 @@ func ValidateEntry(e Entry) error {
 	}
 	return nil
 }
+
+// EntryStatus stores status information
+type EntryStatus struct {
+	ID string `json:"id"`
+	// First sync and last sync
+	First time.Time `json:"first"`
+	Last  time.Time `json:"last"`
+	// Updates stores only successful syncs
+	Updates    int        `json:"updates"`
+	LastUpdate *time.Time `json:"lastupdate,omitempty"`
+	// Changes stores changes in list (md5 changed)
+	Changes    int        `json:"changes"`
+	LastChange *time.Time `json:"lastchange,omitempty"`
+	// Errors stores number of sync errors (not errors in file)
+	Errors    int        `json:"errors"`
+	LastError *time.Time `json:"lasterror,omitempty"`
+	// Last sync state
+	UpdatedOK bool          `json:"updatedok"`
+	Account   []AccountItem `json:"account,omitempty"`
+	ErrorMsg  string        `json:"errormsg,omitempty"`
+}
+
+// AccountItem stores accounting info
+// I can't use a simple map[xlist.Resources] because this issue:
+// //https://github.com/golang/go/issues/29732
+type AccountItem struct {
+	Resource xlist.Resource `json:"resource"`
+	Count    int            `json:"count"`
+}
+
+func (s *EntryStatus) setError(err error) {
+	now := time.Now()
+	s.UpdatedOK = false
+	s.Last = now
+	s.LastError = &now
+	s.ErrorMsg = err.Error()
+	s.Errors++
+}
+
+func (s *EntryStatus) setUpdate(r *Response) {
+	now := time.Now()
+	s.UpdatedOK = true
+	s.ErrorMsg = ""
+	s.Last = now
+	s.LastUpdate = &now
+	s.Updates++
+	if r.Updated {
+		s.LastChange = &now
+		s.Changes++
+	}
+	s.Account = make([]AccountItem, 0, len(xlist.Resources))
+	for _, res := range xlist.Resources {
+		v, _ := r.Account[res]
+		if v > 0 {
+			s.Account = append(s.Account, AccountItem{Resource: res, Count: v})
+		}
+	}
+}
+
+// EntryStatusFromFile returns an EntryStatus
+func EntryStatusFromFile(path string) (EntryStatus, error) {
+	var entry EntryStatus
+	f, err := os.Open(path)
+	defer f.Close()
+	if err != nil {
+		return EntryStatus{}, fmt.Errorf("opening file %s: %v", path, err)
+	}
+	byteValue, err := ioutil.ReadAll(f)
+	if err != nil {
+		return EntryStatus{}, fmt.Errorf("reading file %s: %v", path, err)
+	}
+	err = json.Unmarshal(byteValue, &entry)
+	if err != nil {
+		return EntryStatus{}, fmt.Errorf("unmarshalling from json file %s: %v", path, err)
+	}
+	return entry, nil
+}

pkg/xlget/manager.go

@@ -6,8 +6,10 @@
 package xlget
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"sync"
@@ -21,6 +23,7 @@ import (
 type Manager struct {
 	outputDir string
 	cacheDir  string
+	statusDir string
 	entries   []Entry
 	ids       map[string]bool
 	c         *Client
@@ -30,7 +33,7 @@ type Manager struct {
 }
 
 // NewManager creates a new manager
-func NewManager(outputDir, cacheDir string, opt ...Option) (*Manager, error) {
+func NewManager(outputDir, cacheDir, statusDir string, opt ...Option) (*Manager, error) {
 	//sets default options
 	opts := defaultOptions
 	for _, o := range opt {
@@ -39,6 +42,7 @@ func NewManager(outputDir, cacheDir string, opt ...Option) (*Manager, error) {
 	m := &Manager{
 		cacheDir:  cacheDir,
 		outputDir: outputDir,
+		statusDir: statusDir,
 		logger:    opts.logger,
 		entries:   make([]Entry, 0),
 		ids:       make(map[string]bool),
@@ -136,23 +140,52 @@ func (m *Manager) Update() (CancelFunc, <-chan struct{}, error) {
 func (m *Manager) updateRequests(requests []Entry, closeCh <-chan struct{}, done chan<- struct{}) {
 LOOPREQUESTS:
 	for _, req := range requests {
+		var status EntryStatus
+		if m.statusDir != "" {
+			var err error
+			status, err = m.getStatusFromEntry(req)
+			if err != nil {
+				m.logger.Errorf("can't get status file: %v", err)
+			}
+		}
 		response, err := m.c.Do(req)
 		if err != nil {
 			m.logger.Errorf("processing '%s': %v", req.ID, err)
+			if m.statusDir != "" {
+				status.setError(err)
+				err = m.writeEntryStatus(status)
+				if err != nil {
+					m.logger.Errorf("can't write status file: %v", err)
+				}
+			}
 			continue
 		}
 		select {
 		case <-response.Done:
 			err = response.Err()
 			if err != nil {
 				m.logger.Errorf("in response from '%s': %v", req.ID, err)
+				if m.statusDir != "" {
+					status.setError(err)
+					err = m.writeEntryStatus(status)
+					if err != nil {
+						m.logger.Errorf("can't write status file: %v", err)
+					}
+				}
 				continue
 			}
 			summary := fmt.Sprintf("summary '%s': updated=%v", response.ID, response.Updated)
 			for _, r := range xlist.Resources {
 				summary = summary + " " + fmt.Sprintf("%v=%v", r, response.Account[r])
 			}
 			m.logger.Infof("%s", summary)
+			if m.statusDir != "" {
+				status.setUpdate(response)
+				err = m.writeEntryStatus(status)
+				if err != nil {
+					m.logger.Errorf("can't write status file: %v", err)
+				}
+			}
 		case <-closeCh:
 			response.Cancel()
 			response.Wait()
@@ -163,6 +196,26 @@ LOOPREQUESTS:
 	close(done)
 }
 
+func (m *Manager) getStatusFromEntry(e Entry) (EntryStatus, error) {
+	file := m.statusDir + string(os.PathSeparator) + e.ID + ".status"
+	if fileExists(file) {
+		return EntryStatusFromFile(file)
+	}
+	return EntryStatus{ID: e.ID, First: time.Now()}, nil
+}
+
+func (m *Manager) writeEntryStatus(s EntryStatus) error {
+	if s.ID == "" {
+		return errors.New("status ID is empty")
+	}
+	file := m.statusDir + string(os.PathSeparator) + s.ID + ".status"
+	jsondata, err := json.MarshalIndent(s, "", "\t")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(file, jsondata, 0644)
+}
+
 func (m *Manager) requiresUpdate() []Entry {
 	required := make([]Entry, 0, len(m.entries))
 	for _, e := range m.entries {
@@ -195,6 +248,18 @@ func (m *Manager) isUpdated(e Entry) bool {
 	if os.IsNotExist(err) {
 		return false
 	}
+	//if status enabled, checks if error in previous sync
+	if m.statusDir != "" {
+		status, err := m.getStatusFromEntry(e)
+		if err != nil {
+			m.logger.Errorf("can't get status from entry '%s': %v", e.ID, err)
+			return false
+		}
+		if !status.UpdatedOK {
+			return false
+		}
+	}
+
 	last := info.ModTime()
 
 	md5file := fmt.Sprintf("%s.md5", output)
@@ -224,5 +289,11 @@ func (m *Manager) initDirs() error {
 			return err
 		}
 	}
+	if m.statusDir != "" {
+		err := createDir(m.statusDir)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }