diff --git a/cmd/cimp/main.go b/cmd/cimp/main.go index 28af83c..5243436 100644 --- a/cmd/cimp/main.go +++ b/cmd/cimp/main.go @@ -2,15 +2,16 @@ package main import ( "flag" + "io/ioutil" "path/filepath" "github.com/humans-group/cimp/lib/cimp" + "github.com/humans-group/cimp/lib/tree" ) func main() { pathRaw := flag.String("p", "./config.yaml", "Path to config-file which should be imported") formatRaw := flag.String("f", "", "File format: json, yaml, edn. If empty - got from extension. Default: yaml") - arrayValueFormatRaw := flag.String("a", "", "Array value format: json, yaml, edn. If empty - got from extension. Default: yaml") consulEndpoint := flag.String("c", "127.0.0.1:8500", "Consul endpoint in format `address:port`") prefixRaw := flag.String("pref", "", "Prefix for all keys") @@ -25,11 +26,13 @@ func main() { format, err := cimp.NewFormat(*formatRaw, path) check(err) - arrayValueFormat, err := cimp.NewFormat(*arrayValueFormatRaw, path) + cfgRaw, err := ioutil.ReadFile(path) check(err) - kv := cimp.NewKV("", arrayValueFormat) - check(kv.FillFromFile(path, format)) + kv := cimp.NewKV(tree.New()) + unmarshaler := cimp.NewUnmarshaler(kv, format) + err = unmarshaler.Unmarshal(cfgRaw) + check(err) kv.AddPrefix(*prefixRaw) diff --git a/lib/cimp/errors.go b/lib/cimp/errors.go index f62f461..82c7ce9 100644 --- a/lib/cimp/errors.go +++ b/lib/cimp/errors.go @@ -3,6 +3,7 @@ package cimp import "fmt" var ( - ErrorNotFoundInKV = fmt.Errorf("value is not found in KV") - ErrorTypeIncorrect = fmt.Errorf("type is incorrect") + ErrorNotFoundInKV = fmt.Errorf("value is not found in KV") + ErrorParentNotFoundInKV = fmt.Errorf("parent value is not found in KV") + ErrorTypeIncorrect = fmt.Errorf("type is incorrect") ) diff --git a/lib/cimp/kv.go b/lib/cimp/kv.go index 9d8daab..196311f 100644 --- a/lib/cimp/kv.go +++ b/lib/cimp/kv.go @@ -3,6 +3,7 @@ package cimp import ( "bytes" "encoding/json" + "errors" "fmt" "strconv" "strings" @@ -25,6 +26,8 @@ type treeConverter struct { Indent int } +const consulSep = "/" + func NewKV(t *tree.Tree) *KV { idx := index(make(map[string]tree.Path)) idx.addKeys(t, nil) @@ -70,9 +73,70 @@ func (kv *KV) GetString(key string) (string, error) { } } +func (kv *KV) Exists(fullKey string) bool { + if _, ok := kv.idx[fullKey]; ok { + return true + } + + _, err := kv.tree.GetByFullKey(fullKey) + + return err == nil +} + +func (kv *KV) AddIfNotSet(m tree.Marshalable) error { + if _, ok := kv.idx[m.GetFullKey()]; ok { + return nil + } + + lastSepIdx := strings.LastIndexAny(m.GetFullKey(), consulSep) + if lastSepIdx < 0 { + return ErrorParentNotFoundInKV + } + parentFullKey := m.GetFullKey()[:lastSepIdx] + + parent, err := kv.tree.GetByFullKey(parentFullKey) + if err != nil { + if errors.Is(err, tree.ErrorNotFound) { + return ErrorParentNotFoundInKV + } + return fmt.Errorf("get parent: %w", err) + } + + if item, err := parent.GetByFullKey(m.GetFullKey()); err == nil || item != nil { + return nil + } else if !errors.Is(err, tree.ErrorNotFound) { + return fmt.Errorf("check value %q existence: %w", m.GetFullKey(), err) + } + + switch parItem := parent.(type) { + case *tree.Tree: + parItem.AddOrReplaceDirectly(m.GetName(), m) + case *tree.Branch: + parItem.Add(m) + default: + return ErrorTypeIncorrect + } + + return nil +} + +func (kv *KV) DeleteIfExists(fullKey string) error { + if err := kv.tree.Delete(fullKey); err != nil { + if errors.Is(err, tree.ErrorNotFound) { + return nil + } + + return fmt.Errorf("delete by key %q: %w", fullKey, err) + } + + delete(kv.idx, fullKey) + + return nil +} + func (kv *KV) AddPrefix(prefix string) { - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" + if !strings.HasSuffix(prefix, consulSep) { + prefix = prefix + consulSep } kv.globalPrefix = prefix } @@ -164,10 +228,10 @@ func (c treeConverter) convertBranchesToString(mt *tree.Tree) (*tree.Tree, error } } - leaf := tree.NewLeaf(k, mt.FullKey, mt.NestingLevel) + leaf := tree.NewLeaf(k, mt.FullKey) leafValueBuf := bytes.NewBufferString("\n") leafValueBuf.Write(buf.Bytes()) - endLineAndIndent := []byte("\n" + strings.Repeat(" ", int(leaf.NestingLevel)*c.Indent)) + endLineAndIndent := []byte("\n" + strings.Repeat(" ", int(leaf.GetNestingLevel())*c.Indent)) leafValue := bytes.ReplaceAll( leafValueBuf.Bytes(), []byte("\n"), @@ -195,9 +259,9 @@ func (idx index) clear() { } } -func (idx index) addKeys(m tree.Marshalable, prevPath tree.Path) { - path := make(tree.Path, len(prevPath)) - copy(path, prevPath) +func (idx index) addKeys(m tree.Marshalable, curPath tree.Path) { + path := make(tree.Path, len(curPath)) + copy(path, curPath) switch cur := m.(type) { case *tree.Leaf: diff --git a/lib/tree/errors.go b/lib/tree/errors.go index 4abab06..8427d6e 100644 --- a/lib/tree/errors.go +++ b/lib/tree/errors.go @@ -2,4 +2,7 @@ package tree import "fmt" -var ErrorNotFound = fmt.Errorf("leaf is not found") +var ( + ErrorNotFound = fmt.Errorf("not found") + ErrorUnsupported = fmt.Errorf("method is not supported") +) diff --git a/lib/tree/json.go b/lib/tree/json.go index abf3206..0b6f540 100644 --- a/lib/tree/json.go +++ b/lib/tree/json.go @@ -147,18 +147,18 @@ func (mb *Branch) UnmarshalJSON(raw []byte) error { delim, ok := token.(json.Delim) if !ok { - leaf := NewLeaf(name, mb.FullKey, mb.NestingLevel) + leaf := NewLeaf(name, mb.FullKey) leaf.decoder = mb.decoder leaf.Value = token child = leaf } else { switch delim { case '{': - childTree := NewSubTree(name, mb.FullKey, mb.NestingLevel) + childTree := NewSubTree(name, mb.FullKey) childTree.decoder = mb.decoder child = childTree case '[': - branch := NewBranch(name, mb.FullKey, mb.NestingLevel) + branch := NewBranch(name, mb.FullKey) branch.decoder = mb.decoder child = branch default: @@ -223,18 +223,18 @@ func (mt *Tree) UnmarshalJSON(raw []byte) error { delim, ok := token.(json.Delim) if !ok { - leaf := NewLeaf(name, mt.FullKey, mt.NestingLevel) + leaf := NewLeaf(name, mt.FullKey) leaf.decoder = mt.decoder leaf.Value = token child = leaf } else { switch delim { case '{': - childTree := NewSubTree(name, mt.FullKey, mt.NestingLevel) + childTree := NewSubTree(name, mt.FullKey) childTree.decoder = mt.decoder child = childTree case '[': - branch := NewBranch(name, mt.FullKey, mt.NestingLevel) + branch := NewBranch(name, mt.FullKey) branch.decoder = mt.decoder child = branch default: diff --git a/lib/tree/tree.go b/lib/tree/tree.go index c19c2a3..988d8ac 100644 --- a/lib/tree/tree.go +++ b/lib/tree/tree.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "gopkg.in/yaml.v3" ) @@ -13,6 +14,11 @@ type Marshalable interface { UnmarshalYAML(value *yaml.Node) error MarshalJSON() ([]byte, error) UnmarshalJSON(raw []byte) error + GetByFullKey(string) (Marshalable, error) + GetFullKey() string + GetName() string + GetNestingLevel() int + Delete(string) error } type Tree struct { @@ -20,7 +26,7 @@ type Tree struct { Name string Order []string FullKey string - NestingLevel uint + nestingLevel int decoder *json.Decoder } @@ -28,7 +34,7 @@ type Branch struct { Content []Marshalable Name string FullKey string - NestingLevel uint + nestingLevel int decoder *json.Decoder } @@ -37,7 +43,7 @@ type Leaf struct { Name string FullKey string decoder *json.Decoder - NestingLevel uint + nestingLevel int yamlMarshalStyle yaml.Style } @@ -49,29 +55,29 @@ func New() *Tree { } } -func NewSubTree(name, parentFullKey string, parentNestingLevel uint) *Tree { +func NewSubTree(name, parentFullKey string) *Tree { return &Tree{ Content: make(map[string]Marshalable), Name: name, FullKey: MakeFullKey(parentFullKey, name), - NestingLevel: parentNestingLevel + 1, + nestingLevel: strings.Count(parentFullKey, sep) + 2, } } -func NewBranch(name, parentFullKey string, parentNestingLevel uint) *Branch { +func NewBranch(name, parentFullKey string) *Branch { return &Branch{ Content: []Marshalable{}, Name: name, FullKey: MakeFullKey(parentFullKey, name), - NestingLevel: parentNestingLevel + 1, + nestingLevel: strings.Count(parentFullKey, sep) + 2, } } -func NewLeaf(name, parentFullKey string, parentNestingLevel uint) *Leaf { +func NewLeaf(name, parentFullKey string) *Leaf { return &Leaf{ Name: name, FullKey: MakeFullKey(parentFullKey, name), - NestingLevel: parentNestingLevel + 1, + nestingLevel: strings.Count(parentFullKey, sep) + 2, } } @@ -126,17 +132,172 @@ func (mb *Branch) Get(path Path) (*Leaf, error) { } } +func (mt *Tree) GetByFullKey(fullKey string) (Marshalable, error) { + if mt.FullKey == fullKey { + return mt, nil + } + + relativeKey := strings.TrimPrefix(fullKey, mt.FullKey+sep) + if len(relativeKey) > len(fullKey) { + return nil, ErrorNotFound + } + + sepIdx := strings.IndexAny(relativeKey, sep) + // If separator is not found - it should be final item + isFinal := sepIdx == -1 + curRelativeKey := relativeKey + if !isFinal { + curRelativeKey = relativeKey[:sepIdx] + } + searchKey := curRelativeKey + if len(mt.FullKey) != 0 { + searchKey = mt.FullKey + sep + searchKey + } + + for _, v := range mt.Content { + if v.GetFullKey() == searchKey { + return v.GetByFullKey(fullKey) + } + } + + return nil, ErrorNotFound +} + +func (mb *Branch) GetByFullKey(fullKey string) (Marshalable, error) { + if mb.FullKey == fullKey { + return mb, nil + } + + relativeKey := strings.TrimPrefix(fullKey, mb.FullKey+sep) + if len(relativeKey) > len(fullKey) { + return nil, ErrorNotFound + } + + sepIdx := strings.IndexAny(relativeKey, sep) + // If separator is not found - it should be final item + isFinal := sepIdx == -1 + curRelativeKey := relativeKey + if !isFinal { + curRelativeKey = relativeKey[:sepIdx] + } + searchIdx, err := strconv.Atoi(curRelativeKey) + if err != nil { + return nil, fmt.Errorf("wrong fullKey format, %q is branch: %w", mb.FullKey, err) + } + + if len(mb.Content)-1 >= searchIdx { + return mb.Content[searchIdx].GetByFullKey(fullKey) + } + + return nil, ErrorNotFound +} + +func (ml *Leaf) GetByFullKey(fullKey string) (Marshalable, error) { + if ml.FullKey == fullKey { + return ml, nil + } + + return nil, ErrorNotFound +} + +func (mt *Tree) Delete(fullKey string) error { + if mt.FullKey == fullKey { + return fmt.Errorf("you can't delete a tree from itself") + } + + relativeKey := strings.TrimPrefix(fullKey, mt.FullKey+sep) + if len(relativeKey) > len(fullKey) { + return ErrorNotFound + } + + sepIdx := strings.IndexAny(relativeKey, sep) + isFinal := sepIdx == -1 + curRelativeKey := relativeKey + if !isFinal { + curRelativeKey = relativeKey[:sepIdx] + } + searchKey := curRelativeKey + if len(mt.FullKey) != 0 { + searchKey = mt.FullKey + sep + searchKey + } + + var deletedName string + for _, v := range mt.Content { + if v.GetFullKey() != searchKey { + continue + } + if !isFinal { + return v.Delete(fullKey) + } + deletedName = v.GetName() + delete(mt.Content, deletedName) + break + } + + if len(deletedName) == 0 { + return ErrorNotFound + } + + for i, orderedName := range mt.Order { + if orderedName == deletedName { + copy(mt.Order[i:], mt.Order[i+1:]) // Shift a[i+1:] left one index + mt.Order[len(mt.Order)-1] = "" // Erase last element (write zero value) + mt.Order = mt.Order[:len(mt.Order)-1] // Truncate slice + return nil + } + } + + return fmt.Errorf("item %q was deleted, but its name is not found in order slice", deletedName) +} + +func (mb *Branch) Delete(fullKey string) error { + if mb.FullKey == fullKey { + return fmt.Errorf("you can't delete a branch from itself") + } + + relativeKey := strings.TrimPrefix(fullKey, mb.FullKey+sep) + + sepIdx := strings.IndexAny(relativeKey, sep) + isFinal := sepIdx == -1 + curRelativeKey := relativeKey + if !isFinal { + curRelativeKey = relativeKey[:sepIdx] + } + searchIdx, err := strconv.Atoi(curRelativeKey) + if err != nil { + return fmt.Errorf("wrong fullKey format, %q is branch: %w", mb.FullKey, err) + } + + if len(mb.Content)-1 < searchIdx { + return ErrorNotFound + } + + if !isFinal { + return mb.Content[searchIdx].Delete(fullKey) + } + + copy(mb.Content[searchIdx:], mb.Content[searchIdx+1:]) // Shift a[i+1:] left one index + mb.Content[len(mb.Content)-1] = nil // Erase last element (write zero value) + mb.Content = mb.Content[:len(mb.Content)-1] // Truncate slice + + return nil +} + +func (ml *Leaf) Delete(_ string) error { + return ErrorUnsupported +} + func (mt *Tree) AddOrReplaceDirectly(name string, value Marshalable) { if _, ok := mt.Content[name]; !ok { mt.Order = append(mt.Order, name) } switch item := value.(type) { case *Tree: - item.NestingLevel = mt.NestingLevel + 1 + item.nestingLevel = mt.nestingLevel + 1 case *Branch: - item.NestingLevel = mt.NestingLevel + 1 + item.nestingLevel = mt.nestingLevel + 1 case *Leaf: - item.NestingLevel = mt.NestingLevel + 1 + item.nestingLevel = mt.nestingLevel + 1 } mt.Content[name] = value @@ -161,7 +322,7 @@ func (mt *Tree) ShallowClone() *Tree { Order: newOrder, FullKey: mt.FullKey, decoder: mt.decoder, - NestingLevel: mt.NestingLevel, + nestingLevel: mt.nestingLevel, } return newTree @@ -188,7 +349,7 @@ func (mt *Tree) DeepClone() *Tree { Name: mt.Name, Order: newOrder, FullKey: mt.FullKey, - NestingLevel: mt.NestingLevel, + nestingLevel: mt.nestingLevel, decoder: mt.decoder, } @@ -212,7 +373,7 @@ func (mb *Branch) DeepClone() *Branch { Content: newContent, Name: mb.Name, FullKey: mb.FullKey, - NestingLevel: mb.NestingLevel, + nestingLevel: mb.nestingLevel, decoder: mb.decoder, } @@ -238,7 +399,7 @@ func (ml *Leaf) DeepClone() *Leaf { Value: newValue, Name: ml.Name, FullKey: ml.FullKey, - NestingLevel: ml.NestingLevel, + nestingLevel: ml.nestingLevel, decoder: ml.decoder, yamlMarshalStyle: ml.yamlMarshalStyle, } @@ -280,6 +441,42 @@ func (mb *Branch) Walk(wf WalkFunc) { } } +func (mb *Branch) GetName() string { + return mb.Name +} + +func (mb *Branch) GetFullKey() string { + return mb.FullKey +} + +func (mb *Branch) GetNestingLevel() int { + return mb.nestingLevel +} + +func (mt *Tree) GetName() string { + return mt.Name +} + +func (mt *Tree) GetFullKey() string { + return mt.FullKey +} + +func (mt *Tree) GetNestingLevel() int { + return mt.nestingLevel +} + +func (ml *Leaf) GetName() string { + return ml.Name +} + +func (ml *Leaf) GetFullKey() string { + return ml.FullKey +} + +func (ml *Leaf) GetNestingLevel() int { + return ml.nestingLevel +} + func (mt *Tree) clearValues() { if len(mt.Content) > 0 { mt.Content = make(map[string]Marshalable) diff --git a/lib/tree/tree_test.go b/lib/tree/tree_test.go index 4845d34..aa04d56 100644 --- a/lib/tree/tree_test.go +++ b/lib/tree/tree_test.go @@ -54,7 +54,7 @@ func TestTree_MarshalYAML(t *testing.T) { var m Marshalable switch tc.name { case testBranchSimple, testBranchHard: - m = NewBranch("", "", 0) + m = NewBranch("", "") case testTreeSimple, testTreeHard: m = New() default: @@ -110,7 +110,7 @@ func TestTree_MarshalJSON(t *testing.T) { var m Marshalable switch tc.name { case testBranchSimple, testBranchHard: - m = NewBranch("", "", 0) + m = NewBranch("", "") case testTreeSimple, testTreeHard: m = New() default: @@ -168,7 +168,7 @@ func TestTree_UnmarshalYAML(t *testing.T) { var m Marshalable switch tc.name { case testBranchSimple, testBranchHard: - m = NewBranch("", "", 0) + m = NewBranch("", "") case testTreeSimple, testTreeHard: m = New() default: @@ -227,7 +227,7 @@ func TestTree_UnmarshalJSON(t *testing.T) { var m Marshalable switch tc.name { case testBranchSimple, testBranchHard: - m = NewBranch("", "", 0) + m = NewBranch("", "") case testTreeSimple, testTreeHard: m = New() default: @@ -258,3 +258,132 @@ func TestTree_UnmarshalJSON(t *testing.T) { }) } } + +func TestTree_GetByFullKey(t *testing.T) { + tree := &Tree{ + Content: map[string]Marshalable{ + "Leaf 1": &Leaf{ + Value: "hi", + Name: "Leaf 1", + FullKey: "tree_1/leaf_1", + nestingLevel: 1, + }, + "Leaf 2": &Leaf{ + Value: "hi", + Name: "Leaf 2", + FullKey: "tree_1/leaf_2", + nestingLevel: 1, + }, + "Branch 1": &Branch{ + Content: []Marshalable{ + &Leaf{ + Value: "hi", + Name: "Leaf 1", + FullKey: "tree_1/branch_1/0", + nestingLevel: 2, + }, + &Leaf{ + Value: "hi", + Name: "Leaf 2", + FullKey: "tree_1/branch_1/1", + nestingLevel: 2, + }, + }, + Name: "Branch 1", + FullKey: "tree_1/branch_1", + nestingLevel: 1, + }, + }, + Name: "Tree 1", + Order: []string{"leaf1"}, + FullKey: "tree_1", + nestingLevel: 0, + } + + tests := []struct { + name testName + searchFullKey string + foundFullKey string + expErr error + }{ + { + name: "simple", + searchFullKey: "tree_1/leaf_1", + }, + { + name: "not found", + searchFullKey: "tree_1/leaf_3", + expErr: ErrorNotFound, + }, + { + name: "branch", + searchFullKey: "tree_1/branch_1", + }, + { + name: "from branch", + searchFullKey: "tree_1/branch_1/1", + }, + { + name: "abrakadabra", + searchFullKey: "sdfdsfdsf", + expErr: ErrorNotFound, + }, + { + name: "empty", + searchFullKey: "", + expErr: ErrorNotFound, + }, + { + name: "not found in branch", + searchFullKey: "tree_1/branch_1/3", + expErr: ErrorNotFound, + }, + } + + for _, tc := range tests { + res, err := tree.GetByFullKey(tc.searchFullKey) + if tc.expErr == nil && err != nil { + t.Errorf("unexpected error: %v", err) + } + if tc.expErr == nil && (res == nil || res.GetFullKey() != tc.searchFullKey) { + t.Errorf("result not found") + } + } + + relativeTests := []struct { + name testName + searchFullKey string + foundFullKey string + expErr error + }{ + { + name: "from branch", + searchFullKey: "tree_1/branch_1/1", + }, + { + name: "abrakadabra", + searchFullKey: "sdfdsfdsf", + expErr: ErrorNotFound, + }, + { + name: "empty", + searchFullKey: "", + expErr: ErrorNotFound, + }, + { + name: "not found in branch", + searchFullKey: "tree_1/branch_1/3", + expErr: ErrorNotFound, + }, + } + + for _, tc := range relativeTests { + res, err := tree.Content["Branch 1"].GetByFullKey(tc.searchFullKey) + if tc.expErr == nil && err != nil { + t.Errorf("unexpected error: %v", err) + } + if tc.expErr == nil && (res == nil || res.GetFullKey() != tc.searchFullKey) { + t.Errorf("result not found") + } + } +} diff --git a/lib/tree/yaml.go b/lib/tree/yaml.go index dae3f42..b0c6c72 100644 --- a/lib/tree/yaml.go +++ b/lib/tree/yaml.go @@ -19,19 +19,19 @@ func (mt *Tree) UnmarshalYAML(node *yaml.Node) error { switch curNode.Kind { case yaml.ScalarNode: - leaf := NewLeaf(curKey, mt.FullKey, mt.NestingLevel) + leaf := NewLeaf(curKey, mt.FullKey) if err := leaf.UnmarshalYAML(curNode); err != nil { return fmt.Errorf("unmarshal leaf %q: %w", curKey, err) } mt.AddOrReplaceDirectly(curKey, leaf) case yaml.MappingNode: - childTree := NewSubTree(curKey, mt.FullKey, mt.NestingLevel) + childTree := NewSubTree(curKey, mt.FullKey) if err := childTree.UnmarshalYAML(curNode); err != nil { return fmt.Errorf("unmarshal sub-tree %q: %w", curKey, err) } mt.AddOrReplaceDirectly(curKey, childTree) case yaml.SequenceNode: - branch := NewBranch(curKey, mt.FullKey, mt.NestingLevel) + branch := NewBranch(curKey, mt.FullKey) if err := branch.UnmarshalYAML(curNode); err != nil { return fmt.Errorf("unmarshal branch %q: %w", curKey, err) } @@ -55,17 +55,17 @@ func (mb *Branch) UnmarshalYAML(node *yaml.Node) error { switch curNode.Kind { case yaml.ScalarNode: - leaf := NewLeaf(curKey, mb.FullKey, mb.NestingLevel) + leaf := NewLeaf(curKey, mb.FullKey) leaf.Value = curNode.Value mb.Add(leaf) case yaml.MappingNode: - tree := NewSubTree(curKey, mb.FullKey, mb.NestingLevel) + tree := NewSubTree(curKey, mb.FullKey) if err := tree.UnmarshalYAML(curNode); err != nil { return fmt.Errorf("unmarshal %q: %w", curKey, err) } mb.Add(tree) case yaml.SequenceNode: - childBranch := NewBranch(curKey, mb.FullKey, mb.NestingLevel) + childBranch := NewBranch(curKey, mb.FullKey) if err := childBranch.UnmarshalYAML(curNode); err != nil { return fmt.Errorf("unamrshal child branch #%s: %w", curKey, err) }