Merge pull request #1 from loanpal-engineering/size_optimization_expe…

…riements

Add types for int / uint 8, 16, 32, 64, float 32 and 64

data/tree.go

@@ -12,20 +12,20 @@ import (
 type (
 	node struct {
 		id       uint64
+		min      uint64
+		max      uint64
+		t        pkg.FieldType // 32
+		v        interface{} // 16
 		name     string
-		v        interface{}
+		version  uint          // 32
+		mutex    sync.RWMutex
+		nullable pkg.Nullable
 		parent   pkg.Composer
-		t        *pkg.FieldType
-		children map[uint64]pkg.Composer
-		nm       []map[uint64]bool
-		nullable *pkg.Nullable
-		version  uint
 		next     pkg.Composer
 		prev     pkg.Composer
-		min      uint64
-		max      uint64
-		mutex    sync.RWMutex
 		index    map[interface{}][]uint64
+		children map[uint64]pkg.Composer
+		nm       []map[uint64]bool
 	}
 
 	Opt func(*node) (*node, error)
@@ -51,7 +51,9 @@ func NewNode(id *uint64, opts ...Opt) (pkg.Composer, error) {
 		i.id = *id
 	}
 	i.parent = nil
+	i.v = nil
 	i.nm = make([]map[uint64]bool, 1)
+	i.t = pkg.UNKNOWN
 	// initialize parser  nilmap
 	i.nm[0] = make(map[uint64]bool)
 	i.children = make(map[uint64]pkg.Composer)
@@ -79,17 +81,19 @@ func Index(b bool) Opt {
 }
 
 // Set the nullable property of a node
+// nullable is copied NOT referenced by the node
 func Nullable(nullable *pkg.Nullable) Opt {
 	return func(n *node) (*node, error) {
-		n.nullable = nullable
+		n.nullable = *nullable
 		return n, nil
 	}
 }
 
 // Add the type [t] of value held by this node.
+// t is copied NOT referenced by the node
 func Type(t *pkg.FieldType) Opt {
 	return func(n *node) (*node, error) {
-		n.t = t
+		n.t = *t
 		return n, nil
 	}
 }
@@ -125,8 +129,10 @@ func (i *node) Value() interface{} {
 }
 
 // Get a nodes children
-func (i *node) Children() map[uint64]pkg.Composer {
-	return i.children
+// returns a reference to the children map
+// The child should NOT be mutated, a reference is used to mitigate a potentially large copy operation
+func (i *node) Children() *map[uint64]pkg.Composer {
+	return &i.children
 }
 
 // Get a nodes name
@@ -253,7 +259,7 @@ func (i *node) Id() (uint64, uint32, uint32) {
 }
 
 // Get the Type of this node
-func (i *node) T() *pkg.FieldType {
+func (i *node) T() pkg.FieldType {
 	return i.t
 }
 
@@ -360,7 +366,7 @@ func (i *node) Excludes() []bool {
 		if col.(*node).nullable.Allowed {
 			continue
 		}
-		for _, vv := range col.Children() {
+		for _, vv := range *col.Children() {
 			id, _, row := vv.Id()
 			if ex, _ := col.(*node).Excluded(id); ex {
 				excludes[row] = true
@@ -369,12 +375,13 @@ func (i *node) Excludes() []bool {
 	}
 	return excludes
 }
-func (i *node) Nullable() *pkg.Nullable {
+func (i *node) Nullable() pkg.Nullable {
 	return i.nullable
 }
-func (i *node) LockWhile(fn func()) {
-	n := root(i)
-	n.mutex.RLock()
-	defer n.mutex.RUnlock()
-	fn()
-}
+
+// func (i *node) LockWhile(fn func()) {
+// 	n := root(i)
+// 	n.mutex.RLock()
+// 	defer n.mutex.RUnlock()
+// 	fn()
+// }

io/input/common.go

@@ -2,8 +2,9 @@ package input
 
 import (
 	"encoding/csv"
-	"github.com/loanpal-engineering/exttra/types"
 	"io"
+
+	"github.com/loanpal-engineering/exttra/types"
 )
 
 type (

io/input/flat_file.go

@@ -37,3 +37,4 @@ func Csv(source io.Reader, def types.Signature) Input {
 	i.reader.ReuseRecord = true
 	return i
 }
+

io/output/flat_file.go

@@ -84,7 +84,7 @@ func (i *FlatFile) Flush() error {
 	column := make(chan pkg.Pair)
 	sent := 0
 	complete := 0
-	for id, v := range root.Children() {
+	for id, v := range *root.Children() {
 		if pkg.IsNil(v) || root.Null()[id] {
 			continue
 		}
@@ -103,7 +103,7 @@ func (i *FlatFile) Flush() error {
 				i.header = rows[0]
 				err := writer.WriteAll(rows)
 				if err != nil {
-					pkg.FatalDefect(&pkg.Defect{
+					pkg.FatalDefect(pkg.Defect{
 						Msg: err.Error(),
 					})
 				}
@@ -171,7 +171,7 @@ func (i *FlatFile) buildColumn(out chan pkg.Pair, n pkg.Composer, colIdx int) {
 		format = f
 	}
 	excludes := i.src.(pkg.Editor).Excludes()
-	for _, v := range n.Children() {
+	for _, v := range *n.Children() {
 		_, _, row := v.Id()
 		if excludes[row] {
 			continue

io/output/mem.go

@@ -51,8 +51,8 @@ func (i *Memory) leftMostNode() (pkg.Composer, error) {
 	// Left most node in the tree
 	var lhs pkg.Composer
 	// Doesnt matter which child, just pick one
-	for id := range i.src.Children() {
-		lhs = i.src.Children()[id]
+	for id := range *i.src.Children() {
+		lhs = (*i.src.Children())[id]
 		break
 	}
 	if lhs == nil {
@@ -98,7 +98,7 @@ func (i *Memory) Flush() error {
 		return err
 	}
 	excludes := i.src.(pkg.Editor).Excludes()
-	for _, v := range lhs.Children() {
+	for _, v := range *lhs.Children() {
 		_, _, row := v.Id()
 		if excludes[row] {
 			continue
@@ -191,6 +191,10 @@ func (i *Memory) fillShape(out chan interface{}, quit chan error, n pkg.Composer
 			if field.Type.Kind() == reflect.Float64 {
 				cpy.FieldByIndex(field.Index).SetFloat(n.Value().(float64))
 			}
+		case float32:
+			if field.Type.Kind() == reflect.Float32 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(float32)))
+			}
 		case bool:
 			if field.Type.Kind() == reflect.Bool {
 				cpy.FieldByIndex(field.Index).SetBool(n.Value().(bool))
@@ -205,6 +209,34 @@ func (i *Memory) fillShape(out chan interface{}, quit chan error, n pkg.Composer
 			if field.Type.Kind() == reflect.Int64 {
 				cpy.FieldByIndex(field.Index).SetInt(n.Value().(int64))
 			}
+		case int32:
+			if field.Type.Kind() == reflect.Int32 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(int32)))
+			}
+		case int16:
+			if field.Type.Kind() == reflect.Int16 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(int16)))
+			}
+		case int8:
+			if field.Type.Kind() == reflect.Int8 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(int8)))
+			}
+		case uint64:
+			if field.Type.Kind() == reflect.Uint64 {
+				cpy.FieldByIndex(field.Index).SetUint(n.Value().(uint64))
+			}
+		case uint32:
+			if field.Type.Kind() == reflect.Uint32 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(uint32)))
+			}
+		case uint16:
+			if field.Type.Kind() == reflect.Uint16 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(uint16)))
+			}
+		case uint8:
+			if field.Type.Kind() == reflect.Uint8 {
+				cpy.FieldByIndex(field.Index).Set(reflect.ValueOf(n.Value().(uint8)))
+			}
 		default:
 			quit <- errors.New("output/Memory: unknown type, supported types are: string, float64, int64, bool, time.Time")
 		}