Adapt to new glazed API

cmd/parka/cmds/examples.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/cmds/parameters"
 	"github.com/go-go-golems/glazed/pkg/processor"
+	"github.com/go-go-golems/glazed/pkg/types"
 )
 
 type ExampleCommand struct {
@@ -108,35 +109,36 @@ func (e *ExampleCommand) Run(
 	ctx context.Context,
 	parsedLayers map[string]*layers.ParsedParameterLayer,
 	ps map[string]interface{},
-	gp processor.Processor,
+	gp processor.TableProcessor,
 ) error {
-	obj := map[string]interface{}{
-		"test":             ps["test"],
-		"string":           ps["string"],
-		"string_from_file": ps["string_from_file"],
-		"object_from_file": ps["object_from_file"],
-		"integer":          ps["integer"],
-		"float":            ps["float"],
-		"bool":             ps["bool"],
-		"date":             ps["date"],
-		"string_list":      ps["string_list"],
-		"integer_list":     ps["integer_list"],
-		"float_list":       ps["float_list"],
-		"choice":           ps["choice"],
-	}
-	err := gp.ProcessInputObject(ctx, obj)
+	obj := types.NewRow(
+		types.MRP("test", ps["test"]),
+		types.MRP("string", ps["string"]),
+		types.MRP("string_from_file", ps["string_from_file"]),
+		types.MRP("object_from_file", ps["object_from_file"]),
+		types.MRP("integer", ps["integer"]),
+
+		types.MRP("float", ps["float"]),
+		types.MRP("bool", ps["bool"]),
+		types.MRP("date", ps["date"]),
+		types.MRP("string_list", ps["string_list"]),
+		types.MRP("integer_list", ps["integer_list"]),
+		types.MRP("float_list", ps["float_list"]),
+		types.MRP("choice", ps["choice"]),
+	)
+	err := gp.AddRow(ctx, obj)
 	if err != nil {
 		return err
 	}
 
-	err = gp.ProcessInputObject(ctx, map[string]interface{}{
-		"test":  "test",
-		"test2": []int{123, 123, 123, 123},
-		"test3": map[string]interface{}{
+	err = gp.AddRow(ctx, types.NewRow(
+		types.MRP("test", "test"),
+		types.MRP("test2", []int{123, 123, 123, 123}),
+		types.MRP("test3", map[string]interface{}{
 			"test":  "test",
 			"test2": []int{123, 123, 123, 123},
-		},
-	})
+		}),
+	))
 	if err != nil {
 		return err
 	}

cmd/parka/cmds/serve.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/go-go-golems/glazed/pkg/cli"
 	"github.com/go-go-golems/glazed/pkg/helpers"
+	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/go-go-golems/parka/pkg/render"
 	"github.com/go-go-golems/parka/pkg/server"
 	"github.com/go-go-golems/parka/pkg/utils/fs"
@@ -112,11 +113,14 @@ var LsServerCmd = &cobra.Command{
 		cobra.CheckErr(err)
 
 		for _, cmd := range cmds {
-			err = gp.ProcessInputObject(ctx, cmd)
+			err = gp.AddRow(ctx, types.NewRowFromMap(cmd))
 			cobra.CheckErr(err)
 		}
 
-		err = gp.OutputFormatter().Output(ctx, os.Stdout)
+		err = gp.Finalize(ctx)
+		cobra.CheckErr(err)
+
+		err = gp.OutputFormatter().Output(ctx, gp.GetTable(), os.Stdout)
 		cobra.CheckErr(err)
 	},
 }

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/gin-gonic/contrib v0.0.0-20221130124618-7e01895a63f2
 	github.com/gin-gonic/gin v1.9.0
 	github.com/go-go-golems/clay v0.0.17
-	github.com/go-go-golems/glazed v0.2.84
+	github.com/go-go-golems/glazed v0.3.0
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.29.0
 	github.com/spf13/cobra v1.6.1
@@ -30,7 +30,9 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aymanbagabas/go-osc52 v1.0.3 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.8.0 // indirect
 	github.com/charmbracelet/glamour v0.6.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
@@ -59,6 +61,7 @@ require (
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
@@ -86,6 +89,7 @@ require (
 	github.com/tj/go-naturaldate v1.3.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
 	github.com/xuri/excelize/v2 v2.7.0 // indirect
 	github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect

go.sum

@@ -62,8 +62,12 @@ github.com/aymanbagabas/go-osc52 v1.0.3 h1:DTwqENW7X9arYimJrPeGZcV0ln14sGMt3pHZs
 github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
 github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA=
 github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
@@ -109,8 +113,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-go-golems/clay v0.0.17 h1:lFOXpAs3AiBMu3LwUO2xvJ7JvkIVU4g2beRODyKGvTM=
 github.com/go-go-golems/clay v0.0.17/go.mod h1:2QM24Cz+xBFrnXr5h561DAQ/lxLYlnUO8EjlxiOsYk4=
-github.com/go-go-golems/glazed v0.2.84 h1:rZylJBh/ZrD1Hi4xWXGf5cFQAWbLK9C9FbB426ASWhU=
-github.com/go-go-golems/glazed v0.2.84/go.mod h1:wlKdrmjongDOf2gqPVd8OwPVp50PVgWyv65mhxcEkP0=
+github.com/go-go-golems/glazed v0.3.0 h1:L4lJAz1WvYasKjvkxd3UTJImoPEcl6gEvV21qUqjjcM=
+github.com/go-go-golems/glazed v0.3.0/go.mod h1:zMZnD5ZCj5h/UAQpfpaWV6UkGMdILd8yWXGh9y8sdvU=
 github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc=
 github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk=
 github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k=
@@ -206,6 +210,7 @@ github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm
 github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo=
 github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -229,6 +234,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
@@ -327,6 +334,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
+github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=

pkg/glazed/handler.go

@@ -20,7 +20,7 @@ import (
 // data provided in CommandContext. For example, the user might want to request a specific response
 // format through a query argument or through a header.
 type CreateProcessorFunc func(c *gin.Context, pc *CommandContext) (
-	processor.Processor,
+	processor.TableProcessor,
 	error,
 )
 
@@ -42,7 +42,7 @@ type HandleOptions struct {
 	// I'm not entirely sure if this all makes sense.
 	Handlers []CommandHandlerFunc
 
-	// CreateProcessor takes a gin.Context and a CommandContext and returns a processor.Processor (and a content-type)
+	// CreateProcessor takes a gin.Context and a CommandContext and returns a processor.TableProcessor (and a content-type)
 	CreateProcessor CreateProcessorFunc
 
 	// This is the actual gin output writer
@@ -99,7 +99,7 @@ func WithCreateProcessor(createProcessor CreateProcessorFunc) HandleOption {
 }
 
 func CreateJSONProcessor(_ *gin.Context, pc *CommandContext) (
-	processor.Processor,
+	processor.TableProcessor,
 	error,
 ) {
 	l, ok := pc.ParsedLayers["glazed"]
@@ -198,7 +198,7 @@ func runGlazeCommand(c *gin.Context, cmd cmds.GlazeCommand, opts *HandleOptions)
 		}
 	}
 
-	var gp processor.Processor
+	var gp processor.TableProcessor
 
 	var err error
 	if opts.CreateProcessor != nil {
@@ -228,7 +228,7 @@ func runGlazeCommand(c *gin.Context, cmd cmds.GlazeCommand, opts *HandleOptions)
 	if opts.Writer != nil {
 		writer = opts.Writer
 	}
-	err = of.Output(c, writer)
+	err = of.Output(c, gp.GetTable(), writer)
 	if err != nil {
 		return err
 	}
@@ -248,7 +248,10 @@ func SetupProcessor(pc *CommandContext, options ...processor.GlazeProcessorOptio
 	of := json.NewOutputFormatter(
 		json.WithOutputIndividualRows(true),
 	)
-	gp := processor.NewGlazeProcessor(of, options...)
+	gp, err := processor.NewGlazeProcessor(of, options...)
+	if err != nil {
+		return nil, err
+	}
 
 	return gp, nil
 }

pkg/render/datatables/datatables.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-go-golems/glazed/pkg/formatters/table"
 	"github.com/go-go-golems/glazed/pkg/processor"
 	"github.com/go-go-golems/glazed/pkg/settings"
+	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/go-go-golems/parka/pkg/glazed"
 	"github.com/go-go-golems/parka/pkg/render"
 	"github.com/go-go-golems/parka/pkg/render/layout"
@@ -179,21 +180,21 @@ func (d *DataTablesOutputFormatter) ContentType() string {
 	return "text/html; charset=utf-8"
 }
 
-func (d *DataTablesOutputFormatter) Output(ctx context.Context, w io.Writer) error {
+func (d *DataTablesOutputFormatter) Output(ctx context.Context, table *types.Table, w io.Writer) error {
 	dt := d.dataTablesData
 	if dt.JSRendering {
-		jsonOutputFormatter := json.NewOutputFormatter(json.WithTable(d.OutputFormatter.Table))
-		dt.JSStream = formatters.StartFormatIntoChannel[template.JS](ctx, jsonOutputFormatter)
+		jsonOutputFormatter := json.NewOutputFormatter()
+		dt.JSStream = formatters.StartFormatIntoChannel[template.JS](ctx, table, jsonOutputFormatter)
 	} else {
-		dt.HTMLStream = formatters.StartFormatIntoChannel[template.HTML](ctx, d.OutputFormatter)
+		dt.HTMLStream = formatters.StartFormatIntoChannel[template.HTML](ctx, table, d.OutputFormatter)
 	}
 
 	// TODO(manuel, 2023-06-20) We need to properly pass the columns here, which can't be set upstream
 	// since we already pass in the JSStream here and we keep it, I think we are better off cloning the
 	// DataTables struct, or even separating it out to make d.dataTablesData immutable and just contain the
 	// toplevel config.
-	if d.OutputFormatter.Table != nil {
-		dt.Columns = d.OutputFormatter.Table.Columns
+	if table != nil {
+		dt.Columns = table.Columns
 	}
 
 	err := d.HTMLTemplateOutputFormatter.Template.Execute(w, dt)
@@ -213,7 +214,7 @@ func NewDataTablesCreateOutputProcessorFunc(
 	options ...DataTablesOutputFormatterOption,
 ) glazed.CreateProcessorFunc {
 	return func(c *gin.Context, pc *glazed.CommandContext) (
-		processor.Processor,
+		processor.TableProcessor,
 		error,
 	) {
 		// Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin
@@ -267,7 +268,10 @@ func NewDataTablesCreateOutputProcessorFunc(
 			options_...,
 		)
 
-		gp2 := processor.NewGlazeProcessor(of)
+		gp2, err := processor.NewGlazeProcessor(of)
+		if err != nil {
+			return nil, err
+		}
 
 		return gp2, nil
 	}

pkg/render/html.go

@@ -6,6 +6,7 @@ import (
 	"github.com/go-go-golems/glazed/pkg/formatters/table"
 	"github.com/go-go-golems/glazed/pkg/processor"
 	"github.com/go-go-golems/glazed/pkg/settings"
+	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/go-go-golems/parka/pkg/glazed"
 	"github.com/go-go-golems/parka/pkg/render/layout"
 	"html/template"
@@ -15,16 +16,16 @@ import (
 // NOTE(manuel, 2023-06-04): I don'Template think any of this is necessary.
 //
 // So it looks like the steps to output glazed data is to use a CreateProcessorFunc to create
-// a processor.Processor. Here we create a processor that uses a HTMLTemplateOutputFormatter (which
+// a processor.TableProcessor. Here we create a processor that uses a HTMLTemplateOutputFormatter (which
 // we are converting to a more specialized DataTableOutputFormatter), and then wrap all this through a
 // HTMLTableProcessor. But really the HTMLTableProcessor is just there to wrap the output formatter and
 // the template used. But the template used should be captured by the OutputFormatter in the first place.
 //
-// As such, we can use a generic Processor (why is there even a processor to be overloaded, if the definition of
-// processor.Processor is the following:
+// As such, we can use a generic TableProcessor (why is there even a processor to be overloaded, if the definition of
+// processor.TableProcessor is the following:
 //
-//type Processor interface {
-//	ProcessInputObject(ctx context.Context, obj map[string]interface{}) error
+//type TableProcessor interface {
+//	AddRow(ctx context.Context, obj map[string]interface{}) error
 //	OutputFormatter() formatters.OutputFormatter
 //}
 //
@@ -71,14 +72,13 @@ func NewHTMLTemplateOutputFormatter(
 	return ret
 }
 
-func (H *HTMLTemplateOutputFormatter) Output(ctx context.Context, w io.Writer) error {
+func (H *HTMLTemplateOutputFormatter) Output(ctx context.Context, table *types.Table, w io.Writer) error {
 	data := map[string]interface{}{}
 	for k, v := range H.Data {
 
 		data[k] = v
 	}
-	data["Columns"] = H.OutputFormatter.Table.Columns
-	data["Table"] = H.OutputFormatter.Table
+	data["Columns"] = table.Columns
 
 	err := H.Template.Execute(w, data)
 
@@ -97,7 +97,7 @@ func NewHTMLTemplateLookupCreateProcessorFunc(
 	options ...HTMLTemplateOutputFormatterOption,
 ) glazed.CreateProcessorFunc {
 	return func(c *gin.Context, pc *glazed.CommandContext) (
-		processor.Processor,
+		processor.TableProcessor,
 		error,
 	) {
 		// Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin
@@ -153,7 +153,10 @@ func NewHTMLTemplateLookupCreateProcessorFunc(
 		options_ = append(options_, options...)
 
 		of := NewHTMLTemplateOutputFormatter(t, gp.OutputFormatter().(*table.OutputFormatter), options_...)
-		gp2 := processor.NewGlazeProcessor(of)
+		gp2, err := processor.NewGlazeProcessor(of)
+		if err != nil {
+			return nil, err
+		}
 
 		return gp2, nil
 	}

pkg/server/server_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-go-golems/glazed/pkg/cmds"
 	"github.com/go-go-golems/glazed/pkg/cmds/layers"
 	"github.com/go-go-golems/glazed/pkg/processor"
+	"github.com/go-go-golems/glazed/pkg/types"
 	"github.com/go-go-golems/parka/pkg/server"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -26,12 +27,12 @@ func (t *TestCommand) Run(
 	ctx context.Context,
 	parsedLayers map[string]*layers.ParsedParameterLayer,
 	ps map[string]interface{},
-	gp processor.Processor,
+	gp processor.TableProcessor,
 ) error {
-	err := gp.ProcessInputObject(ctx, map[string]interface{}{
-		"foo": 1,
-		"bar": "baz",
-	})
+	err := gp.AddRow(ctx, types.NewRow(
+		types.MRP("foo", 1),
+		types.MRP("bar", "baz"),
+	))
 
 	if err != nil {
 		return err
@@ -58,7 +59,9 @@ func TestRunGlazedCommand(t *testing.T) {
 	t.Run("test-simple-command", func(t *testing.T) {
 		resp, err := http.Get(server.URL + "/test")
 		require.NoError(t, err)
-		defer resp.Body.Close()
+		defer func(Body io.ReadCloser) {
+			_ = Body.Close()
+		}(resp.Body)
 
 		body, err := io.ReadAll(resp.Body)
 		require.NoError(t, err)