From 70edecfcd9d55c29e0030abb407efafeabba039e Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Sat, 1 Jul 2023 11:04:37 -0400 Subject: [PATCH 01/11] :art: :poop: Further clean up formatter and middleware APIs - Add Close() to OutputFormatter - Add Close() to TableMiddleware - Add ColumnsChannelMiddleware - Continue messing with row output formatting in datatables.go --- cmd/parka/cmds/examples.go | 11 +- cmd/parka/cmds/serve.go | 5 +- pkg/glazed/handler.go | 148 +++++++------ pkg/handlers/command-dir/command-dir.go | 14 +- pkg/render/datatables/datatables.go | 276 ++++++++---------------- pkg/render/html.go | 128 ++++------- pkg/server/server_test.go | 5 +- 7 files changed, 232 insertions(+), 355 deletions(-) diff --git a/cmd/parka/cmds/examples.go b/cmd/parka/cmds/examples.go index be682ff..ec334fc 100644 --- a/cmd/parka/cmds/examples.go +++ b/cmd/parka/cmds/examples.go @@ -6,7 +6,7 @@ import ( "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" "github.com/go-go-golems/glazed/pkg/cmds/parameters" - "github.com/go-go-golems/glazed/pkg/processor" + "github.com/go-go-golems/glazed/pkg/middlewares" "github.com/go-go-golems/glazed/pkg/types" ) @@ -109,7 +109,7 @@ func (e *ExampleCommand) Run( ctx context.Context, parsedLayers map[string]*layers.ParsedParameterLayer, ps map[string]interface{}, - gp processor.TableProcessor, + gp middlewares.Processor, ) error { obj := types.NewRow( types.MRP("test", ps["test"]), @@ -149,6 +149,11 @@ func (e *ExampleCommand) Description() *cmds.CommandDescription { return e.description } -func (e *ExampleCommand) RunFromParka(c *gin.Context, parsedLayers map[string]*layers.ParsedParameterLayer, ps map[string]interface{}, gp *processor.GlazeProcessor) error { +func (e *ExampleCommand) RunFromParka( + c *gin.Context, + parsedLayers map[string]*layers.ParsedParameterLayer, + ps map[string]interface{}, + gp middlewares.Processor, +) error { return e.Run(c, parsedLayers, ps, gp) } diff --git a/cmd/parka/cmds/serve.go b/cmd/parka/cmds/serve.go index 095cf42..a839df2 100644 --- a/cmd/parka/cmds/serve.go +++ b/cmd/parka/cmds/serve.go @@ -117,10 +117,7 @@ var LsServerCmd = &cobra.Command{ cobra.CheckErr(err) } - err = gp.Finalize(ctx) - cobra.CheckErr(err) - - err = gp.OutputFormatter().Output(ctx, gp.GetTable(), os.Stdout) + err = gp.RunTableMiddlewares(ctx) cobra.CheckErr(err) }, } diff --git a/pkg/glazed/handler.go b/pkg/glazed/handler.go index 234cfb2..d60b2a3 100644 --- a/pkg/glazed/handler.go +++ b/pkg/glazed/handler.go @@ -4,25 +4,26 @@ import ( "bytes" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" + "github.com/go-go-golems/glazed/pkg/formatters" "github.com/go-go-golems/glazed/pkg/formatters/json" - "github.com/go-go-golems/glazed/pkg/processor" + "github.com/go-go-golems/glazed/pkg/middlewares" + "github.com/go-go-golems/glazed/pkg/middlewares/table" "github.com/go-go-golems/glazed/pkg/settings" "github.com/go-go-golems/parka/pkg/glazed/parser" + "golang.org/x/sync/errgroup" "io" "net/http" "os" ) -// CreateProcessorFunc is a simple func type to create a cmds.GlazeProcessor -// and formatters.OutputFormatter out of a CommandContext. -// -// This is so that we can create a processor that is configured based on the input -// data provided in CommandContext. For example, the user might want to request a specific response -// format through a query argument or through a header. -type CreateProcessorFunc func(c *gin.Context, pc *CommandContext) ( - processor.TableProcessor, - error, -) +type GinOutputFormatter interface { + Output(w io.Writer) error + RegisterMiddlewares(p *middlewares.TableProcessor) error +} + +type GinOutputFormatterFactory interface { + CreateOutputFormatter(c *gin.Context, pc *CommandContext) (GinOutputFormatter, error) +} // HandleOptions groups all the settings for a gin handler that handles a glazed command. type HandleOptions struct { @@ -43,7 +44,7 @@ type HandleOptions struct { Handlers []CommandHandlerFunc // CreateProcessor takes a gin.Context and a CommandContext and returns a processor.TableProcessor (and a content-type) - CreateProcessor CreateProcessorFunc + OutputFormatterFactory GinOutputFormatterFactory // This is the actual gin output writer Writer io.Writer @@ -53,10 +54,10 @@ type HandleOption func(*HandleOptions) func (h *HandleOptions) Copy(options ...HandleOption) *HandleOptions { ret := &HandleOptions{ - ParserOptions: h.ParserOptions, - Handlers: h.Handlers, - CreateProcessor: h.CreateProcessor, - Writer: h.Writer, + ParserOptions: h.ParserOptions, + Handlers: h.Handlers, + OutputFormatterFactory: h.OutputFormatterFactory, + Writer: h.Writer, } for _, option := range options { @@ -92,26 +93,20 @@ func WithWriter(w io.Writer) HandleOption { } } -func WithCreateProcessor(createProcessor CreateProcessorFunc) HandleOption { - return func(o *HandleOptions) { - o.CreateProcessor = createProcessor - } -} - func CreateJSONProcessor(_ *gin.Context, pc *CommandContext) ( - processor.TableProcessor, + *middlewares.TableProcessor, error, ) { l, ok := pc.ParsedLayers["glazed"] l.Parameters["output"] = "json" - var gp *processor.GlazeProcessor + var gp *middlewares.TableProcessor var err error if ok { - gp, err = settings.SetupProcessor(l.Parameters) + gp, err = settings.SetupTableProcessor(l.Parameters) } else { - gp, err = settings.SetupProcessor(map[string]interface{}{ + gp, err = settings.SetupTableProcessor(map[string]interface{}{ "output": "json", }) } @@ -146,7 +141,7 @@ func GinHandleGlazedCommand( // running the GlazeCommand, and then returning the output file as an attachment. // This usually requires the caller to provide a temporary file path. // -// TODO(manuel, 2023-06-22) Now that OutputFormatter renders directly into a io.Writer, +// TODO(manuel, 2023-06-22) Now that TableOutputFormatter renders directly into a io.Writer, // I don't think we need all this anymore, we just need to set the relevant header. func GinHandleGlazedCommandWithOutputFile( cmd cmds.GlazeCommand, @@ -198,60 +193,83 @@ func runGlazeCommand(c *gin.Context, cmd cmds.GlazeCommand, opts *HandleOptions) } } - var gp processor.TableProcessor - + var gp *middlewares.TableProcessor var err error - if opts.CreateProcessor != nil { - // TODO(manuel, 2023-03-02) We might want to switch on the requested content type here too - // This would be done by passing in a handler that configures the glazed layer accordingly. - gp, err = opts.CreateProcessor(c, pc) + + glazedLayer := pc.ParsedLayers["glazed"] + + if glazedLayer != nil { + gp, err = settings.SetupTableProcessor(glazedLayer.Parameters) + if err != nil { + return err + } } else { - gp, err = SetupProcessor(pc) + gp = middlewares.NewTableProcessor() } - if err != nil { - return err + + var writer io.Writer = c.Writer + if opts.Writer != nil { + writer = opts.Writer } - of := gp.OutputFormatter() - contentType := of.ContentType() + if opts.OutputFormatterFactory != nil { + of, err := opts.OutputFormatterFactory.CreateOutputFormatter(c, pc) + if err != nil { + return err + } + // remove table middlewares to do streaming rows + gp.ReplaceTableMiddleware() - if opts.Writer == nil { - c.Writer.Header().Set("Content-Type", contentType) - } + // create rowOutputChannelMiddleware here? But that's actually a responsibility of the OutputFormatterFactory. + // we need to create these before running the command, and we need to figure out a way to get the Columns. - err = cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) - if err != nil { - return err + err = of.RegisterMiddlewares(gp) + if err != nil { + return err + } + + eg := &errgroup.Group{} + eg.Go(func() error { + return cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) + }) + + eg.Go(func() error { + // we somehow need to pass the channels to the OutputFormatterFactory + return of.Output(writer) + }) + + // no cancellation on error? + + return eg.Wait() } - var writer io.Writer = c.Writer - if opts.Writer != nil { - writer = opts.Writer + // here we run a normal full table render + var of formatters.TableOutputFormatter + if glazedLayer != nil { + of, err = settings.SetupTableOutputFormatter(glazedLayer.Parameters) + if err != nil { + return err + } + } else { + of = json.NewOutputFormatter( + json.WithOutputIndividualRows(true), + ) } - err = of.Output(c, gp.GetTable(), writer) - if err != nil { - return err + if opts.Writer == nil { + c.Writer.Header().Set("Content-Type", of.ContentType()) } - return err -} + gp.AddTableMiddleware(table.NewOutputMiddleware(of, writer)) -// SetupProcessor creates a new cmds.GlazeProcessor. It uses the parsed layer glazed if present, and return -// a simple JsonOutputFormatter and standard glazed processor otherwise. -func SetupProcessor(pc *CommandContext, options ...processor.GlazeProcessorOption) (*processor.GlazeProcessor, error) { - l, ok := pc.ParsedLayers["glazed"] - if ok { - gp, err := settings.SetupProcessor(l.Parameters) - return gp, err + err = cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) + if err != nil { + return err } - of := json.NewOutputFormatter( - json.WithOutputIndividualRows(true), - ) - gp, err := processor.NewGlazeProcessor(of, options...) + err = gp.RunTableMiddlewares(c) if err != nil { - return nil, err + return err } - return gp, nil + return nil } diff --git a/pkg/handlers/command-dir/command-dir.go b/pkg/handlers/command-dir/command-dir.go index 1c55606..76d5597 100644 --- a/pkg/handlers/command-dir/command-dir.go +++ b/pkg/handlers/command-dir/command-dir.go @@ -480,13 +480,19 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { // See https://github.com/go-go-golems/sqleton/issues/162 _ = cd.IndexTemplateName + dt := &datatables.DataTables{ + Command: sqlCommand.Description(), + Links: links, + BasePath: path, + JSRendering: true, + UseDataTables: false, + AdditionalData: cd.AdditionalData, + } + dataTablesProcessorFunc := datatables.NewDataTablesCreateOutputProcessorFunc( cd.TemplateLookup, cd.TemplateName, - datatables.WithLinks(links...), - datatables.WithJSRendering(), - datatables.WithAdditionalData(cd.AdditionalData), - datatables.WithBasePath(path), + dt, ) handle := server.HandleSimpleQueryCommand( diff --git a/pkg/render/datatables/datatables.go b/pkg/render/datatables/datatables.go index dd547f8..d2a6950 100644 --- a/pkg/render/datatables/datatables.go +++ b/pkg/render/datatables/datatables.go @@ -1,15 +1,14 @@ package datatables import ( - "context" "embed" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/formatters" "github.com/go-go-golems/glazed/pkg/formatters/json" - "github.com/go-go-golems/glazed/pkg/formatters/table" - "github.com/go-go-golems/glazed/pkg/processor" - "github.com/go-go-golems/glazed/pkg/settings" + table_formatter "github.com/go-go-golems/glazed/pkg/formatters/table" + "github.com/go-go-golems/glazed/pkg/middlewares" + "github.com/go-go-golems/glazed/pkg/middlewares/row" "github.com/go-go-golems/glazed/pkg/types" "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/render" @@ -56,94 +55,6 @@ type DataTables struct { AdditionalData map[string]interface{} } -type DataTablesOutputFormatter struct { - *render.HTMLTemplateOutputFormatter - dataTablesData *DataTables -} - -type DataTablesOutputFormatterOption func(*DataTablesOutputFormatter) - -func WithCommand(cmd *cmds.CommandDescription) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Command = cmd - } -} - -func WithLongDescription(desc string) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.LongDescription = desc - } -} - -func WithReplaceAdditionalData(data map[string]interface{}) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.AdditionalData = data - } -} - -func WithAdditionalData(data map[string]interface{}) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - if d.dataTablesData.AdditionalData == nil { - d.dataTablesData.AdditionalData = data - } else { - for k, v := range data { - d.dataTablesData.AdditionalData[k] = v - } - } - } -} - -// WithJSRendering enables JS rendering for the DataTables renderer. -// This means that we will render the table into the toplevel element -// `tableData` in javascript, and not call the parent output formatter -func WithJSRendering() DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.JSRendering = true - } -} - -func WithLayout(l *layout.Layout) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Layout = l - } -} - -func WithLinks(links ...layout.Link) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Links = links - } -} - -func WithBasePath(path string) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.BasePath = path - } -} - -func WithAppendLinks(links ...layout.Link) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Links = append(d.dataTablesData.Links, links...) - } -} - -func WithPrependLinks(links ...layout.Link) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Links = append(links, d.dataTablesData.Links...) - } -} - -func WithColumns(columns ...string) DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.Columns = columns - } -} - -func WithUseDataTables() DataTablesOutputFormatterOption { - return func(d *DataTablesOutputFormatter) { - d.dataTablesData.UseDataTables = true - } -} - //go:embed templates/* var templateFS embed.FS @@ -159,120 +70,113 @@ func NewDataTablesLookupTemplate() *render.LookupTemplateFromFS { return l } -func NewDataTablesOutputFormatter( +func (dt *DataTables) Clone() *DataTables { + return &DataTables{ + Command: dt.Command, + LongDescription: dt.LongDescription, + Layout: dt.Layout, + Links: dt.Links, + BasePath: dt.BasePath, + JSStream: dt.JSStream, + HTMLStream: dt.HTMLStream, + JSRendering: dt.JSRendering, + Columns: dt.Columns, + UseDataTables: dt.UseDataTables, + AdditionalData: dt.AdditionalData, + } +} + +type OutputFormatter struct { + t *template.Template + dt *DataTables + // rowC is the channel where the rows are sent to. They will need to get converted + // to template.JS or template.HTML before being sent to either + rowC chan string + // columnsC is the channel where the column names are sent to. Since the row.ColumnsChannelMiddleware + // instance that sends columns to this channel is running before the row firmware, we should be careful + // about not blocking. Potentially, this could be done by starting a goroutine in the middleware, + // since we have a context there, and there is no need to block the middleware processing. + columnsC chan []types.FieldName +} + +func NewOutputFormatter( t *template.Template, - of *table.OutputFormatter, - options ...DataTablesOutputFormatterOption, -) *DataTablesOutputFormatter { - ret := &DataTablesOutputFormatter{ - HTMLTemplateOutputFormatter: render.NewHTMLTemplateOutputFormatter(t, of), - dataTablesData: &DataTables{}, - } + dt *DataTables) *OutputFormatter { - for _, option := range options { - option(ret) - } + // make the NewOutputChannelMiddleware generic to send string/template.JS/template.HTML over the wire + rowC := make(chan string, 100) + + // make a channel to receive column names + columnsC := make(chan []types.FieldName, 10) - return ret + // we need to make sure that we are closing the channel correctly. Should middlewares have a Close method? + // that actually sounds reasonable + return &OutputFormatter{ + t: t, + dt: dt, + rowC: rowC, + columnsC: columnsC, + } } -func (d *DataTablesOutputFormatter) ContentType() string { - return "text/html; charset=utf-8" +type OutputFormatterFactory struct { + TemplateName string + Lookup render.TemplateLookup + DataTables *DataTables } -func (d *DataTablesOutputFormatter) Output(ctx context.Context, table *types.Table, w io.Writer) error { - dt := d.dataTablesData - if dt.JSRendering { - jsonOutputFormatter := json.NewOutputFormatter() - dt.JSStream = formatters.StartFormatIntoChannel[template.JS](ctx, table, jsonOutputFormatter) - } else { - dt.HTMLStream = formatters.StartFormatIntoChannel[template.HTML](ctx, table, d.OutputFormatter) +func (dtoff *OutputFormatterFactory) CreateOutputFormatter( + c *gin.Context, + pc *glazed.CommandContext, +) (*OutputFormatter, error) { + // Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin + // server. + t, err := dtoff.Lookup.Lookup(dtoff.TemplateName) + if err != nil { + return nil, err } - // TODO(manuel, 2023-06-20) We need to properly pass the columns here, which can't be set upstream - // since we already pass in the JSStream here and we keep it, I think we are better off cloning the - // DataTables struct, or even separating it out to make d.dataTablesData immutable and just contain the - // toplevel config. - if table != nil { - dt.Columns = table.Columns + layout_, err := layout.ComputeLayout(pc) + if err != nil { + return nil, err } - err := d.HTMLTemplateOutputFormatter.Template.Execute(w, dt) + description := pc.Cmd.Description() + dt_ := dtoff.DataTables.Clone() + longHTML, err := render.RenderMarkdownToHTML(description.Long) if err != nil { - return err + return nil, err } - return nil -} - -// NewDataTablesCreateOutputProcessorFunc creates a glazed.CreateProcessorFunc based on a TemplateLookup -// and a template name. -func NewDataTablesCreateOutputProcessorFunc( - lookup render.TemplateLookup, - templateName string, - options ...DataTablesOutputFormatterOption, -) glazed.CreateProcessorFunc { - return func(c *gin.Context, pc *glazed.CommandContext) ( - processor.TableProcessor, - error, - ) { - // Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin - // server. - t, err := lookup.Lookup(templateName) - if err != nil { - return nil, err - } - - l, ok := pc.ParsedLayers["glazed"] - var gp *processor.GlazeProcessor - - if ok { - l.Parameters["output"] = "table" - l.Parameters["table-format"] = "html" - - gp, err = settings.SetupProcessor(l.Parameters) - } else { - gp, err = settings.SetupProcessor(map[string]interface{}{ - "output": "table", - "table-format": "html", - }) - } - - if err != nil { - return nil, err - } + dt_.LongDescription = longHTML + dt_.Layout = layout_ - layout_, err := layout.ComputeLayout(pc) - if err != nil { - return nil, err - } + return NewOutputFormatter(t, dtoff.DataTables), nil +} - description := pc.Cmd.Description() +func (dt *OutputFormatter) RegisterMiddlewares( + m *middlewares.TableProcessor, +) error { + var of formatters.RowOutputFormatter + if dt.dt.JSRendering { + of = json.NewOutputFormatter() + dt.dt.JSStream = make(chan template.JS, 100) + } else { + of = table_formatter.NewOutputFormatter("html") + dt.dt.HTMLStream = make(chan template.HTML, 100) + } - longHTML, err := render.RenderMarkdownToHTML(description.Long) - if err != nil { - return nil, err - } + // TODO add a "get columns" middleware that can be used to get the columns from the table + // over a single channel that we can wait on in the render call - options_ := []DataTablesOutputFormatterOption{ - WithCommand(description), - WithLongDescription(longHTML), - WithLayout(layout_), - } - options_ = append(options_, options...) + m.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) - of := NewDataTablesOutputFormatter( - t, - gp.OutputFormatter().(*table.OutputFormatter), - options_..., - ) + return nil +} - gp2, err := processor.NewGlazeProcessor(of) - if err != nil { - return nil, err - } +func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { + // clear all table middlewares because we are streaming using a row output middleware - return gp2, nil - } + return nil } diff --git a/pkg/render/html.go b/pkg/render/html.go index 893885b..666675f 100644 --- a/pkg/render/html.go +++ b/pkg/render/html.go @@ -1,12 +1,7 @@ package render import ( - "context" "github.com/gin-gonic/gin" - "github.com/go-go-golems/glazed/pkg/formatters/table" - "github.com/go-go-golems/glazed/pkg/processor" - "github.com/go-go-golems/glazed/pkg/settings" - "github.com/go-go-golems/glazed/pkg/types" "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/render/layout" "html/template" @@ -19,14 +14,14 @@ import ( // a processor.TableProcessor. Here we create a processor that uses a HTMLTemplateOutputFormatter (which // we are converting to a more specialized DataTableOutputFormatter), and then wrap all this through a // HTMLTableProcessor. But really the HTMLTableProcessor is just there to wrap the output formatter and -// the template used. But the template used should be captured by the OutputFormatter in the first place. +// the template used. But the template used should be captured by the TableOutputFormatter in the first place. // // As such, we can use a generic TableProcessor (why is there even a processor to be overloaded, if the definition of // processor.TableProcessor is the following: // //type TableProcessor interface { // AddRow(ctx context.Context, obj map[string]interface{}) error -// OutputFormatter() formatters.OutputFormatter +// TableOutputFormatter() formatters.TableOutputFormatter //} // // Probably because we use the processor.GlazeProcessor class as a helper, which is able to handle the different @@ -36,10 +31,9 @@ import ( // HTMLTemplateOutputFormatter wraps a normal HTML table output formatter, and allows // a template to be added in the back in the front. type HTMLTemplateOutputFormatter struct { - // We use a table outputFormatter because we need to access the Table itself. - *table.OutputFormatter - Template *template.Template - Data map[string]interface{} + TemplateName string + Lookup TemplateLookup + Data map[string]interface{} } type HTMLTemplateOutputFormatterOption func(*HTMLTemplateOutputFormatter) @@ -56,13 +50,13 @@ func WithHTMLTemplateOutputFormatterData(data map[string]interface{}) HTMLTempla } func NewHTMLTemplateOutputFormatter( - t *template.Template, - of *table.OutputFormatter, + lookup TemplateLookup, + templateName string, options ...HTMLTemplateOutputFormatterOption, ) *HTMLTemplateOutputFormatter { ret := &HTMLTemplateOutputFormatter{ - OutputFormatter: of, - Template: t, + TemplateName: templateName, + Lookup: lookup, } for _, option := range options { @@ -72,92 +66,46 @@ func NewHTMLTemplateOutputFormatter( return ret } -func (H *HTMLTemplateOutputFormatter) Output(ctx context.Context, table *types.Table, w io.Writer) error { - data := map[string]interface{}{} - for k, v := range H.Data { +func (H *HTMLTemplateOutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { + // Here, we use the parsed layer to configure the glazed middlewares. + // We then use the created formatters.TableOutputFormatter as a basis for + // our own output formatter that renders into an HTML template. + var err error - data[k] = v + layout_, err := layout.ComputeLayout(pc) + if err != nil { + return err } - data["Columns"] = table.Columns - err := H.Template.Execute(w, data) + description := pc.Cmd.Description() + longHTML, err := RenderMarkdownToHTML(description.Long) if err != nil { return err } - return err -} - -// NewHTMLTemplateLookupCreateProcessorFunc creates a glazed.CreateProcessorFunc based on a TemplateLookup -// and a template name. -func NewHTMLTemplateLookupCreateProcessorFunc( - lookup TemplateLookup, - templateName string, - options ...HTMLTemplateOutputFormatterOption, -) glazed.CreateProcessorFunc { - return func(c *gin.Context, pc *glazed.CommandContext) ( - processor.TableProcessor, - error, - ) { - // Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin - // server. - t, err := lookup.Lookup(templateName) - if err != nil { - return nil, err - } - - // Here, we use the parsed layer to configure the glazed middlewares. - // We then use the created formatters.OutputFormatter as a basis for - // our own output formatter that renders into an HTML template. - l, ok := pc.ParsedLayers["glazed"] - - var gp *processor.GlazeProcessor - - if ok { - l.Parameters["output"] = "table" - l.Parameters["table-format"] = "html" - - gp, err = settings.SetupProcessor(l.Parameters) - } else { - gp, err = settings.SetupProcessor(map[string]interface{}{ - "output": "table", - "table-format": "html", - }) - } - - if err != nil { - return nil, err - } - - layout_, err := layout.ComputeLayout(pc) - if err != nil { - return nil, err - } - - description := pc.Cmd.Description() + data := map[string]interface{}{} + for k, v := range H.Data { + data[k] = v + } + data["Command"] = description + data["LongDescription"] = template.HTML(longHTML) + data["Layout"] = layout_ - longHTML, err := RenderMarkdownToHTML(description.Long) - if err != nil { - return nil, err - } + // TODO(manuel, 2023-06-30) Get the column names out of a RowOutputMiddleware + //data["Columns"] = table.Columns - options_ := []HTMLTemplateOutputFormatterOption{ - WithHTMLTemplateOutputFormatterData( - map[string]interface{}{ - "Command": description, - "LongDescription": template.HTML(longHTML), - "Layout": layout_, - }), - } - options_ = append(options_, options...) + t, err := H.Lookup.Lookup(H.TemplateName) + if err != nil { + return err + } - of := NewHTMLTemplateOutputFormatter(t, gp.OutputFormatter().(*table.OutputFormatter), options_...) - gp2, err := processor.NewGlazeProcessor(of) - if err != nil { - return nil, err - } + // TODO: we are missing the background processing of the rows here - return gp2, nil + err = t.Execute(w, data) + if err != nil { + return err } + + return err } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index dabbb8b..107d696 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -6,7 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" - "github.com/go-go-golems/glazed/pkg/processor" + "github.com/go-go-golems/glazed/pkg/middlewares" "github.com/go-go-golems/glazed/pkg/types" "github.com/go-go-golems/parka/pkg/server" "github.com/stretchr/testify/assert" @@ -27,7 +27,7 @@ func (t *TestCommand) Run( ctx context.Context, parsedLayers map[string]*layers.ParsedParameterLayer, ps map[string]interface{}, - gp processor.TableProcessor, + gp middlewares.Processor, ) error { err := gp.AddRow(ctx, types.NewRow( types.MRP("foo", 1), @@ -75,5 +75,4 @@ func TestRunGlazedCommand(t *testing.T) { assert.Equal(t, float64(1), v["foo"]) assert.Equal(t, "baz", v["bar"]) }) - } From 3b42d535873f5d0af523e4890f1d82d0348e5c37 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Sun, 2 Jul 2023 15:10:33 -0400 Subject: [PATCH 02/11] :sparkles: First JSON streaming handler! --- pkg/glazed/command.go | 143 ++++++---- pkg/glazed/handler.go | 269 ++---------------- pkg/glazed/handlers/helpers.go | 30 ++ pkg/glazed/handlers/json/json.go | 96 +++++++ pkg/handlers/command-dir/command-dir.go | 44 +-- .../{ => formatters}/datatables/datatables.go | 124 ++++++-- .../templates/data-tables.tmpl.html | 0 pkg/render/{ => formatters}/html.go | 9 +- pkg/server/server.go | 51 +--- pkg/server/server_test.go | 2 +- 10 files changed, 376 insertions(+), 392 deletions(-) create mode 100644 pkg/glazed/handlers/helpers.go create mode 100644 pkg/glazed/handlers/json/json.go rename pkg/render/{ => formatters}/datatables/datatables.go (72%) rename pkg/render/{ => formatters}/datatables/templates/data-tables.tmpl.html (100%) rename pkg/render/{ => formatters}/html.go (94%) diff --git a/pkg/glazed/command.go b/pkg/glazed/command.go index a5a6d9c..fdaa9cf 100644 --- a/pkg/glazed/command.go +++ b/pkg/glazed/command.go @@ -74,45 +74,64 @@ func (pc *CommandContext) GetAllParameterValues() map[string]interface{} { return ret } -// CommandHandlerFunc mirrors gin's HandlerFunc, but also gets passed a CommandContext. -// That allows it to reuse data from the gin.Context, most importantly the request itself. -type CommandHandlerFunc func(*gin.Context, *CommandContext) error - -// HandlePrepopulatedParameters sets the given parameters in the CommandContext's ParsedParameters. -// If any of the given parameters also belong to a layer, they are also set there. -func HandlePrepopulatedParameters(ps map[string]interface{}) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - for k, v := range ps { - pc.ParsedParameters[k] = v - - // Now check if the parameter is in any of the layers, and if so, set it there as well - for _, layer := range pc.ParsedLayers { - if _, ok := layer.Parameters[k]; ok { - layer.Parameters[k] = v - } +// ContextMiddleware is used to build up a CommandContext. It is similar to the HandlerFunc +// in gin, but instead of handling the HTTP response, it is only used to build up the CommandContext. +type ContextMiddleware interface { + Handle(*gin.Context, *CommandContext) error +} + +// PrepopulatedContextMiddleware is a ContextMiddleware that prepopulates the CommandContext with +// the given map of parameters. It overwrites existing parameters in pc.ParsedParameters, +// and overwrites parameters in individual parser layers if they have already been set. +type PrepopulatedContextMiddleware struct { + ps map[string]interface{} +} + +func (p *PrepopulatedContextMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + for k, v := range p.ps { + pc.ParsedParameters[k] = v + + // Now check if the parameter is in any of the layers, and if so, set it there as well + for _, layer := range pc.ParsedLayers { + if _, ok := layer.Parameters[k]; ok { + layer.Parameters[k] = v } } - return nil - } -} - -// HandlePrepopulatedParsedLayers sets the given layers in the CommandContext's Layers, -// overriding the parameters of any layers that are already present. -// This means that if a parameter is not set in layers_ but is set in the Layers, -// the value in the Layers will be kept. -func HandlePrepopulatedParsedLayers(layers_ map[string]*layers.ParsedParameterLayer) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - for k, v := range layers_ { - parsedLayer, ok := pc.ParsedLayers[k] - if ok { - for k2, v2 := range v.Parameters { - parsedLayer.Parameters[k2] = v2 - } - } else { - pc.ParsedLayers[k] = v + } + return nil +} + +func NewPrepopulatedContextMiddleware(ps map[string]interface{}) *PrepopulatedContextMiddleware { + return &PrepopulatedContextMiddleware{ + ps: ps, + } +} + +// PrepopulatedParsedLayersContextMiddleware is a ContextMiddleware that prepopulates the CommandContext with +// the given map of layers. If a layer already exists, it overwrites its parameters. +type PrepopulatedParsedLayersContextMiddleware struct { + layers map[string]*layers.ParsedParameterLayer +} + +func (p *PrepopulatedParsedLayersContextMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + for k, v := range p.layers { + parsedLayer, ok := pc.ParsedLayers[k] + if ok { + for k2, v2 := range v.Parameters { + parsedLayer.Parameters[k2] = v2 } + } else { + pc.ParsedLayers[k] = v } - return nil + } + return nil +} + +func NewPrepopulatedParsedLayersContextMiddleware( + layers map[string]*layers.ParsedParameterLayer, +) *PrepopulatedParsedLayersContextMiddleware { + return &PrepopulatedParsedLayersContextMiddleware{ + layers: layers, } } @@ -164,31 +183,39 @@ func NewCommandFormParser(cmd cmds.GlazeCommand, options ...parser.ParserOption) return ph } -// NewParserCommandHandlerFunc creates a CommandHandlerFunc using the given parser.Parser. -// It also first establishes a set of defaults by loading them from an alias definition. -func NewParserCommandHandlerFunc(cmd cmds.GlazeCommand, parserHandler *parser.Parser) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - parseState := parser.NewParseStateFromCommandDescription(cmd) - err := parserHandler.Parse(c, parseState) - if err != nil { - return err - } +type ContextParserMiddleware struct { + command cmds.GlazeCommand + parser *parser.Parser +} - pc.ParsedParameters = parseState.FlagsAndArguments.Parameters - pc.ParsedLayers = map[string]*layers.ParsedParameterLayer{} - commandLayers := pc.Cmd.Description().Layers - for _, v := range commandLayers { - parsedParameterLayer := &layers.ParsedParameterLayer{ - Layer: v, - Parameters: map[string]interface{}{}, - } - parsedLayers, ok := parseState.Layers[v.GetSlug()] - if ok { - parsedParameterLayer.Parameters = parsedLayers.Parameters - } - pc.ParsedLayers[v.GetSlug()] = parsedParameterLayer +func (cpm *ContextParserMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + parseState := parser.NewParseStateFromCommandDescription(cpm.command) + err := cpm.parser.Parse(c, parseState) + if err != nil { + return err + } + + pc.ParsedParameters = parseState.FlagsAndArguments.Parameters + pc.ParsedLayers = map[string]*layers.ParsedParameterLayer{} + commandLayers := pc.Cmd.Description().Layers + for _, v := range commandLayers { + parsedParameterLayer := &layers.ParsedParameterLayer{ + Layer: v, + Parameters: map[string]interface{}{}, + } + parsedLayers, ok := parseState.Layers[v.GetSlug()] + if ok { + parsedParameterLayer.Parameters = parsedLayers.Parameters } + pc.ParsedLayers[v.GetSlug()] = parsedParameterLayer + } + + return nil +} - return nil +func NewContextParserMiddleware(cmd cmds.GlazeCommand, parser *parser.Parser) *ContextParserMiddleware { + return &ContextParserMiddleware{ + command: cmd, + parser: parser, } } diff --git a/pkg/glazed/handler.go b/pkg/glazed/handler.go index d60b2a3..97bc2f6 100644 --- a/pkg/glazed/handler.go +++ b/pkg/glazed/handler.go @@ -3,270 +3,51 @@ package glazed import ( "bytes" "github.com/gin-gonic/gin" - "github.com/go-go-golems/glazed/pkg/cmds" - "github.com/go-go-golems/glazed/pkg/formatters" - "github.com/go-go-golems/glazed/pkg/formatters/json" - "github.com/go-go-golems/glazed/pkg/middlewares" - "github.com/go-go-golems/glazed/pkg/middlewares/table" - "github.com/go-go-golems/glazed/pkg/settings" - "github.com/go-go-golems/parka/pkg/glazed/parser" - "golang.org/x/sync/errgroup" "io" - "net/http" "os" + "path/filepath" ) -type GinOutputFormatter interface { - Output(w io.Writer) error - RegisterMiddlewares(p *middlewares.TableProcessor) error +type Handler interface { + Handle(c *gin.Context, w io.Writer) error } -type GinOutputFormatterFactory interface { - CreateOutputFormatter(c *gin.Context, pc *CommandContext) (GinOutputFormatter, error) +type OutputFileHandler struct { + handler Handler + outputFileName string } -// HandleOptions groups all the settings for a gin handler that handles a glazed command. -type HandleOptions struct { - // ParserOptions are passed to the given parser (the thing that gathers the glazed.Command - // flags and arguments. - ParserOptions []parser.ParserOption - - // Handlers are run right at the start of the gin.Handler to build up the CommandContext based on the - // gin.Context. They can be chained because they get passed the previous CommandContext. - // - // NOTE(manuel, 2023-06-22) We currently use a single CommandHandler, which is created with NewParserCommandHandlerFunc. - // This creates a command handler that uses a parser.Parser to parse the gin.Context and return a CommandContext. - // For example, the FormParser will parse command parameters passed as a HTML form. - // - // While we currently only use a single handler, the current setup allows us to chain a middleware of handlers. - // This would potentially allow us to catch parse errors and return an appropriate error template - // I'm not entirely sure if this all makes sense. - Handlers []CommandHandlerFunc - - // CreateProcessor takes a gin.Context and a CommandContext and returns a processor.TableProcessor (and a content-type) - OutputFormatterFactory GinOutputFormatterFactory - - // This is the actual gin output writer - Writer io.Writer -} - -type HandleOption func(*HandleOptions) - -func (h *HandleOptions) Copy(options ...HandleOption) *HandleOptions { - ret := &HandleOptions{ - ParserOptions: h.ParserOptions, - Handlers: h.Handlers, - OutputFormatterFactory: h.OutputFormatterFactory, - Writer: h.Writer, - } - - for _, option := range options { - option(ret) - } - - return ret -} - -func NewHandleOptions(options []HandleOption) *HandleOptions { - opts := &HandleOptions{} - for _, option := range options { - option(opts) - } - return opts -} - -func WithParserOptions(parserOptions ...parser.ParserOption) HandleOption { - return func(o *HandleOptions) { - o.ParserOptions = parserOptions +func NewOutputFileHandler(handler Handler, outputFileName string) *OutputFileHandler { + h := &OutputFileHandler{ + handler: handler, + outputFileName: outputFileName, } -} - -func WithHandlers(handlers ...CommandHandlerFunc) HandleOption { - return func(o *HandleOptions) { - o.Handlers = handlers - } -} -func WithWriter(w io.Writer) HandleOption { - return func(o *HandleOptions) { - o.Writer = w - } + return h } -func CreateJSONProcessor(_ *gin.Context, pc *CommandContext) ( - *middlewares.TableProcessor, - error, -) { - l, ok := pc.ParsedLayers["glazed"] - l.Parameters["output"] = "json" - - var gp *middlewares.TableProcessor - var err error - - if ok { - gp, err = settings.SetupTableProcessor(l.Parameters) - } else { - gp, err = settings.SetupTableProcessor(map[string]interface{}{ - "output": "json", - }) - } - +func (h *OutputFileHandler) Handle(c *gin.Context, w io.Writer) error { + buf := &bytes.Buffer{} + err := h.handler.Handle(c, buf) if err != nil { - return nil, err - } - - return gp, nil -} - -// GinHandleGlazedCommand returns a gin.HandlerFunc that runs a glazed.Command and writes -// the results to the gin.Context ResponseWriter. -func GinHandleGlazedCommand( - cmd cmds.GlazeCommand, - opts *HandleOptions, -) gin.HandlerFunc { - return func(c *gin.Context) { - err := runGlazeCommand(c, cmd, opts) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": err.Error(), - }) - return - } - - c.Status(200) - } -} - -// GinHandleGlazedCommandWithOutputFile returns a gin.HandlerFunc that is responsible for -// running the GlazeCommand, and then returning the output file as an attachment. -// This usually requires the caller to provide a temporary file path. -// -// TODO(manuel, 2023-06-22) Now that TableOutputFormatter renders directly into a io.Writer, -// I don't think we need all this anymore, we just need to set the relevant header. -func GinHandleGlazedCommandWithOutputFile( - cmd cmds.GlazeCommand, - outputFile string, - fileName string, - opts *HandleOptions, -) gin.HandlerFunc { - return func(c *gin.Context) { - buf := &bytes.Buffer{} - opts_ := opts.Copy(WithWriter(buf)) - err := runGlazeCommand(c, cmd, opts_) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": err.Error(), - }) - return - } - - c.Status(200) - - f, err := os.Open(outputFile) - if err != nil { - _ = c.AbortWithError(http.StatusInternalServerError, err) - return - } - defer func(f *os.File) { - _ = f.Close() - }(f) - - c.Writer.Header().Set("Content-Disposition", "attachment; filename="+fileName) - - _, err = io.Copy(c.Writer, f) - if err != nil { - if err != nil { - _ = c.AbortWithError(http.StatusInternalServerError, err) - return - } - } - } -} - -func runGlazeCommand(c *gin.Context, cmd cmds.GlazeCommand, opts *HandleOptions) error { - pc := NewCommandContext(cmd) - - for _, h := range opts.Handlers { - err := h(c, pc) - if err != nil { - return err - } - } - - var gp *middlewares.TableProcessor - var err error - - glazedLayer := pc.ParsedLayers["glazed"] - - if glazedLayer != nil { - gp, err = settings.SetupTableProcessor(glazedLayer.Parameters) - if err != nil { - return err - } - } else { - gp = middlewares.NewTableProcessor() - } - - var writer io.Writer = c.Writer - if opts.Writer != nil { - writer = opts.Writer - } - - if opts.OutputFormatterFactory != nil { - of, err := opts.OutputFormatterFactory.CreateOutputFormatter(c, pc) - if err != nil { - return err - } - // remove table middlewares to do streaming rows - gp.ReplaceTableMiddleware() - - // create rowOutputChannelMiddleware here? But that's actually a responsibility of the OutputFormatterFactory. - // we need to create these before running the command, and we need to figure out a way to get the Columns. - - err = of.RegisterMiddlewares(gp) - if err != nil { - return err - } - - eg := &errgroup.Group{} - eg.Go(func() error { - return cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) - }) - - eg.Go(func() error { - // we somehow need to pass the channels to the OutputFormatterFactory - return of.Output(writer) - }) - - // no cancellation on error? - - return eg.Wait() - } - - // here we run a normal full table render - var of formatters.TableOutputFormatter - if glazedLayer != nil { - of, err = settings.SetupTableOutputFormatter(glazedLayer.Parameters) - if err != nil { - return err - } - } else { - of = json.NewOutputFormatter( - json.WithOutputIndividualRows(true), - ) - } - if opts.Writer == nil { - c.Writer.Header().Set("Content-Type", of.ContentType()) + return err } - gp.AddTableMiddleware(table.NewOutputMiddleware(of, writer)) + c.Status(200) - err = cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) + f, err := os.Open(h.outputFileName) if err != nil { return err } + defer func(f *os.File) { + _ = f.Close() + }(f) + + baseName := filepath.Base(h.outputFileName) + + c.Writer.Header().Set("Content-Disposition", "attachment; filename="+baseName) - err = gp.RunTableMiddlewares(c) + _, err = io.Copy(c.Writer, f) if err != nil { return err } diff --git a/pkg/glazed/handlers/helpers.go b/pkg/glazed/handlers/helpers.go new file mode 100644 index 0000000..2ccd921 --- /dev/null +++ b/pkg/glazed/handlers/helpers.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "github.com/go-go-golems/glazed/pkg/middlewares" + "github.com/go-go-golems/glazed/pkg/settings" + "github.com/go-go-golems/parka/pkg/glazed" +) + +func CreateTableProcessor(pc *glazed.CommandContext, outputType string, tableType string) (*middlewares.TableProcessor, error) { + var gp *middlewares.TableProcessor + var err error + + glazedLayer := pc.ParsedLayers["glazed"] + + if glazedLayer != nil { + glazedLayer.Parameters["output"] = outputType + glazedLayer.Parameters["table"] = tableType + gp, err = settings.SetupTableProcessor(glazedLayer.Parameters) + if err != nil { + return nil, err + } + } else { + gp, err = settings.SetupTableProcessor(map[string]interface{}{ + "output": outputType, + "table": tableType, + }) + } + + return gp, err +} diff --git a/pkg/glazed/handlers/json/json.go b/pkg/glazed/handlers/json/json.go new file mode 100644 index 0000000..e237788 --- /dev/null +++ b/pkg/glazed/handlers/json/json.go @@ -0,0 +1,96 @@ +package json + +import ( + "github.com/gin-gonic/gin" + "github.com/go-go-golems/glazed/pkg/cmds" + json2 "github.com/go-go-golems/glazed/pkg/formatters/json" + "github.com/go-go-golems/glazed/pkg/middlewares/row" + "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/glazed/handlers" + "github.com/go-go-golems/parka/pkg/glazed/parser" + "io" +) + +type QueryHandler struct { + cmd cmds.GlazeCommand + contextMiddlewares []glazed.ContextMiddleware + parserOptions []parser.ParserOption +} + +type QueryHandlerOption func(*QueryHandler) + +func NewQueryHandler(cmd cmds.GlazeCommand, options ...QueryHandlerOption) *QueryHandler { + h := &QueryHandler{ + cmd: cmd, + } + + for _, option := range options { + option(h) + } + + return h +} + +func WithQueryHandlerContextMiddlewares(middlewares ...glazed.ContextMiddleware) QueryHandlerOption { + return func(h *QueryHandler) { + h.contextMiddlewares = middlewares + } +} + +// WithQueryHandlerParserOptions sets the parser options for the QueryHandler +func WithQueryHandlerParserOptions(options ...parser.ParserOption) QueryHandlerOption { + return func(h *QueryHandler) { + h.parserOptions = options + } +} + +func (h *QueryHandler) Handle(c *gin.Context, writer io.Writer) error { + pc := glazed.NewCommandContext(h.cmd) + + h.contextMiddlewares = append( + h.contextMiddlewares, + glazed.NewContextParserMiddleware( + h.cmd, + glazed.NewCommandQueryParser(h.cmd, h.parserOptions...), + ), + ) + + for _, h := range h.contextMiddlewares { + err := h.Handle(c, pc) + if err != nil { + return err + } + } + + gp, err := handlers.CreateTableProcessor(pc, "json", "") + if err != nil { + return err + } + + // remove table middlewares because we are a streaming handler + gp.ReplaceTableMiddleware() + gp.AddRowMiddleware(row.NewOutputMiddleware(json2.NewOutputFormatter(), writer)) + + _, err = writer.Write([]byte("[\n")) + if err != nil { + return err + } + + ctx := c.Request.Context() + err = h.cmd.Run(ctx, pc.ParsedLayers, pc.ParsedParameters, gp) + if err != nil { + return err + } + + err = gp.Close(ctx) + if err != nil { + return err + } + + _, err = writer.Write([]byte("\n]")) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/handlers/command-dir/command-dir.go b/pkg/handlers/command-dir/command-dir.go index 76d5597..ac335c6 100644 --- a/pkg/handlers/command-dir/command-dir.go +++ b/pkg/handlers/command-dir/command-dir.go @@ -6,11 +6,10 @@ import ( "github.com/go-go-golems/clay/pkg/repositories" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" - "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/glazed/parser" "github.com/go-go-golems/parka/pkg/handlers/config" "github.com/go-go-golems/parka/pkg/render" - "github.com/go-go-golems/parka/pkg/render/datatables" + "github.com/go-go-golems/parka/pkg/render/formatters/datatables" "github.com/go-go-golems/parka/pkg/render/layout" parka "github.com/go-go-golems/parka/pkg/server" "github.com/pkg/errors" @@ -413,7 +412,7 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { path = strings.TrimSuffix(path, "/") server.Router.GET(path+"/data/*path", func(c *gin.Context) { - commandPath := c.Param("CommandPath") + commandPath := c.Param("path") commandPath = strings.TrimPrefix(commandPath, "/") sqlCommand, ok := getRepositoryCommand(c, cd.Repository, commandPath) if !ok { @@ -421,9 +420,8 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { return } - handle := server.HandleSimpleQueryCommand(sqlCommand, - glazed.WithCreateProcessor(glazed.CreateJSONProcessor), - glazed.WithParserOptions(cd.computeParserOptions()...), + handle := server.HandleJSONQueryHandler(sqlCommand, + cd.computeParserOptions()..., ) handle(c) @@ -489,19 +487,21 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { AdditionalData: cd.AdditionalData, } - dataTablesProcessorFunc := datatables.NewDataTablesCreateOutputProcessorFunc( + ofFactory := datatables.NewOutputFormatterFactory( cd.TemplateLookup, cd.TemplateName, dt, ) - handle := server.HandleSimpleQueryCommand( - sqlCommand, - glazed.WithCreateProcessor(dataTablesProcessorFunc), - glazed.WithParserOptions(cd.computeParserOptions()...), - ) + _ = ofFactory - handle(c) + //handle := server.HandleJSONQueryHandler( + // sqlCommand, + // glazed.WithOutputFormatterFactory(ofFactory), + // glazed.WithParserOptions(cd.computeParserOptions()...), + //) + // + //handle(c) }) server.Router.GET(path+"/download/*path", func(c *gin.Context) { @@ -565,18 +565,18 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { } parserOptions := cd.computeParserOptions() - // override parameter layers at the end parserOptions = append(parserOptions, parser.WithAppendOverrides("glazed", glazedOverrides)) - handle := server.HandleSimpleQueryOutputFileCommand( - sqlCommand, - tmpFile.Name(), - fileName, - glazed.WithParserOptions(parserOptions...), - ) - - handle(c) + _ = sqlCommand + //handle := server.HandleSimpleQueryOutputFileCommand( + // sqlCommand, + // tmpFile.Name(), + // fileName, + // glazed.WithParserOptions(parserOptions...), + //) + // + //handle(c) }) return nil diff --git a/pkg/render/datatables/datatables.go b/pkg/render/formatters/datatables/datatables.go similarity index 72% rename from pkg/render/datatables/datatables.go rename to pkg/render/formatters/datatables/datatables.go index d2a6950..a4213af 100644 --- a/pkg/render/datatables/datatables.go +++ b/pkg/render/formatters/datatables/datatables.go @@ -1,6 +1,7 @@ package datatables import ( + "context" "embed" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" @@ -13,6 +14,7 @@ import ( "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/render" "github.com/go-go-golems/parka/pkg/render/layout" + "golang.org/x/sync/errgroup" "html/template" "io" ) @@ -23,7 +25,7 @@ import ( type DataTables struct { Command *cmds.CommandDescription // LongDescription is the HTML of the rendered markdown of the long description of the command. - LongDescription string + LongDescription template.HTML Layout *layout.Layout Links []layout.Link @@ -37,8 +39,8 @@ type DataTables struct { // // TODO(manuel, 2023-06-04): Maybe we could make this be an iterator of rows that provide access to the individual // columns for more interesting HTML shenanigans too. - JSStream <-chan template.JS - HTMLStream <-chan template.HTML + JSStream chan template.JS + HTMLStream chan template.HTML // Configuring the template to load the table data through javascript, and provide the data itself // as a JSON array inlined in the HTML of the page. JSRendering bool @@ -119,6 +121,87 @@ func NewOutputFormatter( } } +func (of *OutputFormatter) Close(ctx context.Context) error { + close(of.rowC) + close(of.columnsC) + return nil +} + +func (dt *OutputFormatter) RegisterMiddlewares(p *middlewares.TableProcessor, writer io.Writer) error { + var of formatters.RowOutputFormatter + if dt.dt.JSRendering { + of = json.NewOutputFormatter() + dt.dt.JSStream = make(chan template.JS, 100) + } else { + of = table_formatter.NewOutputFormatter("html") + dt.dt.HTMLStream = make(chan template.HTML, 100) + } + + p.AddRowMiddleware(row.NewColumnsChannelMiddleware(dt.columnsC, true)) + p.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) + + return nil +} + +func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { + // Here, we use the parsed layer to configure the glazed middlewares. + // We then use the created formatters.TableOutputFormatter as a basis for + // our own output formatter that renders into an HTML template. + var err error + + layout_, err := layout.ComputeLayout(pc) + if err != nil { + return err + } + + description := pc.Cmd.Description() + + longHTML, err := render.RenderMarkdownToHTML(description.Long) + if err != nil { + return err + } + + dt_ := dt.dt.Clone() + dt_.Layout = layout_ + dt_.LongDescription = template.HTML(longHTML) + dt_.Command = description + + // Wait for the column names to be sent to the channel. This will only + // take the first row into account. + columns := <-dt.columnsC + dt_.Columns = columns + + // start copying from rowC to HTML or JS stream + + eg, ctx2 := errgroup.WithContext(c) + + eg.Go(func() error { + err := dt.t.Execute(w, dt_) + if err != nil { + return err + } + + return nil + }) + + eg.Go(func() error { + for { + select { + case <-ctx2.Done(): + return ctx2.Err() + case row_ := <-dt.rowC: + if dt.dt.JSRendering { + dt.dt.JSStream <- template.JS(row_) + } else { + dt.dt.HTMLStream <- template.HTML(row_) + } + } + } + }) + + return eg.Wait() +} + type OutputFormatterFactory struct { TemplateName string Lookup render.TemplateLookup @@ -149,34 +232,21 @@ func (dtoff *OutputFormatterFactory) CreateOutputFormatter( return nil, err } - dt_.LongDescription = longHTML + dt_.LongDescription = template.HTML(longHTML) + dt_.Layout = layout_ return NewOutputFormatter(t, dtoff.DataTables), nil } -func (dt *OutputFormatter) RegisterMiddlewares( - m *middlewares.TableProcessor, -) error { - var of formatters.RowOutputFormatter - if dt.dt.JSRendering { - of = json.NewOutputFormatter() - dt.dt.JSStream = make(chan template.JS, 100) - } else { - of = table_formatter.NewOutputFormatter("html") - dt.dt.HTMLStream = make(chan template.HTML, 100) +func NewOutputFormatterFactory( + lookup render.TemplateLookup, + templateName string, + dataTables *DataTables, +) *OutputFormatterFactory { + return &OutputFormatterFactory{ + TemplateName: templateName, + Lookup: lookup, + DataTables: dataTables, } - - // TODO add a "get columns" middleware that can be used to get the columns from the table - // over a single channel that we can wait on in the render call - - m.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) - - return nil -} - -func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { - // clear all table middlewares because we are streaming using a row output middleware - - return nil } diff --git a/pkg/render/datatables/templates/data-tables.tmpl.html b/pkg/render/formatters/datatables/templates/data-tables.tmpl.html similarity index 100% rename from pkg/render/datatables/templates/data-tables.tmpl.html rename to pkg/render/formatters/datatables/templates/data-tables.tmpl.html diff --git a/pkg/render/html.go b/pkg/render/formatters/html.go similarity index 94% rename from pkg/render/html.go rename to pkg/render/formatters/html.go index 666675f..ea43620 100644 --- a/pkg/render/html.go +++ b/pkg/render/formatters/html.go @@ -1,8 +1,9 @@ -package render +package formatters import ( "github.com/gin-gonic/gin" "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/render" "github.com/go-go-golems/parka/pkg/render/layout" "html/template" "io" @@ -32,7 +33,7 @@ import ( // a template to be added in the back in the front. type HTMLTemplateOutputFormatter struct { TemplateName string - Lookup TemplateLookup + Lookup render.TemplateLookup Data map[string]interface{} } @@ -50,7 +51,7 @@ func WithHTMLTemplateOutputFormatterData(data map[string]interface{}) HTMLTempla } func NewHTMLTemplateOutputFormatter( - lookup TemplateLookup, + lookup render.TemplateLookup, templateName string, options ...HTMLTemplateOutputFormatterOption, ) *HTMLTemplateOutputFormatter { @@ -79,7 +80,7 @@ func (H *HTMLTemplateOutputFormatter) Output(c *gin.Context, pc *glazed.CommandC description := pc.Cmd.Description() - longHTML, err := RenderMarkdownToHTML(description.Long) + longHTML, err := render.RenderMarkdownToHTML(description.Long) if err != nil { return err } diff --git a/pkg/server/server.go b/pkg/server/server.go index a396b23..76cb984 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,9 +7,11 @@ import ( "github.com/gin-gonic/contrib/gzip" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" - "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/glazed/handlers/json" + "github.com/go-go-golems/parka/pkg/glazed/parser" "github.com/go-go-golems/parka/pkg/render" utils_fs "github.com/go-go-golems/parka/pkg/utils/fs" + "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" "io/fs" "net/http" @@ -218,43 +220,20 @@ func (s *Server) Run(ctx context.Context) error { return eg.Wait() } -func (s *Server) HandleSimpleQueryCommand( +func (s *Server) HandleJSONQueryHandler( cmd cmds.GlazeCommand, - options ...glazed.HandleOption, + parserOptions ...parser.ParserOption, ) gin.HandlerFunc { - opts := glazed.NewHandleOptions(options) - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, - glazed.NewCommandQueryParser(cmd, opts.ParserOptions...)), + handler := json.NewQueryHandler(cmd, + json.WithQueryHandlerParserOptions(parserOptions...), ) - return glazed.GinHandleGlazedCommand(cmd, opts) -} - -func (s *Server) HandleSimpleQueryOutputFileCommand( - cmd cmds.GlazeCommand, - outputFile string, - fileName string, - options ...glazed.HandleOption, -) gin.HandlerFunc { - opts := glazed.NewHandleOptions(options) - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, glazed.NewCommandQueryParser(cmd, opts.ParserOptions...)), - ) - return glazed.GinHandleGlazedCommandWithOutputFile(cmd, outputFile, fileName, opts) -} - -// TODO(manuel, 2023-02-28) We want to provide a handler to catch errors while parsing parameters - -func (s *Server) HandleSimpleFormCommand( - cmd cmds.GlazeCommand, - options ...glazed.HandleOption, -) gin.HandlerFunc { - opts := &glazed.HandleOptions{} - for _, option := range options { - option(opts) + return func(c *gin.Context) { + err := handler.Handle(c, c.Writer) + if err != nil { + log.Error().Err(err).Msg("failed to handle query") + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + }) + } } - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, glazed.NewCommandFormParser(cmd, opts.ParserOptions...)), - ) - return glazed.GinHandleGlazedCommand(cmd, opts) } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 107d696..6775d67 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -47,7 +47,7 @@ func TestRunGlazedCommand(t *testing.T) { s, err := server.NewServer() require.NoError(t, err) - handler := s.HandleSimpleQueryCommand(tc) + handler := s.HandleJSONQueryHandler(tc) gin.SetMode(gin.TestMode) From 303ba8f0f0c1e53c2da74c22b05711bd7a19752f Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Sun, 2 Jul 2023 16:06:06 -0400 Subject: [PATCH 03/11] :sparkles: Add support for streaming template handler --- .../handlers}/datatables/datatables.go | 250 +++++++++++------- .../templates/data-tables.tmpl.html | 2 +- pkg/handlers/command-dir/command-dir.go | 39 ++- 3 files changed, 174 insertions(+), 117 deletions(-) rename pkg/{render/formatters => glazed/handlers}/datatables/datatables.go (62%) rename pkg/{render/formatters => glazed/handlers}/datatables/templates/data-tables.tmpl.html (99%) diff --git a/pkg/render/formatters/datatables/datatables.go b/pkg/glazed/handlers/datatables/datatables.go similarity index 62% rename from pkg/render/formatters/datatables/datatables.go rename to pkg/glazed/handlers/datatables/datatables.go index a4213af..0af54c9 100644 --- a/pkg/render/formatters/datatables/datatables.go +++ b/pkg/glazed/handlers/datatables/datatables.go @@ -1,17 +1,17 @@ package datatables import ( - "context" "embed" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/formatters" "github.com/go-go-golems/glazed/pkg/formatters/json" table_formatter "github.com/go-go-golems/glazed/pkg/formatters/table" - "github.com/go-go-golems/glazed/pkg/middlewares" "github.com/go-go-golems/glazed/pkg/middlewares/row" "github.com/go-go-golems/glazed/pkg/types" "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/glazed/handlers" + "github.com/go-go-golems/parka/pkg/glazed/parser" "github.com/go-go-golems/parka/pkg/render" "github.com/go-go-golems/parka/pkg/render/layout" "golang.org/x/sync/errgroup" @@ -88,95 +88,148 @@ func (dt *DataTables) Clone() *DataTables { } } -type OutputFormatter struct { - t *template.Template +type QueryHandler struct { + cmd cmds.GlazeCommand + contextMiddlewares []glazed.ContextMiddleware + parserOptions []parser.ParserOption + + templateName string + lookup render.TemplateLookup + dt *DataTables - // rowC is the channel where the rows are sent to. They will need to get converted - // to template.JS or template.HTML before being sent to either - rowC chan string - // columnsC is the channel where the column names are sent to. Since the row.ColumnsChannelMiddleware - // instance that sends columns to this channel is running before the row firmware, we should be careful - // about not blocking. Potentially, this could be done by starting a goroutine in the middleware, - // since we have a context there, and there is no need to block the middleware processing. - columnsC chan []types.FieldName } -func NewOutputFormatter( - t *template.Template, - dt *DataTables) *OutputFormatter { +type QueryHandlerOption func(qh *QueryHandler) - // make the NewOutputChannelMiddleware generic to send string/template.JS/template.HTML over the wire - rowC := make(chan string, 100) - - // make a channel to receive column names - columnsC := make(chan []types.FieldName, 10) +func NewQueryHandler( + cmd cmds.GlazeCommand, + lookup render.TemplateLookup, + templateName string, + options ...QueryHandlerOption, +) *QueryHandler { + qh := &QueryHandler{ + cmd: cmd, + dt: &DataTables{}, + lookup: lookup, + templateName: templateName, + } - // we need to make sure that we are closing the channel correctly. Should middlewares have a Close method? - // that actually sounds reasonable - return &OutputFormatter{ - t: t, - dt: dt, - rowC: rowC, - columnsC: columnsC, + for _, option := range options { + option(qh) } -} -func (of *OutputFormatter) Close(ctx context.Context) error { - close(of.rowC) - close(of.columnsC) - return nil + return qh } -func (dt *OutputFormatter) RegisterMiddlewares(p *middlewares.TableProcessor, writer io.Writer) error { - var of formatters.RowOutputFormatter - if dt.dt.JSRendering { - of = json.NewOutputFormatter() - dt.dt.JSStream = make(chan template.JS, 100) - } else { - of = table_formatter.NewOutputFormatter("html") - dt.dt.HTMLStream = make(chan template.HTML, 100) +func WithDataTables(dt *DataTables) QueryHandlerOption { + return func(qh *QueryHandler) { + qh.dt = dt } +} - p.AddRowMiddleware(row.NewColumnsChannelMiddleware(dt.columnsC, true)) - p.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) +func WithQueryHandlerContextMiddlewares(middlewares ...glazed.ContextMiddleware) QueryHandlerOption { + return func(h *QueryHandler) { + h.contextMiddlewares = middlewares + } +} - return nil +// WithQueryHandlerParserOptions sets the parser options for the QueryHandler +func WithQueryHandlerParserOptions(options ...parser.ParserOption) QueryHandlerOption { + return func(h *QueryHandler) { + h.parserOptions = options + } } -func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { - // Here, we use the parsed layer to configure the glazed middlewares. - // We then use the created formatters.TableOutputFormatter as a basis for - // our own output formatter that renders into an HTML template. - var err error +func (qh *QueryHandler) Handle(c *gin.Context, w io.Writer) error { + pc := glazed.NewCommandContext(qh.cmd) - layout_, err := layout.ComputeLayout(pc) - if err != nil { - return err + qh.contextMiddlewares = append( + qh.contextMiddlewares, + glazed.NewContextParserMiddleware( + qh.cmd, + glazed.NewCommandQueryParser(qh.cmd, qh.parserOptions...), + ), + ) + + for _, h := range qh.contextMiddlewares { + err := h.Handle(c, pc) + if err != nil { + return err + } } - description := pc.Cmd.Description() + // rowC is the channel where the rows are sent to. They will need to get converted + // to template.JS or template.HTML before being sent to either + rowC := make(chan string, 100) - longHTML, err := render.RenderMarkdownToHTML(description.Long) + // columnsC is the channel where the column names are sent to. Since the row.ColumnsChannelMiddleware + // instance that sends columns to this channel is running before the row firmware, we should be careful + // about not blocking. Potentially, this could be done by starting a goroutine in the middleware, + // since we have a context there, and there is no need to block the middleware processing. + columnsC := make(chan []types.FieldName, 10) + + dt_ := qh.dt.Clone() + var of formatters.RowOutputFormatter + if dt_.JSRendering { + of = json.NewOutputFormatter() + dt_.JSStream = make(chan template.JS, 100) + } else { + of = table_formatter.NewOutputFormatter("html") + dt_.HTMLStream = make(chan template.HTML, 100) + } + + gp, err := handlers.CreateTableProcessor(pc, "table", "") if err != nil { return err } - dt_ := dt.dt.Clone() - dt_.Layout = layout_ - dt_.LongDescription = template.HTML(longHTML) - dt_.Command = description + gp.ReplaceTableMiddleware() + gp.AddRowMiddleware(row.NewColumnsChannelMiddleware(columnsC, true)) + gp.AddRowMiddleware(row.NewOutputChannelMiddleware(of, rowC)) - // Wait for the column names to be sent to the channel. This will only - // take the first row into account. - columns := <-dt.columnsC - dt_.Columns = columns + eg, ctx2 := errgroup.WithContext(c.Request.Context()) - // start copying from rowC to HTML or JS stream + // copy the json rows to the template stream + eg.Go(func() error { + defer func() { + if dt_.JSRendering { + close(dt_.JSStream) + } else { + close(dt_.HTMLStream) + } + }() + for { + select { + case <-ctx2.Done(): + return ctx2.Err() + case row_, ok := <-rowC: + // check if channel is closed + if !ok { + return nil + } - eg, ctx2 := errgroup.WithContext(c) + if dt_.JSRendering { + dt_.JSStream <- template.JS(row_) + } else { + dt_.HTMLStream <- template.HTML(row_) + } + } + } + }) + // actually run the command eg.Go(func() error { - err := dt.t.Execute(w, dt_) + defer func() { + defer close(rowC) + defer close(columnsC) + }() + + err = qh.cmd.Run(ctx2, pc.ParsedLayers, pc.ParsedParameters, gp) + if err != nil { + return err + } + + err = gp.Close(ctx2) if err != nil { return err } @@ -185,68 +238,61 @@ func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w i }) eg.Go(func() error { - for { - select { - case <-ctx2.Done(): - return ctx2.Err() - case row_ := <-dt.rowC: - if dt.dt.JSRendering { - dt.dt.JSStream <- template.JS(row_) - } else { - dt.dt.HTMLStream <- template.HTML(row_) - } - } + err := qh.renderTemplate(c, pc, w, dt_, columnsC) + if err != nil { + return err } + + return nil }) return eg.Wait() } -type OutputFormatterFactory struct { - TemplateName string - Lookup render.TemplateLookup - DataTables *DataTables -} - -func (dtoff *OutputFormatterFactory) CreateOutputFormatter( +func (qh *QueryHandler) renderTemplate( c *gin.Context, pc *glazed.CommandContext, -) (*OutputFormatter, error) { - // Lookup template on every request, not up front. That way, templates can be reloaded without recreating the gin - // server. - t, err := dtoff.Lookup.Lookup(dtoff.TemplateName) + w io.Writer, + dt_ *DataTables, + columnsC chan []types.FieldName, +) error { + // Here, we use the parsed layer to configure the glazed middlewares. + // We then use the created formatters.TableOutputFormatter as a basis for + // our own output formatter that renders into an HTML template. + var err error + + t, err := qh.lookup.Lookup(qh.templateName) if err != nil { - return nil, err + return err } layout_, err := layout.ComputeLayout(pc) if err != nil { - return nil, err + return err } description := pc.Cmd.Description() - dt_ := dtoff.DataTables.Clone() longHTML, err := render.RenderMarkdownToHTML(description.Long) if err != nil { - return nil, err + return err } + dt_.Layout = layout_ dt_.LongDescription = template.HTML(longHTML) + dt_.Command = description - dt_.Layout = layout_ + // Wait for the column names to be sent to the channel. This will only + // take the first row into account. + columns := <-columnsC + dt_.Columns = columns - return NewOutputFormatter(t, dtoff.DataTables), nil -} + // start copying from rowC to HTML or JS stream -func NewOutputFormatterFactory( - lookup render.TemplateLookup, - templateName string, - dataTables *DataTables, -) *OutputFormatterFactory { - return &OutputFormatterFactory{ - TemplateName: templateName, - Lookup: lookup, - DataTables: dataTables, + err = t.Execute(w, dt_) + if err != nil { + return err } + + return nil } diff --git a/pkg/render/formatters/datatables/templates/data-tables.tmpl.html b/pkg/glazed/handlers/datatables/templates/data-tables.tmpl.html similarity index 99% rename from pkg/render/formatters/datatables/templates/data-tables.tmpl.html rename to pkg/glazed/handlers/datatables/templates/data-tables.tmpl.html index be92393..c38f0df 100644 --- a/pkg/render/formatters/datatables/templates/data-tables.tmpl.html +++ b/pkg/glazed/handlers/datatables/templates/data-tables.tmpl.html @@ -133,7 +133,7 @@

{{.Command.Name}}

+ + + {{ $hasError := false }} + {{ range .ErrorStream }} + {{ $hasError = true }} +
+
+
+
+ Error: {{.}} +
+
+
+
+ {{ end }} + {{ if $hasError }} + + {{ end }} + diff --git a/pkg/glazed/handlers/glazed/glazed.go b/pkg/glazed/handlers/glazed/glazed.go index 396afbf..115b3fc 100644 --- a/pkg/glazed/handlers/glazed/glazed.go +++ b/pkg/glazed/handlers/glazed/glazed.go @@ -3,7 +3,6 @@ package glazed import ( "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" - "github.com/go-go-golems/glazed/pkg/middlewares/table" "github.com/go-go-golems/glazed/pkg/settings" "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/glazed/parser" @@ -75,15 +74,11 @@ func (h *QueryHandler) Handle(c *gin.Context, writer io.Writer) error { return err } - of, err := settings.SetupTableOutputFormatter(ps) + of, err := settings.SetupProcessorOutput(gp, ps, writer) if err != nil { return err } - // TODO(manuel, 2023-07-02) It would be good to use streaming here for the formats that support it - // See: https://github.com/go-go-golems/parka/issues/68 - gp.AddTableMiddleware(table.NewOutputMiddleware(of, writer)) - c.Header("Content-Type", of.ContentType()) ctx := c.Request.Context() diff --git a/pkg/glazed/handlers/json/json.go b/pkg/glazed/handlers/json/json.go index a028134..05f0b7e 100644 --- a/pkg/glazed/handlers/json/json.go +++ b/pkg/glazed/handlers/json/json.go @@ -99,7 +99,7 @@ func (h *QueryHandler) Handle(c *gin.Context, writer io.Writer) error { return nil } -func HandleJSONQueryHandler( +func CreateJSONQueryHandler( cmd cmds.GlazeCommand, parserOptions ...parser.ParserOption, ) gin.HandlerFunc { diff --git a/pkg/glazed/handlers/output-file/output-file.go b/pkg/glazed/handlers/output-file/output-file.go index b81541a..4647415 100644 --- a/pkg/glazed/handlers/output-file/output-file.go +++ b/pkg/glazed/handlers/output-file/output-file.go @@ -57,7 +57,7 @@ func (h *OutputFileHandler) Handle(c *gin.Context, w io.Writer) error { return nil } -func HandleGlazedOutputFileHandler( +func CreateGlazedFileHandler( cmd cmds.GlazeCommand, fileName string, parserOptions ...parser.ParserOption, diff --git a/pkg/handlers/command-dir/command-dir.go b/pkg/handlers/command-dir/command-dir.go index 1cabdf9..4c84c46 100644 --- a/pkg/handlers/command-dir/command-dir.go +++ b/pkg/handlers/command-dir/command-dir.go @@ -413,20 +413,20 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { server.Router.GET(path+"/data/*path", func(c *gin.Context) { commandPath := c.Param("path") - commandPath = strings.TrimPrefix(commandPath, path) + commandPath = strings.TrimPrefix(commandPath, path+"/") sqlCommand, ok := getRepositoryCommand(c, cd.Repository, commandPath) if !ok { c.JSON(404, gin.H{"error": "command not found"}) return } - json.HandleJSONQueryHandler(sqlCommand) + json.CreateJSONQueryHandler(sqlCommand)(c) }) server.Router.GET(path+"/sqleton/*path", func(c *gin.Context) { commandPath := c.Param("path") - commandPath = strings.TrimPrefix(commandPath, path) + commandPath = strings.TrimPrefix(commandPath, path+"/") // Get repository command sqlCommand, ok := getRepositoryCommand(c, cd.Repository, commandPath) @@ -442,23 +442,24 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { datatables.WithAdditionalData(cd.AdditionalData), } - datatables.HandleDataTables(sqlCommand, path, commandPath, options...) + datatables.CreateDataTablesHandler(sqlCommand, path, commandPath, options...)(c) }) server.Router.GET(path+"/download/*path", func(c *gin.Context) { - // get file name at end of path - index := strings.LastIndex(path, "/") + path_ := c.Param("path") + path_ = strings.TrimPrefix(path_, path+"/") + index := strings.LastIndex(path_, "/") if index == -1 { c.JSON(500, gin.H{"error": "could not find file name"}) return } - if index >= len(path)-1 { + if index >= len(path_)-1 { c.JSON(500, gin.H{"error": "could not find file name"}) return } - fileName := path[index+1:] + fileName := path_[index+1:] - commandPath := strings.TrimPrefix(path[:index], "/") + commandPath := strings.TrimPrefix(path_[:index], path+"/") sqlCommand, ok := getRepositoryCommand(c, cd.Repository, commandPath) if !ok { // JSON output and error code already handled by getRepositoryCommand @@ -466,11 +467,13 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { } parserOptions := cd.computeParserOptions() - output_file.HandleGlazedOutputFileHandler( + // TODO(manuel, 223-07-03) this is really only necessary for excel, I think. + // Other formats can render straight to the stream + output_file.CreateGlazedFileHandler( sqlCommand, fileName, parserOptions..., - ) + )(c) }) return nil diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 5243480..34b3d38 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -48,7 +48,7 @@ func TestRunGlazedCommand(t *testing.T) { s, err := server.NewServer() require.NoError(t, err) - handler := json2.HandleJSONQueryHandler(tc) + handler := json2.CreateJSONQueryHandler(tc) gin.SetMode(gin.TestMode) From cb19553529d91d7336b085838e20981d400d0469 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Mon, 3 Jul 2023 16:12:47 -0400 Subject: [PATCH 10/11] :sparkles: Add streaming row file download output --- .../handlers/output-file/output-file.go | 66 ++++++++++++------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/pkg/glazed/handlers/output-file/output-file.go b/pkg/glazed/handlers/output-file/output-file.go index 4647415..cdd924e 100644 --- a/pkg/glazed/handlers/output-file/output-file.go +++ b/pkg/glazed/handlers/output-file/output-file.go @@ -45,10 +45,6 @@ func (h *OutputFileHandler) Handle(c *gin.Context, w io.Writer) error { _ = f.Close() }(f) - baseName := filepath.Base(h.outputFileName) - - c.Writer.Header().Set("Content-Disposition", "attachment; filename="+baseName) - _, err = io.Copy(c.Writer, f) if err != nil { return err @@ -63,20 +59,10 @@ func CreateGlazedFileHandler( parserOptions ...parser.ParserOption, ) gin.HandlerFunc { return func(c *gin.Context) { - // create a temporary file for glazed output - tmpFile, err := os.CreateTemp("/tmp", fmt.Sprintf("glazed-output-*.%s", fileName)) - if err != nil { - c.JSON(500, gin.H{"error": "could not create temporary file"}) - return - } - defer func(name string) { - _ = os.Remove(name) - }(tmpFile.Name()) + glazedOverrides := map[string]interface{}{} + needsRealFileOutput := false - // now check file suffix for content-type - glazedOverrides := map[string]interface{}{ - "output-file": tmpFile.Name(), - } + // create a temporary file for glazed output if strings.HasSuffix(fileName, ".csv") { glazedOverrides["output"] = "table" glazedOverrides["table-format"] = "csv" @@ -95,6 +81,7 @@ func CreateGlazedFileHandler( glazedOverrides["output"] = "yaml" } else if strings.HasSuffix(fileName, ".xlsx") { glazedOverrides["output"] = "excel" + needsRealFileOutput = true } else if strings.HasSuffix(fileName, ".txt") { glazedOverrides["output"] = "table" glazedOverrides["table-format"] = "ascii" @@ -103,17 +90,48 @@ func CreateGlazedFileHandler( return } - // override parameter layers at the end - parserOptions = append(parserOptions, parser.WithAppendOverrides("glazed", glazedOverrides)) + var tmpFile *os.File + var err error + + // excel output needs a real output file, otherwise we can go stream to the HTTP response + if needsRealFileOutput { + tmpFile, err = os.CreateTemp("/tmp", fmt.Sprintf("glazed-output-*.%s", fileName)) + if err != nil { + c.JSON(500, gin.H{"error": "could not create temporary file"}) + return + } + defer func(name string) { + _ = os.Remove(name) + }(tmpFile.Name()) + + // now check file suffix for content-type + glazedOverrides["output-file"] = tmpFile.Name() + } + parserOptions = append(parserOptions, parser.WithAppendOverrides("glazed", glazedOverrides)) handler := glazed.NewQueryHandler(cmd, glazed.WithQueryHandlerParserOptions(parserOptions...)) - outputFileHandler := NewOutputFileHandler(handler, tmpFile.Name()) - err = outputFileHandler.Handle(c, c.Writer) - if err != nil { - c.JSON(500, gin.H{"error": err.Error()}) - return + baseName := filepath.Base(fileName) + c.Writer.Header().Set("Content-Disposition", "attachment; filename="+baseName) + + if needsRealFileOutput { + outputFileHandler := NewOutputFileHandler(handler, tmpFile.Name()) + + err = outputFileHandler.Handle(c, c.Writer) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + } else { + + err = handler.Handle(c, c.Writer) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } } + // override parameter layers at the end + } } From f49fbbf7dc0a888410ab425210aaa1a8a4edb4e6 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Mon, 3 Jul 2023 16:27:20 -0400 Subject: [PATCH 11/11] :arrow_up: Bump glazed --- go.mod | 6 +++--- go.sum | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 235d621..dd7a397 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/alecthomas/chroma/v2 v2.2.0 github.com/gin-gonic/contrib v0.0.0-20221130124618-7e01895a63f2 github.com/gin-gonic/gin v1.9.0 - github.com/go-go-golems/clay v0.0.17 - github.com/go-go-golems/glazed v0.3.0 + github.com/go-go-golems/clay v0.0.18 + github.com/go-go-golems/glazed v0.3.2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.29.0 github.com/spf13/cobra v1.6.1 @@ -69,7 +69,6 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect @@ -102,6 +101,7 @@ require ( golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/errgo.v2 v2.1.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5e93c1a..c3561e8 100644 --- a/go.sum +++ b/go.sum @@ -111,10 +111,10 @@ github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH8 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-go-golems/clay v0.0.17 h1:lFOXpAs3AiBMu3LwUO2xvJ7JvkIVU4g2beRODyKGvTM= -github.com/go-go-golems/clay v0.0.17/go.mod h1:2QM24Cz+xBFrnXr5h561DAQ/lxLYlnUO8EjlxiOsYk4= -github.com/go-go-golems/glazed v0.3.0 h1:L4lJAz1WvYasKjvkxd3UTJImoPEcl6gEvV21qUqjjcM= -github.com/go-go-golems/glazed v0.3.0/go.mod h1:zMZnD5ZCj5h/UAQpfpaWV6UkGMdILd8yWXGh9y8sdvU= +github.com/go-go-golems/clay v0.0.18 h1:smhX//d2xuinPWAx+MOlZxMva2bTQDg5Kl+dN1b3YVc= +github.com/go-go-golems/clay v0.0.18/go.mod h1:lwCix3m4IQDIbXnb5F6uVayADuKVHNKPngJdxCTEelU= +github.com/go-go-golems/glazed v0.3.2 h1:VS55mFMXYiZfaEXGQo+gDS7p+n5GF+Vx2egDc0PzXPY= +github.com/go-go-golems/glazed v0.3.2/go.mod h1:nv4FnbQ7kgWQlzyxA3+10jmEc/GDsw3SB0A/sWH14FQ= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= @@ -255,8 +255,6 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3 h1:2YMbJ6WbdQI9K73chxh9OWMDsZ2PNjAIRGTonp3T0l0= -github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3/go.mod h1:LQkXsHRSPIEklPCq8OMQAzYNS2NGtYStdNE/ej1oJU8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -688,6 +686,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=