From 3b42d535873f5d0af523e4890f1d82d0348e5c37 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Sun, 2 Jul 2023 15:10:33 -0400 Subject: [PATCH] :sparkles: First JSON streaming handler! --- pkg/glazed/command.go | 143 ++++++---- pkg/glazed/handler.go | 269 ++---------------- pkg/glazed/handlers/helpers.go | 30 ++ pkg/glazed/handlers/json/json.go | 96 +++++++ pkg/handlers/command-dir/command-dir.go | 44 +-- .../{ => formatters}/datatables/datatables.go | 124 ++++++-- .../templates/data-tables.tmpl.html | 0 pkg/render/{ => formatters}/html.go | 9 +- pkg/server/server.go | 51 +--- pkg/server/server_test.go | 2 +- 10 files changed, 376 insertions(+), 392 deletions(-) create mode 100644 pkg/glazed/handlers/helpers.go create mode 100644 pkg/glazed/handlers/json/json.go rename pkg/render/{ => formatters}/datatables/datatables.go (72%) rename pkg/render/{ => formatters}/datatables/templates/data-tables.tmpl.html (100%) rename pkg/render/{ => formatters}/html.go (94%) diff --git a/pkg/glazed/command.go b/pkg/glazed/command.go index a5a6d9c..fdaa9cf 100644 --- a/pkg/glazed/command.go +++ b/pkg/glazed/command.go @@ -74,45 +74,64 @@ func (pc *CommandContext) GetAllParameterValues() map[string]interface{} { return ret } -// CommandHandlerFunc mirrors gin's HandlerFunc, but also gets passed a CommandContext. -// That allows it to reuse data from the gin.Context, most importantly the request itself. -type CommandHandlerFunc func(*gin.Context, *CommandContext) error - -// HandlePrepopulatedParameters sets the given parameters in the CommandContext's ParsedParameters. -// If any of the given parameters also belong to a layer, they are also set there. -func HandlePrepopulatedParameters(ps map[string]interface{}) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - for k, v := range ps { - pc.ParsedParameters[k] = v - - // Now check if the parameter is in any of the layers, and if so, set it there as well - for _, layer := range pc.ParsedLayers { - if _, ok := layer.Parameters[k]; ok { - layer.Parameters[k] = v - } +// ContextMiddleware is used to build up a CommandContext. It is similar to the HandlerFunc +// in gin, but instead of handling the HTTP response, it is only used to build up the CommandContext. +type ContextMiddleware interface { + Handle(*gin.Context, *CommandContext) error +} + +// PrepopulatedContextMiddleware is a ContextMiddleware that prepopulates the CommandContext with +// the given map of parameters. It overwrites existing parameters in pc.ParsedParameters, +// and overwrites parameters in individual parser layers if they have already been set. +type PrepopulatedContextMiddleware struct { + ps map[string]interface{} +} + +func (p *PrepopulatedContextMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + for k, v := range p.ps { + pc.ParsedParameters[k] = v + + // Now check if the parameter is in any of the layers, and if so, set it there as well + for _, layer := range pc.ParsedLayers { + if _, ok := layer.Parameters[k]; ok { + layer.Parameters[k] = v } } - return nil - } -} - -// HandlePrepopulatedParsedLayers sets the given layers in the CommandContext's Layers, -// overriding the parameters of any layers that are already present. -// This means that if a parameter is not set in layers_ but is set in the Layers, -// the value in the Layers will be kept. -func HandlePrepopulatedParsedLayers(layers_ map[string]*layers.ParsedParameterLayer) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - for k, v := range layers_ { - parsedLayer, ok := pc.ParsedLayers[k] - if ok { - for k2, v2 := range v.Parameters { - parsedLayer.Parameters[k2] = v2 - } - } else { - pc.ParsedLayers[k] = v + } + return nil +} + +func NewPrepopulatedContextMiddleware(ps map[string]interface{}) *PrepopulatedContextMiddleware { + return &PrepopulatedContextMiddleware{ + ps: ps, + } +} + +// PrepopulatedParsedLayersContextMiddleware is a ContextMiddleware that prepopulates the CommandContext with +// the given map of layers. If a layer already exists, it overwrites its parameters. +type PrepopulatedParsedLayersContextMiddleware struct { + layers map[string]*layers.ParsedParameterLayer +} + +func (p *PrepopulatedParsedLayersContextMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + for k, v := range p.layers { + parsedLayer, ok := pc.ParsedLayers[k] + if ok { + for k2, v2 := range v.Parameters { + parsedLayer.Parameters[k2] = v2 } + } else { + pc.ParsedLayers[k] = v } - return nil + } + return nil +} + +func NewPrepopulatedParsedLayersContextMiddleware( + layers map[string]*layers.ParsedParameterLayer, +) *PrepopulatedParsedLayersContextMiddleware { + return &PrepopulatedParsedLayersContextMiddleware{ + layers: layers, } } @@ -164,31 +183,39 @@ func NewCommandFormParser(cmd cmds.GlazeCommand, options ...parser.ParserOption) return ph } -// NewParserCommandHandlerFunc creates a CommandHandlerFunc using the given parser.Parser. -// It also first establishes a set of defaults by loading them from an alias definition. -func NewParserCommandHandlerFunc(cmd cmds.GlazeCommand, parserHandler *parser.Parser) CommandHandlerFunc { - return func(c *gin.Context, pc *CommandContext) error { - parseState := parser.NewParseStateFromCommandDescription(cmd) - err := parserHandler.Parse(c, parseState) - if err != nil { - return err - } +type ContextParserMiddleware struct { + command cmds.GlazeCommand + parser *parser.Parser +} - pc.ParsedParameters = parseState.FlagsAndArguments.Parameters - pc.ParsedLayers = map[string]*layers.ParsedParameterLayer{} - commandLayers := pc.Cmd.Description().Layers - for _, v := range commandLayers { - parsedParameterLayer := &layers.ParsedParameterLayer{ - Layer: v, - Parameters: map[string]interface{}{}, - } - parsedLayers, ok := parseState.Layers[v.GetSlug()] - if ok { - parsedParameterLayer.Parameters = parsedLayers.Parameters - } - pc.ParsedLayers[v.GetSlug()] = parsedParameterLayer +func (cpm *ContextParserMiddleware) Handle(c *gin.Context, pc *CommandContext) error { + parseState := parser.NewParseStateFromCommandDescription(cpm.command) + err := cpm.parser.Parse(c, parseState) + if err != nil { + return err + } + + pc.ParsedParameters = parseState.FlagsAndArguments.Parameters + pc.ParsedLayers = map[string]*layers.ParsedParameterLayer{} + commandLayers := pc.Cmd.Description().Layers + for _, v := range commandLayers { + parsedParameterLayer := &layers.ParsedParameterLayer{ + Layer: v, + Parameters: map[string]interface{}{}, + } + parsedLayers, ok := parseState.Layers[v.GetSlug()] + if ok { + parsedParameterLayer.Parameters = parsedLayers.Parameters } + pc.ParsedLayers[v.GetSlug()] = parsedParameterLayer + } + + return nil +} - return nil +func NewContextParserMiddleware(cmd cmds.GlazeCommand, parser *parser.Parser) *ContextParserMiddleware { + return &ContextParserMiddleware{ + command: cmd, + parser: parser, } } diff --git a/pkg/glazed/handler.go b/pkg/glazed/handler.go index d60b2a3..97bc2f6 100644 --- a/pkg/glazed/handler.go +++ b/pkg/glazed/handler.go @@ -3,270 +3,51 @@ package glazed import ( "bytes" "github.com/gin-gonic/gin" - "github.com/go-go-golems/glazed/pkg/cmds" - "github.com/go-go-golems/glazed/pkg/formatters" - "github.com/go-go-golems/glazed/pkg/formatters/json" - "github.com/go-go-golems/glazed/pkg/middlewares" - "github.com/go-go-golems/glazed/pkg/middlewares/table" - "github.com/go-go-golems/glazed/pkg/settings" - "github.com/go-go-golems/parka/pkg/glazed/parser" - "golang.org/x/sync/errgroup" "io" - "net/http" "os" + "path/filepath" ) -type GinOutputFormatter interface { - Output(w io.Writer) error - RegisterMiddlewares(p *middlewares.TableProcessor) error +type Handler interface { + Handle(c *gin.Context, w io.Writer) error } -type GinOutputFormatterFactory interface { - CreateOutputFormatter(c *gin.Context, pc *CommandContext) (GinOutputFormatter, error) +type OutputFileHandler struct { + handler Handler + outputFileName string } -// HandleOptions groups all the settings for a gin handler that handles a glazed command. -type HandleOptions struct { - // ParserOptions are passed to the given parser (the thing that gathers the glazed.Command - // flags and arguments. - ParserOptions []parser.ParserOption - - // Handlers are run right at the start of the gin.Handler to build up the CommandContext based on the - // gin.Context. They can be chained because they get passed the previous CommandContext. - // - // NOTE(manuel, 2023-06-22) We currently use a single CommandHandler, which is created with NewParserCommandHandlerFunc. - // This creates a command handler that uses a parser.Parser to parse the gin.Context and return a CommandContext. - // For example, the FormParser will parse command parameters passed as a HTML form. - // - // While we currently only use a single handler, the current setup allows us to chain a middleware of handlers. - // This would potentially allow us to catch parse errors and return an appropriate error template - // I'm not entirely sure if this all makes sense. - Handlers []CommandHandlerFunc - - // CreateProcessor takes a gin.Context and a CommandContext and returns a processor.TableProcessor (and a content-type) - OutputFormatterFactory GinOutputFormatterFactory - - // This is the actual gin output writer - Writer io.Writer -} - -type HandleOption func(*HandleOptions) - -func (h *HandleOptions) Copy(options ...HandleOption) *HandleOptions { - ret := &HandleOptions{ - ParserOptions: h.ParserOptions, - Handlers: h.Handlers, - OutputFormatterFactory: h.OutputFormatterFactory, - Writer: h.Writer, - } - - for _, option := range options { - option(ret) - } - - return ret -} - -func NewHandleOptions(options []HandleOption) *HandleOptions { - opts := &HandleOptions{} - for _, option := range options { - option(opts) - } - return opts -} - -func WithParserOptions(parserOptions ...parser.ParserOption) HandleOption { - return func(o *HandleOptions) { - o.ParserOptions = parserOptions +func NewOutputFileHandler(handler Handler, outputFileName string) *OutputFileHandler { + h := &OutputFileHandler{ + handler: handler, + outputFileName: outputFileName, } -} - -func WithHandlers(handlers ...CommandHandlerFunc) HandleOption { - return func(o *HandleOptions) { - o.Handlers = handlers - } -} -func WithWriter(w io.Writer) HandleOption { - return func(o *HandleOptions) { - o.Writer = w - } + return h } -func CreateJSONProcessor(_ *gin.Context, pc *CommandContext) ( - *middlewares.TableProcessor, - error, -) { - l, ok := pc.ParsedLayers["glazed"] - l.Parameters["output"] = "json" - - var gp *middlewares.TableProcessor - var err error - - if ok { - gp, err = settings.SetupTableProcessor(l.Parameters) - } else { - gp, err = settings.SetupTableProcessor(map[string]interface{}{ - "output": "json", - }) - } - +func (h *OutputFileHandler) Handle(c *gin.Context, w io.Writer) error { + buf := &bytes.Buffer{} + err := h.handler.Handle(c, buf) if err != nil { - return nil, err - } - - return gp, nil -} - -// GinHandleGlazedCommand returns a gin.HandlerFunc that runs a glazed.Command and writes -// the results to the gin.Context ResponseWriter. -func GinHandleGlazedCommand( - cmd cmds.GlazeCommand, - opts *HandleOptions, -) gin.HandlerFunc { - return func(c *gin.Context) { - err := runGlazeCommand(c, cmd, opts) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": err.Error(), - }) - return - } - - c.Status(200) - } -} - -// GinHandleGlazedCommandWithOutputFile returns a gin.HandlerFunc that is responsible for -// running the GlazeCommand, and then returning the output file as an attachment. -// This usually requires the caller to provide a temporary file path. -// -// TODO(manuel, 2023-06-22) Now that TableOutputFormatter renders directly into a io.Writer, -// I don't think we need all this anymore, we just need to set the relevant header. -func GinHandleGlazedCommandWithOutputFile( - cmd cmds.GlazeCommand, - outputFile string, - fileName string, - opts *HandleOptions, -) gin.HandlerFunc { - return func(c *gin.Context) { - buf := &bytes.Buffer{} - opts_ := opts.Copy(WithWriter(buf)) - err := runGlazeCommand(c, cmd, opts_) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": err.Error(), - }) - return - } - - c.Status(200) - - f, err := os.Open(outputFile) - if err != nil { - _ = c.AbortWithError(http.StatusInternalServerError, err) - return - } - defer func(f *os.File) { - _ = f.Close() - }(f) - - c.Writer.Header().Set("Content-Disposition", "attachment; filename="+fileName) - - _, err = io.Copy(c.Writer, f) - if err != nil { - if err != nil { - _ = c.AbortWithError(http.StatusInternalServerError, err) - return - } - } - } -} - -func runGlazeCommand(c *gin.Context, cmd cmds.GlazeCommand, opts *HandleOptions) error { - pc := NewCommandContext(cmd) - - for _, h := range opts.Handlers { - err := h(c, pc) - if err != nil { - return err - } - } - - var gp *middlewares.TableProcessor - var err error - - glazedLayer := pc.ParsedLayers["glazed"] - - if glazedLayer != nil { - gp, err = settings.SetupTableProcessor(glazedLayer.Parameters) - if err != nil { - return err - } - } else { - gp = middlewares.NewTableProcessor() - } - - var writer io.Writer = c.Writer - if opts.Writer != nil { - writer = opts.Writer - } - - if opts.OutputFormatterFactory != nil { - of, err := opts.OutputFormatterFactory.CreateOutputFormatter(c, pc) - if err != nil { - return err - } - // remove table middlewares to do streaming rows - gp.ReplaceTableMiddleware() - - // create rowOutputChannelMiddleware here? But that's actually a responsibility of the OutputFormatterFactory. - // we need to create these before running the command, and we need to figure out a way to get the Columns. - - err = of.RegisterMiddlewares(gp) - if err != nil { - return err - } - - eg := &errgroup.Group{} - eg.Go(func() error { - return cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) - }) - - eg.Go(func() error { - // we somehow need to pass the channels to the OutputFormatterFactory - return of.Output(writer) - }) - - // no cancellation on error? - - return eg.Wait() - } - - // here we run a normal full table render - var of formatters.TableOutputFormatter - if glazedLayer != nil { - of, err = settings.SetupTableOutputFormatter(glazedLayer.Parameters) - if err != nil { - return err - } - } else { - of = json.NewOutputFormatter( - json.WithOutputIndividualRows(true), - ) - } - if opts.Writer == nil { - c.Writer.Header().Set("Content-Type", of.ContentType()) + return err } - gp.AddTableMiddleware(table.NewOutputMiddleware(of, writer)) + c.Status(200) - err = cmd.Run(c, pc.ParsedLayers, pc.ParsedParameters, gp) + f, err := os.Open(h.outputFileName) if err != nil { return err } + defer func(f *os.File) { + _ = f.Close() + }(f) + + baseName := filepath.Base(h.outputFileName) + + c.Writer.Header().Set("Content-Disposition", "attachment; filename="+baseName) - err = gp.RunTableMiddlewares(c) + _, err = io.Copy(c.Writer, f) if err != nil { return err } diff --git a/pkg/glazed/handlers/helpers.go b/pkg/glazed/handlers/helpers.go new file mode 100644 index 0000000..2ccd921 --- /dev/null +++ b/pkg/glazed/handlers/helpers.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "github.com/go-go-golems/glazed/pkg/middlewares" + "github.com/go-go-golems/glazed/pkg/settings" + "github.com/go-go-golems/parka/pkg/glazed" +) + +func CreateTableProcessor(pc *glazed.CommandContext, outputType string, tableType string) (*middlewares.TableProcessor, error) { + var gp *middlewares.TableProcessor + var err error + + glazedLayer := pc.ParsedLayers["glazed"] + + if glazedLayer != nil { + glazedLayer.Parameters["output"] = outputType + glazedLayer.Parameters["table"] = tableType + gp, err = settings.SetupTableProcessor(glazedLayer.Parameters) + if err != nil { + return nil, err + } + } else { + gp, err = settings.SetupTableProcessor(map[string]interface{}{ + "output": outputType, + "table": tableType, + }) + } + + return gp, err +} diff --git a/pkg/glazed/handlers/json/json.go b/pkg/glazed/handlers/json/json.go new file mode 100644 index 0000000..e237788 --- /dev/null +++ b/pkg/glazed/handlers/json/json.go @@ -0,0 +1,96 @@ +package json + +import ( + "github.com/gin-gonic/gin" + "github.com/go-go-golems/glazed/pkg/cmds" + json2 "github.com/go-go-golems/glazed/pkg/formatters/json" + "github.com/go-go-golems/glazed/pkg/middlewares/row" + "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/glazed/handlers" + "github.com/go-go-golems/parka/pkg/glazed/parser" + "io" +) + +type QueryHandler struct { + cmd cmds.GlazeCommand + contextMiddlewares []glazed.ContextMiddleware + parserOptions []parser.ParserOption +} + +type QueryHandlerOption func(*QueryHandler) + +func NewQueryHandler(cmd cmds.GlazeCommand, options ...QueryHandlerOption) *QueryHandler { + h := &QueryHandler{ + cmd: cmd, + } + + for _, option := range options { + option(h) + } + + return h +} + +func WithQueryHandlerContextMiddlewares(middlewares ...glazed.ContextMiddleware) QueryHandlerOption { + return func(h *QueryHandler) { + h.contextMiddlewares = middlewares + } +} + +// WithQueryHandlerParserOptions sets the parser options for the QueryHandler +func WithQueryHandlerParserOptions(options ...parser.ParserOption) QueryHandlerOption { + return func(h *QueryHandler) { + h.parserOptions = options + } +} + +func (h *QueryHandler) Handle(c *gin.Context, writer io.Writer) error { + pc := glazed.NewCommandContext(h.cmd) + + h.contextMiddlewares = append( + h.contextMiddlewares, + glazed.NewContextParserMiddleware( + h.cmd, + glazed.NewCommandQueryParser(h.cmd, h.parserOptions...), + ), + ) + + for _, h := range h.contextMiddlewares { + err := h.Handle(c, pc) + if err != nil { + return err + } + } + + gp, err := handlers.CreateTableProcessor(pc, "json", "") + if err != nil { + return err + } + + // remove table middlewares because we are a streaming handler + gp.ReplaceTableMiddleware() + gp.AddRowMiddleware(row.NewOutputMiddleware(json2.NewOutputFormatter(), writer)) + + _, err = writer.Write([]byte("[\n")) + if err != nil { + return err + } + + ctx := c.Request.Context() + err = h.cmd.Run(ctx, pc.ParsedLayers, pc.ParsedParameters, gp) + if err != nil { + return err + } + + err = gp.Close(ctx) + if err != nil { + return err + } + + _, err = writer.Write([]byte("\n]")) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/handlers/command-dir/command-dir.go b/pkg/handlers/command-dir/command-dir.go index 76d5597..ac335c6 100644 --- a/pkg/handlers/command-dir/command-dir.go +++ b/pkg/handlers/command-dir/command-dir.go @@ -6,11 +6,10 @@ import ( "github.com/go-go-golems/clay/pkg/repositories" "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" - "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/glazed/parser" "github.com/go-go-golems/parka/pkg/handlers/config" "github.com/go-go-golems/parka/pkg/render" - "github.com/go-go-golems/parka/pkg/render/datatables" + "github.com/go-go-golems/parka/pkg/render/formatters/datatables" "github.com/go-go-golems/parka/pkg/render/layout" parka "github.com/go-go-golems/parka/pkg/server" "github.com/pkg/errors" @@ -413,7 +412,7 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { path = strings.TrimSuffix(path, "/") server.Router.GET(path+"/data/*path", func(c *gin.Context) { - commandPath := c.Param("CommandPath") + commandPath := c.Param("path") commandPath = strings.TrimPrefix(commandPath, "/") sqlCommand, ok := getRepositoryCommand(c, cd.Repository, commandPath) if !ok { @@ -421,9 +420,8 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { return } - handle := server.HandleSimpleQueryCommand(sqlCommand, - glazed.WithCreateProcessor(glazed.CreateJSONProcessor), - glazed.WithParserOptions(cd.computeParserOptions()...), + handle := server.HandleJSONQueryHandler(sqlCommand, + cd.computeParserOptions()..., ) handle(c) @@ -489,19 +487,21 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { AdditionalData: cd.AdditionalData, } - dataTablesProcessorFunc := datatables.NewDataTablesCreateOutputProcessorFunc( + ofFactory := datatables.NewOutputFormatterFactory( cd.TemplateLookup, cd.TemplateName, dt, ) - handle := server.HandleSimpleQueryCommand( - sqlCommand, - glazed.WithCreateProcessor(dataTablesProcessorFunc), - glazed.WithParserOptions(cd.computeParserOptions()...), - ) + _ = ofFactory - handle(c) + //handle := server.HandleJSONQueryHandler( + // sqlCommand, + // glazed.WithOutputFormatterFactory(ofFactory), + // glazed.WithParserOptions(cd.computeParserOptions()...), + //) + // + //handle(c) }) server.Router.GET(path+"/download/*path", func(c *gin.Context) { @@ -565,18 +565,18 @@ func (cd *CommandDirHandler) Serve(server *parka.Server, path string) error { } parserOptions := cd.computeParserOptions() - // override parameter layers at the end parserOptions = append(parserOptions, parser.WithAppendOverrides("glazed", glazedOverrides)) - handle := server.HandleSimpleQueryOutputFileCommand( - sqlCommand, - tmpFile.Name(), - fileName, - glazed.WithParserOptions(parserOptions...), - ) - - handle(c) + _ = sqlCommand + //handle := server.HandleSimpleQueryOutputFileCommand( + // sqlCommand, + // tmpFile.Name(), + // fileName, + // glazed.WithParserOptions(parserOptions...), + //) + // + //handle(c) }) return nil diff --git a/pkg/render/datatables/datatables.go b/pkg/render/formatters/datatables/datatables.go similarity index 72% rename from pkg/render/datatables/datatables.go rename to pkg/render/formatters/datatables/datatables.go index d2a6950..a4213af 100644 --- a/pkg/render/datatables/datatables.go +++ b/pkg/render/formatters/datatables/datatables.go @@ -1,6 +1,7 @@ package datatables import ( + "context" "embed" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" @@ -13,6 +14,7 @@ import ( "github.com/go-go-golems/parka/pkg/glazed" "github.com/go-go-golems/parka/pkg/render" "github.com/go-go-golems/parka/pkg/render/layout" + "golang.org/x/sync/errgroup" "html/template" "io" ) @@ -23,7 +25,7 @@ import ( type DataTables struct { Command *cmds.CommandDescription // LongDescription is the HTML of the rendered markdown of the long description of the command. - LongDescription string + LongDescription template.HTML Layout *layout.Layout Links []layout.Link @@ -37,8 +39,8 @@ type DataTables struct { // // TODO(manuel, 2023-06-04): Maybe we could make this be an iterator of rows that provide access to the individual // columns for more interesting HTML shenanigans too. - JSStream <-chan template.JS - HTMLStream <-chan template.HTML + JSStream chan template.JS + HTMLStream chan template.HTML // Configuring the template to load the table data through javascript, and provide the data itself // as a JSON array inlined in the HTML of the page. JSRendering bool @@ -119,6 +121,87 @@ func NewOutputFormatter( } } +func (of *OutputFormatter) Close(ctx context.Context) error { + close(of.rowC) + close(of.columnsC) + return nil +} + +func (dt *OutputFormatter) RegisterMiddlewares(p *middlewares.TableProcessor, writer io.Writer) error { + var of formatters.RowOutputFormatter + if dt.dt.JSRendering { + of = json.NewOutputFormatter() + dt.dt.JSStream = make(chan template.JS, 100) + } else { + of = table_formatter.NewOutputFormatter("html") + dt.dt.HTMLStream = make(chan template.HTML, 100) + } + + p.AddRowMiddleware(row.NewColumnsChannelMiddleware(dt.columnsC, true)) + p.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) + + return nil +} + +func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { + // Here, we use the parsed layer to configure the glazed middlewares. + // We then use the created formatters.TableOutputFormatter as a basis for + // our own output formatter that renders into an HTML template. + var err error + + layout_, err := layout.ComputeLayout(pc) + if err != nil { + return err + } + + description := pc.Cmd.Description() + + longHTML, err := render.RenderMarkdownToHTML(description.Long) + if err != nil { + return err + } + + dt_ := dt.dt.Clone() + dt_.Layout = layout_ + dt_.LongDescription = template.HTML(longHTML) + dt_.Command = description + + // Wait for the column names to be sent to the channel. This will only + // take the first row into account. + columns := <-dt.columnsC + dt_.Columns = columns + + // start copying from rowC to HTML or JS stream + + eg, ctx2 := errgroup.WithContext(c) + + eg.Go(func() error { + err := dt.t.Execute(w, dt_) + if err != nil { + return err + } + + return nil + }) + + eg.Go(func() error { + for { + select { + case <-ctx2.Done(): + return ctx2.Err() + case row_ := <-dt.rowC: + if dt.dt.JSRendering { + dt.dt.JSStream <- template.JS(row_) + } else { + dt.dt.HTMLStream <- template.HTML(row_) + } + } + } + }) + + return eg.Wait() +} + type OutputFormatterFactory struct { TemplateName string Lookup render.TemplateLookup @@ -149,34 +232,21 @@ func (dtoff *OutputFormatterFactory) CreateOutputFormatter( return nil, err } - dt_.LongDescription = longHTML + dt_.LongDescription = template.HTML(longHTML) + dt_.Layout = layout_ return NewOutputFormatter(t, dtoff.DataTables), nil } -func (dt *OutputFormatter) RegisterMiddlewares( - m *middlewares.TableProcessor, -) error { - var of formatters.RowOutputFormatter - if dt.dt.JSRendering { - of = json.NewOutputFormatter() - dt.dt.JSStream = make(chan template.JS, 100) - } else { - of = table_formatter.NewOutputFormatter("html") - dt.dt.HTMLStream = make(chan template.HTML, 100) +func NewOutputFormatterFactory( + lookup render.TemplateLookup, + templateName string, + dataTables *DataTables, +) *OutputFormatterFactory { + return &OutputFormatterFactory{ + TemplateName: templateName, + Lookup: lookup, + DataTables: dataTables, } - - // TODO add a "get columns" middleware that can be used to get the columns from the table - // over a single channel that we can wait on in the render call - - m.AddRowMiddleware(row.NewOutputChannelMiddleware(of, dt.rowC)) - - return nil -} - -func (dt *OutputFormatter) Output(c *gin.Context, pc *glazed.CommandContext, w io.Writer) error { - // clear all table middlewares because we are streaming using a row output middleware - - return nil } diff --git a/pkg/render/datatables/templates/data-tables.tmpl.html b/pkg/render/formatters/datatables/templates/data-tables.tmpl.html similarity index 100% rename from pkg/render/datatables/templates/data-tables.tmpl.html rename to pkg/render/formatters/datatables/templates/data-tables.tmpl.html diff --git a/pkg/render/html.go b/pkg/render/formatters/html.go similarity index 94% rename from pkg/render/html.go rename to pkg/render/formatters/html.go index 666675f..ea43620 100644 --- a/pkg/render/html.go +++ b/pkg/render/formatters/html.go @@ -1,8 +1,9 @@ -package render +package formatters import ( "github.com/gin-gonic/gin" "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/render" "github.com/go-go-golems/parka/pkg/render/layout" "html/template" "io" @@ -32,7 +33,7 @@ import ( // a template to be added in the back in the front. type HTMLTemplateOutputFormatter struct { TemplateName string - Lookup TemplateLookup + Lookup render.TemplateLookup Data map[string]interface{} } @@ -50,7 +51,7 @@ func WithHTMLTemplateOutputFormatterData(data map[string]interface{}) HTMLTempla } func NewHTMLTemplateOutputFormatter( - lookup TemplateLookup, + lookup render.TemplateLookup, templateName string, options ...HTMLTemplateOutputFormatterOption, ) *HTMLTemplateOutputFormatter { @@ -79,7 +80,7 @@ func (H *HTMLTemplateOutputFormatter) Output(c *gin.Context, pc *glazed.CommandC description := pc.Cmd.Description() - longHTML, err := RenderMarkdownToHTML(description.Long) + longHTML, err := render.RenderMarkdownToHTML(description.Long) if err != nil { return err } diff --git a/pkg/server/server.go b/pkg/server/server.go index a396b23..76cb984 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,9 +7,11 @@ import ( "github.com/gin-gonic/contrib/gzip" "github.com/gin-gonic/gin" "github.com/go-go-golems/glazed/pkg/cmds" - "github.com/go-go-golems/parka/pkg/glazed" + "github.com/go-go-golems/parka/pkg/glazed/handlers/json" + "github.com/go-go-golems/parka/pkg/glazed/parser" "github.com/go-go-golems/parka/pkg/render" utils_fs "github.com/go-go-golems/parka/pkg/utils/fs" + "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" "io/fs" "net/http" @@ -218,43 +220,20 @@ func (s *Server) Run(ctx context.Context) error { return eg.Wait() } -func (s *Server) HandleSimpleQueryCommand( +func (s *Server) HandleJSONQueryHandler( cmd cmds.GlazeCommand, - options ...glazed.HandleOption, + parserOptions ...parser.ParserOption, ) gin.HandlerFunc { - opts := glazed.NewHandleOptions(options) - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, - glazed.NewCommandQueryParser(cmd, opts.ParserOptions...)), + handler := json.NewQueryHandler(cmd, + json.WithQueryHandlerParserOptions(parserOptions...), ) - return glazed.GinHandleGlazedCommand(cmd, opts) -} - -func (s *Server) HandleSimpleQueryOutputFileCommand( - cmd cmds.GlazeCommand, - outputFile string, - fileName string, - options ...glazed.HandleOption, -) gin.HandlerFunc { - opts := glazed.NewHandleOptions(options) - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, glazed.NewCommandQueryParser(cmd, opts.ParserOptions...)), - ) - return glazed.GinHandleGlazedCommandWithOutputFile(cmd, outputFile, fileName, opts) -} - -// TODO(manuel, 2023-02-28) We want to provide a handler to catch errors while parsing parameters - -func (s *Server) HandleSimpleFormCommand( - cmd cmds.GlazeCommand, - options ...glazed.HandleOption, -) gin.HandlerFunc { - opts := &glazed.HandleOptions{} - for _, option := range options { - option(opts) + return func(c *gin.Context) { + err := handler.Handle(c, c.Writer) + if err != nil { + log.Error().Err(err).Msg("failed to handle query") + c.JSON(http.StatusInternalServerError, gin.H{ + "error": err.Error(), + }) + } } - opts.Handlers = append(opts.Handlers, - glazed.NewParserCommandHandlerFunc(cmd, glazed.NewCommandFormParser(cmd, opts.ParserOptions...)), - ) - return glazed.GinHandleGlazedCommand(cmd, opts) } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 107d696..6775d67 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -47,7 +47,7 @@ func TestRunGlazedCommand(t *testing.T) { s, err := server.NewServer() require.NoError(t, err) - handler := s.HandleSimpleQueryCommand(tc) + handler := s.HandleJSONQueryHandler(tc) gin.SetMode(gin.TestMode)