From b6026f58ba4d0399cc2dd03ceb9ddc96cc0b26c3 Mon Sep 17 00:00:00 2001 From: Manuel Odendahl Date: Fri, 7 Feb 2025 11:50:21 -0500 Subject: [PATCH] :ambulance: Pass middlewares through all other handlers --- pkg/glazed/handlers/glazed/glazed.go | 15 +- pkg/glazed/handlers/json/json.go | 9 +- .../handlers/output-file/output-file.go | 254 ++++++++++-------- pkg/glazed/handlers/sse/sse.go | 23 +- pkg/glazed/handlers/text/text.go | 20 +- pkg/handlers/generic-command/generic.go | 72 +++-- 6 files changed, 238 insertions(+), 155 deletions(-) diff --git a/pkg/glazed/handlers/glazed/glazed.go b/pkg/glazed/handlers/glazed/glazed.go index 5040cf5..881f829 100644 --- a/pkg/glazed/handlers/glazed/glazed.go +++ b/pkg/glazed/handlers/glazed/glazed.go @@ -1,6 +1,8 @@ package glazed import ( + "net/http" + "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" "github.com/go-go-golems/glazed/pkg/cmds/middlewares" @@ -10,7 +12,6 @@ import ( middlewares2 "github.com/go-go-golems/parka/pkg/glazed/middlewares" "github.com/labstack/echo/v4" "github.com/pkg/errors" - "net/http" ) type QueryHandler struct { @@ -44,16 +45,20 @@ func (h *QueryHandler) Handle(c echo.Context) error { description := h.cmd.Description() parsedLayers := layers.NewParsedLayers() - middlewares_ := append(h.middlewares, - middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), - middlewares.SetFromDefaults(), + middlewares_ := append( + []middlewares.Middleware{ + middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), + }, + h.middlewares..., ) + middlewares_ = append(middlewares_, middlewares.SetFromDefaults()) + err := middlewares.ExecuteMiddlewares(description.Layers, parsedLayers, middlewares_...) if err != nil { return err } - glazedLayer, ok := parsedLayers.Get("glazed") + glazedLayer, ok := parsedLayers.Get(settings.GlazedSlug) if !ok { return errors.New("glazed layer not found") } diff --git a/pkg/glazed/handlers/json/json.go b/pkg/glazed/handlers/json/json.go index d12282c..994ebed 100644 --- a/pkg/glazed/handlers/json/json.go +++ b/pkg/glazed/handlers/json/json.go @@ -47,10 +47,13 @@ func (h *QueryHandler) Handle(c echo.Context) error { parsedLayers := layers.NewParsedLayers() middlewares_ := append( - h.middlewares, - middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), - middlewares.SetFromDefaults(), + []middlewares.Middleware{ + middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), + }, + h.middlewares..., ) + middlewares_ = append(middlewares_, middlewares.SetFromDefaults()) + err := middlewares.ExecuteMiddlewares(description.Layers, parsedLayers, middlewares_...) if err != nil { return err diff --git a/pkg/glazed/handlers/output-file/output-file.go b/pkg/glazed/handlers/output-file/output-file.go index e655451..8349472 100644 --- a/pkg/glazed/handlers/output-file/output-file.go +++ b/pkg/glazed/handlers/output-file/output-file.go @@ -2,131 +2,159 @@ package output_file import ( "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/middlewares" "github.com/go-go-golems/glazed/pkg/cmds/parameters" - "github.com/go-go-golems/glazed/pkg/helpers/list" "github.com/go-go-golems/glazed/pkg/settings" "github.com/go-go-golems/parka/pkg/glazed/handlers/glazed" - parka_middlewares "github.com/go-go-golems/parka/pkg/glazed/middlewares" "github.com/labstack/echo/v4" "github.com/pkg/errors" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" ) -// CreateGlazedFileHandler creates a handler that will run a glazed command and write the output -// with a Content-Disposition header to the response writer. -// -// If an output format requires writing to a temporary file locally, such as excel, -// the handler is wrapped in a temporary file handler. -func CreateGlazedFileHandler( - cmd cmds.GlazeCommand, - fileName string, - middlewares_ ...middlewares.Middleware, -) echo.HandlerFunc { - return func(c echo.Context) error { - glazedOverrides := map[string]interface{}{} - needsRealFileOutput := false - - // create a temporary file for glazed output - if strings.HasSuffix(fileName, ".csv") { - glazedOverrides["output"] = "table" - glazedOverrides["table-format"] = "csv" - } else if strings.HasSuffix(fileName, ".tsv") { - glazedOverrides["output"] = "table" - glazedOverrides["table-format"] = "tsv" - } else if strings.HasSuffix(fileName, ".md") { - glazedOverrides["output"] = "table" - glazedOverrides["table-format"] = "markdown" - } else if strings.HasSuffix(fileName, ".html") { - glazedOverrides["output"] = "table" - glazedOverrides["table-format"] = "html" - } else if strings.HasSuffix(fileName, ".json") { - glazedOverrides["output"] = "json" - } else if strings.HasSuffix(fileName, ".yaml") { - glazedOverrides["output"] = "yaml" - } else if strings.HasSuffix(fileName, ".xlsx") { - glazedOverrides["output"] = "excel" - needsRealFileOutput = true - } else if strings.HasSuffix(fileName, ".txt") { - glazedOverrides["output"] = "table" - glazedOverrides["table-format"] = "ascii" - } else { - return errors.New("unsupported file format") +type QueryHandler struct { + cmd cmds.GlazeCommand + fileName string + middlewares []middlewares.Middleware +} + +type QueryHandlerOption func(*QueryHandler) + +func NewQueryHandler(cmd cmds.GlazeCommand, fileName string, options ...QueryHandlerOption) *QueryHandler { + h := &QueryHandler{ + cmd: cmd, + fileName: fileName, + } + + for _, option := range options { + option(h) + } + + return h +} + +func WithMiddlewares(middlewares ...middlewares.Middleware) QueryHandlerOption { + return func(handler *QueryHandler) { + handler.middlewares = middlewares + } +} + +func (h *QueryHandler) Handle(c echo.Context) error { + glazedOverrides := map[string]interface{}{} + needsRealFileOutput := false + + // create a temporary file for glazed output + if strings.HasSuffix(h.fileName, ".csv") { + glazedOverrides["output"] = "table" + glazedOverrides["table-format"] = "csv" + } else if strings.HasSuffix(h.fileName, ".tsv") { + glazedOverrides["output"] = "table" + glazedOverrides["table-format"] = "tsv" + } else if strings.HasSuffix(h.fileName, ".md") { + glazedOverrides["output"] = "table" + glazedOverrides["table-format"] = "markdown" + } else if strings.HasSuffix(h.fileName, ".html") { + glazedOverrides["output"] = "table" + glazedOverrides["table-format"] = "html" + } else if strings.HasSuffix(h.fileName, ".json") { + glazedOverrides["output"] = "json" + } else if strings.HasSuffix(h.fileName, ".yaml") { + glazedOverrides["output"] = "yaml" + } else if strings.HasSuffix(h.fileName, ".xlsx") { + glazedOverrides["output"] = "excel" + needsRealFileOutput = true + } else if strings.HasSuffix(h.fileName, ".txt") { + glazedOverrides["output"] = "table" + glazedOverrides["table-format"] = "ascii" + } else { + return errors.New("unsupported file format") + } + + var tmpFile *os.File + var err error + + glazedOverride := middlewares.UpdateFromMap( + map[string]map[string]interface{}{ + settings.GlazedSlug: glazedOverrides, + }, + parameters.WithParseStepSource("output-file-glazed-override"), + ) + + middlewares_ := append( + []middlewares.Middleware{ + glazedOverride, + }, + h.middlewares..., + ) + + handler := glazed.NewQueryHandler(h.cmd, + glazed.WithMiddlewares(middlewares_...), + ) + + baseName := filepath.Base(h.fileName) + c.Response().Header().Set("Content-Disposition", "attachment; filename="+baseName) + + // excel output needs a real output file, otherwise we can go stream to the HTTP response + if needsRealFileOutput { + tmpFile, err = os.CreateTemp("/tmp", fmt.Sprintf("glazed-output-*.%s", h.fileName)) + if err != nil { + return errors.Wrap(err, "could not create temporary file") } + defer func(name string) { + _ = os.Remove(name) + }(tmpFile.Name()) + + // now check file suffix for content-type + glazedOverrides["output-file"] = tmpFile.Name() + + // here we have the output of the handler go to a request that we discard, and + // we instead copy the temporary file to the response writer + res := httptest.NewRecorder() + req := c.Request() + newCtx := c.Echo().NewContext(req, res) - var tmpFile *os.File - var err error - - glazedOverride := middlewares.UpdateFromMap( - map[string]map[string]interface{}{ - settings.GlazedSlug: glazedOverrides, - }, - parameters.WithParseStepSource("output-file-glazed-override"), - ) - - handler := glazed.NewQueryHandler(cmd, - glazed.WithMiddlewares( - list.Prepend(middlewares_, - parka_middlewares.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), - glazedOverride)..., - )) - - baseName := filepath.Base(fileName) - c.Response().Header().Set("Content-Disposition", "attachment; filename="+baseName) - - // excel output needs a real output file, otherwise we can go stream to the HTTP response - if needsRealFileOutput { - tmpFile, err = os.CreateTemp("/tmp", fmt.Sprintf("glazed-output-*.%s", fileName)) - if err != nil { - return errors.Wrap(err, "could not create temporary file") - } - defer func(name string) { - _ = os.Remove(name) - }(tmpFile.Name()) - - // now check file suffix for content-type - glazedOverrides["output-file"] = tmpFile.Name() - - // here we have the output of the handler go to a request that we discard, and - // we instead copy the temporary file to the response writer - res := httptest.NewRecorder() - req := c.Request() - newCtx := c.Echo().NewContext(req, res) - - err = handler.Handle(newCtx) - if err != nil { - return err - } - - // copy tmpFile to output - f, err := os.Open(tmpFile.Name()) - if err != nil { - return errors.Wrap(err, "could not open temporary file") - } - defer func(f *os.File) { - _ = f.Close() - }(f) - - c.Response().Header().Set("Content-Type", "application/octet-stream") - c.Response().WriteHeader(http.StatusOK) - - _, err = io.Copy(c.Response().Writer, f) - if err != nil { - return err - } - } else { - err = handler.Handle(c) - if err != nil { - return err - } + err = handler.Handle(newCtx) + if err != nil { + return err } - return nil + // copy tmpFile to output + f, err := os.Open(tmpFile.Name()) + if err != nil { + return errors.Wrap(err, "could not open temporary file") + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + c.Response().Header().Set("Content-Type", "application/octet-stream") + c.Response().WriteHeader(http.StatusOK) + + _, err = io.Copy(c.Response().Writer, f) + if err != nil { + return err + } + } else { + err = handler.Handle(c) + if err != nil { + return err + } } + + return nil +} + +func CreateGlazedFileHandler( + cmd cmds.GlazeCommand, + fileName string, + options ...QueryHandlerOption, +) echo.HandlerFunc { + handler := NewQueryHandler(cmd, fileName, options...) + return handler.Handle } diff --git a/pkg/glazed/handlers/sse/sse.go b/pkg/glazed/handlers/sse/sse.go index 4353e06..1007b09 100644 --- a/pkg/glazed/handlers/sse/sse.go +++ b/pkg/glazed/handlers/sse/sse.go @@ -4,6 +4,8 @@ package sse import ( "fmt" + "net/http" + "github.com/go-go-golems/glazed/pkg/cmds" "github.com/go-go-golems/glazed/pkg/cmds/layers" "github.com/go-go-golems/glazed/pkg/cmds/middlewares" @@ -14,7 +16,6 @@ import ( middlewares2 "github.com/go-go-golems/parka/pkg/glazed/middlewares" "github.com/kucherenkovova/safegroup" "github.com/labstack/echo/v4" - "net/http" ) type QueryHandler struct { @@ -42,17 +43,24 @@ func WithMiddlewares(middlewares ...middlewares.Middleware) QueryHandlerOption { } } +var _ handlers.Handler = (*QueryHandler)(nil) + func (h *QueryHandler) Handle(c echo.Context) error { description := h.cmd.Description() parsedLayers := layers.NewParsedLayers() - middlewares_ := append([]middlewares.Middleware{ - middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), - }, h.middlewares...) + middlewares_ := append( + []middlewares.Middleware{ + middlewares2.UpdateFromQueryParameters(c, parameters.WithParseStepSource("query")), + }, + h.middlewares..., + ) + middlewares_ = append(middlewares_, middlewares.SetFromDefaults()) err := middlewares.ExecuteMiddlewares(description.Layers, parsedLayers, middlewares_...) if err != nil { return err } + c.Response().Header().Set("Content-Type", "text/event-stream") c.Response().Header().Set("Cache-Control", "no-cache") c.Response().Header().Set("Connection", "keep-alive") @@ -148,15 +156,14 @@ func (h *QueryHandler) Handle(c echo.Context) error { default: return &handlers.UnsupportedCommandError{Command: h.cmd} } + return nil } func CreateQueryHandler( cmd cmds.Command, - middlewares_ ...middlewares.Middleware, + options ...QueryHandlerOption, ) echo.HandlerFunc { - handler := NewQueryHandler(cmd, - WithMiddlewares(middlewares_...), - ) + handler := NewQueryHandler(cmd, options...) return handler.Handle } diff --git a/pkg/glazed/handlers/text/text.go b/pkg/glazed/handlers/text/text.go index 2478063..91dd8cb 100644 --- a/pkg/glazed/handlers/text/text.go +++ b/pkg/glazed/handlers/text/text.go @@ -44,12 +44,16 @@ func (h *QueryHandler) Handle(c echo.Context) error { description := h.cmd.Description() parsedLayers := layers.NewParsedLayers() - middlewares_ := append(h.middlewares, - parka_middlewares.UpdateFromQueryParameters(c, - parameters.WithParseStepSource("query"), - ), - middlewares.SetFromDefaults(), + middlewares_ := append( + []middlewares.Middleware{ + parka_middlewares.UpdateFromQueryParameters(c, + parameters.WithParseStepSource("query"), + ), + }, + h.middlewares..., ) + middlewares_ = append(middlewares_, middlewares.SetFromDefaults()) + err := middlewares.ExecuteMiddlewares(description.Layers, parsedLayers, middlewares_...) if err != nil { return err @@ -110,10 +114,8 @@ func (h *QueryHandler) Handle(c echo.Context) error { func CreateQueryHandler( cmd cmds.Command, - middlewares_ ...middlewares.Middleware, + options ...QueryHandlerOption, ) echo.HandlerFunc { - handler := NewQueryHandler(cmd, - WithMiddlewares(middlewares_...), - ) + handler := NewQueryHandler(cmd, options...) return handler.Handle } diff --git a/pkg/handlers/generic-command/generic.go b/pkg/handlers/generic-command/generic.go index 97ad1af..8cd53cb 100644 --- a/pkg/handlers/generic-command/generic.go +++ b/pkg/handlers/generic-command/generic.go @@ -322,37 +322,76 @@ func (gch *GenericCommandHandler) ServeRepository(server *parka.Server, basePath return nil } +// computeDataTablesOptions returns the options used for DataTables handlers +func (gch *GenericCommandHandler) computeDataTablesOptions() []datatables.QueryHandlerOption { + return []datatables.QueryHandlerOption{ + datatables.WithMiddlewares(gch.preMiddlewares...), + datatables.WithMiddlewares(gch.middlewares...), + datatables.WithMiddlewares(gch.postMiddlewares...), + datatables.WithTemplateLookup(gch.TemplateLookup), + datatables.WithTemplateName(gch.TemplateName), + datatables.WithAdditionalData(gch.AdditionalData), + datatables.WithStreamRows(gch.Stream), + } +} + +// computeJSONOptions returns the options used for JSON handlers +func (gch *GenericCommandHandler) computeJSONOptions() []json.QueryHandlerOption { + return []json.QueryHandlerOption{ + json.WithMiddlewares(gch.preMiddlewares...), + json.WithMiddlewares(gch.middlewares...), + json.WithMiddlewares(gch.postMiddlewares...), + } +} + +// computeTextOptions returns the options used for text handlers +func (gch *GenericCommandHandler) computeTextOptions() []text.QueryHandlerOption { + return []text.QueryHandlerOption{ + text.WithMiddlewares(gch.preMiddlewares...), + text.WithMiddlewares(gch.middlewares...), + text.WithMiddlewares(gch.postMiddlewares...), + } +} + +// computeSSEOptions returns the options used for SSE handlers +func (gch *GenericCommandHandler) computeSSEOptions() []sse.QueryHandlerOption { + return []sse.QueryHandlerOption{ + sse.WithMiddlewares(gch.preMiddlewares...), + sse.WithMiddlewares(gch.middlewares...), + sse.WithMiddlewares(gch.postMiddlewares...), + } +} + +// computeOutputFileOptions returns the options used for output file handlers +func (gch *GenericCommandHandler) computeOutputFileOptions() []output_file.QueryHandlerOption { + return []output_file.QueryHandlerOption{ + output_file.WithMiddlewares(gch.preMiddlewares...), + output_file.WithMiddlewares(gch.middlewares...), + output_file.WithMiddlewares(gch.postMiddlewares...), + } +} + func (gch *GenericCommandHandler) ServeData(c echo.Context, command cmds.Command) error { switch v := command.(type) { case cmds.GlazeCommand: - return json.CreateJSONQueryHandler(v, json.WithMiddlewares(gch.middlewares...))(c) + return json.CreateJSONQueryHandler(v, gch.computeJSONOptions()...)(c) default: - return text.CreateQueryHandler(v)(c) + return text.CreateQueryHandler(v, gch.computeTextOptions()...)(c) } } func (gch *GenericCommandHandler) ServeText(c echo.Context, command cmds.Command) error { - return text.CreateQueryHandler(command, gch.middlewares...)(c) + return text.CreateQueryHandler(command, gch.computeTextOptions()...)(c) } func (gch *GenericCommandHandler) ServeStreaming(c echo.Context, command cmds.Command) error { - return sse.CreateQueryHandler(command, gch.middlewares...)(c) + return sse.CreateQueryHandler(command, gch.computeSSEOptions()...)(c) } func (gch *GenericCommandHandler) ServeDataTables(c echo.Context, command cmds.Command, downloadPath string) error { switch v := command.(type) { case cmds.GlazeCommand: - options := []datatables.QueryHandlerOption{ - datatables.WithMiddlewares(gch.preMiddlewares...), - datatables.WithMiddlewares(gch.middlewares...), - datatables.WithMiddlewares(gch.postMiddlewares...), - datatables.WithTemplateLookup(gch.TemplateLookup), - datatables.WithTemplateName(gch.TemplateName), - datatables.WithAdditionalData(gch.AdditionalData), - datatables.WithStreamRows(gch.Stream), - } - - return datatables.CreateDataTablesHandler(v, gch.BasePath, downloadPath, options...)(c) + return datatables.CreateDataTablesHandler(v, gch.BasePath, downloadPath, gch.computeDataTablesOptions()...)(c) default: return c.JSON(http.StatusInternalServerError, utils.H{"error": "command is not a glazed command"}) } @@ -374,7 +413,7 @@ func (gch *GenericCommandHandler) ServeDownload(c echo.Context, command cmds.Com return output_file.CreateGlazedFileHandler( v, fileName, - gch.middlewares..., + gch.computeOutputFileOptions()..., )(c) case cmds.WriterCommand: @@ -393,7 +432,6 @@ func (gch *GenericCommandHandler) ServeDownload(c echo.Context, command cmds.Com default: return c.JSON(http.StatusInternalServerError, utils.H{"error": "command is not a glazed/writer command"}) } - } // getRepositoryCommand lookups a command in the given repository and return success as bool and the given command,