From a2b2109b76497e8b7eeea5b3e10f664a6d15dbb3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 3 Nov 2022 15:52:36 +0100 Subject: [PATCH] AppProvider: make CodiMD config non hard-coded (#3401) --- changelog/unreleased/app-tweaks.md | 7 ++ .../custom-mime-types-demo.json | 4 +- examples/nextcloud-integration/revad.toml | 2 +- .../appprovider-codimd.toml | 19 +++ examples/storage-references/gateway.toml | 4 - go.mod | 1 + go.sum | 2 + .../grpc/services/appprovider/appprovider.go | 56 +++++++-- .../http/services/appprovider/appprovider.go | 7 +- pkg/app/provider/wopi/wopi.go | 109 ++++++------------ pkg/mime/mime.go | 25 +++- 11 files changed, 140 insertions(+), 96 deletions(-) create mode 100644 changelog/unreleased/app-tweaks.md create mode 100644 examples/storage-references/appprovider-codimd.toml diff --git a/changelog/unreleased/app-tweaks.md b/changelog/unreleased/app-tweaks.md new file mode 100644 index 0000000000..4706676788 --- /dev/null +++ b/changelog/unreleased/app-tweaks.md @@ -0,0 +1,7 @@ +Enhancement: make WOPI bridged apps (CodiMD) configuration non hard-coded + +The configuration of the custom mimetypes has been moved to the AppProvider, +and the given mimetypes are used to configure bridged apps by sharing +the corresponding config item to the drivers. + +https://github.com/cs3org/reva/pull/3401 diff --git a/examples/nextcloud-integration/custom-mime-types-demo.json b/examples/nextcloud-integration/custom-mime-types-demo.json index a48c986790..5e5fbf4e54 100644 --- a/examples/nextcloud-integration/custom-mime-types-demo.json +++ b/examples/nextcloud-integration/custom-mime-types-demo.json @@ -1,4 +1,4 @@ { - ".zmd": "application/compressed-markdown", - ".zep": "application/compressed-etherpad" + "zmd": "application/compressed-markdown", + "zep": "application/compressed-etherpad" } diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index 34b2c6d6dc..fda4fe6428 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -67,13 +67,13 @@ driver = "memory" [grpc.services.appprovider] driver = "wopi" +custom_mime_types_json = "custom-mime-types-demo.json" [grpc.services.appprovider.drivers.wopi] iop_secret = "hello" wopi_url = "http://0.0.0.0:8880/" app_name = "Collabora" app_url = "https://your-collabora-server.org:9980" -custom_mime_types_json = "custom-mime-types-demo.json" [grpc.services.appregistry] driver = "static" diff --git a/examples/storage-references/appprovider-codimd.toml b/examples/storage-references/appprovider-codimd.toml new file mode 100644 index 0000000000..da04af947a --- /dev/null +++ b/examples/storage-references/appprovider-codimd.toml @@ -0,0 +1,19 @@ +[shared] +gatewaysvc = "localhost:your-revad-gateway-port" + +[grpc] +address = "0.0.0.0:12345" + +[grpc.services.appprovider] +driver = "wopi" +custom_mime_types_json = "custom-mime-types-demo.json" +mime_types = ["text/markdown", "application/compressed-markdown", "text/plain"] +app_provider_url = "localhost:12345" +language = "en-GB" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "hello" +wopi_url = "http://0.0.0.0:8880/" +app_name = "CodiMD" +app_url = "https://your-codimd-server.org:3000" +app_int_url = "https://your-codimd-server.org:3000" diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 963d0fe21d..098bc2325e 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -44,10 +44,6 @@ mime_types = [ {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} ] -[grpc.services.appprovider] -mime_types = ["text/plain"] -language = "en-GB" - [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] diff --git a/go.mod b/go.mod index 2cc60c830d..c576dc2d61 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect diff --git a/go.sum b/go.sum index 5485bf24e3..386e5f7220 100644 --- a/go.sum +++ b/go.sum @@ -282,6 +282,8 @@ github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4Vg github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9 h1:3um08ooi0/lyRmK2eE1XTKmRQHDzPu0IvpCPMljyMZ8= +github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9/go.mod h1:EJaddanP+JfU3UkVvn0rYYF3b/gD7eZRejbTHqiQExA= github.com/go-acme/lego/v4 v4.4.0/go.mod h1:l3+tFUFZb590dWcqhWZegynUthtaHJbG2fevUpoOOE0= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 881dc4c4b7..9c209e8fb8 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -20,7 +20,10 @@ package appprovider import ( "context" + "encoding/json" "errors" + "fmt" + "io/ioutil" "os" "strconv" "time" @@ -33,6 +36,7 @@ import ( "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -52,13 +56,14 @@ type service struct { } type config struct { - Driver string `mapstructure:"driver"` - Drivers map[string]map[string]interface{} `mapstructure:"drivers"` - AppProviderURL string `mapstructure:"app_provider_url"` - GatewaySvc string `mapstructure:"gatewaysvc"` - MimeTypes []string `mapstructure:"mime_types"` - Priority uint64 `mapstructure:"priority"` - Language string `mapstructure:"language"` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` + AppProviderURL string `mapstructure:"app_provider_url"` + GatewaySvc string `mapstructure:"gatewaysvc"` + MimeTypes []string `mapstructure:"mime_types" docs:"nil;A list of mime types supported by this app."` + CustomMimeTypesJSON string `mapstructure:"custom_mime_types_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` + Priority uint64 `mapstructure:"priority"` + Language string `mapstructure:"language"` } func (c *config) init() { @@ -85,6 +90,12 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } + // read and register custom mime types if configured + err = registerMimeTypes(c.CustomMimeTypesJSON) + if err != nil { + return nil, err + } + provider, err := getProvider(c) if err != nil { return nil, err @@ -99,9 +110,31 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return service, nil } +func registerMimeTypes(mappingFile string) error { + // TODO(lopresti) this function also exists in the storage provider, to be seen if we want to factor it out, though a + // fileext <-> mimetype "service" would have to be served by the gateway for it to be accessible both by storage providers and app providers. + if mappingFile != "" { + f, err := ioutil.ReadFile(mappingFile) + if err != nil { + return fmt.Errorf("appprovider: error reading the custom mime types file: +%v", err) + } + mimeTypes := map[string]string{} + err = json.Unmarshal(f, &mimeTypes) + if err != nil { + return fmt.Errorf("appprovider: error unmarshalling the custom mime types file: +%v", err) + } + // register all mime types that were read + for e, m := range mimeTypes { + mime.RegisterMime(e, m) + } + } + return nil +} + func (s *service) registerProvider() { // Give the appregistry service time to come up - time.Sleep(2 * time.Second) + // TODO(lopresti) we should register the appproviders after all other microservices + time.Sleep(3 * time.Second) ctx := context.Background() log := logger.New().With().Int("pid", os.Getpid()).Logger() @@ -119,6 +152,7 @@ func (s *service) registerProvider() { mimeTypes = append(mimeTypes, m.(string)) } pInfo.MimeTypes = mimeTypes + log.Info().Str("appprovider", s.conf.AppProviderURL).Interface("mimetypes", mimeTypes).Msg("appprovider supported mimetypes") } client, err := pool.GetGatewayServiceClient(pool.Endpoint(s.conf.GatewaySvc)) @@ -165,7 +199,10 @@ func (s *service) Register(ss *grpc.Server) { func getProvider(c *config) (app.Provider, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + driverConf := c.Drivers[c.Driver] + // share the mime_types config entry to the drivers + driverConf["mime_types"] = c.MimeTypes + return f(driverConf) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -183,5 +220,4 @@ func (s *service) OpenInApp(ctx context.Context, req *providerpb.OpenInAppReques AppUrl: appURL, } return res, nil - } diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 7c16c324fd..7cf0e79fab 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -373,9 +373,9 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { return } - viewMode := getViewMode(statRes.Info, r.Form.Get("view_mode")) + viewMode := resolveViewMode(statRes.Info, r.Form.Get("view_mode")) if viewMode == gateway.OpenInAppRequest_VIEW_MODE_INVALID { - writeError(w, r, appErrorInvalidParameter, "invalid view mode", err) + writeError(w, r, appErrorUnauthenticated, "permission denied when accessing the file", err) return } @@ -436,7 +436,7 @@ func filterAppsByUserAgent(mimeTypes []*appregistry.MimeTypeInfo, userAgent stri return res } -func getViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { +func resolveViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { if vm != "" { return utils.GetViewMode(vm) } @@ -451,6 +451,7 @@ func getViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest case canView: viewMode = gateway.OpenInAppRequest_VIEW_MODE_READ_ONLY default: + // no permissions, will return access denied viewMode = gateway.OpenInAppRequest_VIEW_MODE_INVALID } return viewMode diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 40dd91cb7a..49004de73d 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -56,17 +56,17 @@ func init() { } type config struct { - IOPSecret string `mapstructure:"iop_secret" docs:";The IOP secret used to connect to the wopiserver."` - WopiURL string `mapstructure:"wopi_url" docs:";The wopiserver's URL."` - AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` - AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` - AppURL string `mapstructure:"app_url" docs:";The App URL."` - AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` - AppAPIKey string `mapstructure:"app_api_key" docs:";The API key used by the app, if applicable."` - JWTSecret string `mapstructure:"jwt_secret" docs:";The JWT secret to be used to retrieve the token TTL."` - CustomMimeTypesJSON string `mapstructure:"custom_mime_types_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` - AppDesktopOnly bool `mapstructure:"app_desktop_only" docs:"false;Specifies if the app can be opened only on desktop."` - InsecureConnections bool `mapstructure:"insecure_connections"` + MimeTypes []string `mapstructure:"mime_types" docs:";Inherited from the appprovider."` + IOPSecret string `mapstructure:"iop_secret" docs:";The IOP secret used to connect to the wopiserver."` + WopiURL string `mapstructure:"wopi_url" docs:";The wopiserver's URL."` + AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` + AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` + AppURL string `mapstructure:"app_url" docs:";The App URL."` + AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` + AppAPIKey string `mapstructure:"app_api_key" docs:";The API key used by the app, if applicable."` + JWTSecret string `mapstructure:"jwt_secret" docs:";The JWT secret to be used to retrieve the token TTL."` + AppDesktopOnly bool `mapstructure:"app_desktop_only" docs:"false;Specifies if the app can be opened only on desktop."` + InsecureConnections bool `mapstructure:"insecure_connections"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -112,12 +112,6 @@ func New(m map[string]interface{}) (app.Provider, error) { return http.ErrUseLastResponse } - // read and register custom mime types if configured - err = registerMimeTypes(c.CustomMimeTypesJSON) - if err != nil { - return nil, err - } - return &wopiProvider{ conf: c, wopiClient: wopiClient, @@ -203,11 +197,11 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc httpReq.Header.Set("Authorization", "Bearer "+p.conf.IOPSecret) httpReq.Header.Set("TokenHeader", token) - log.Debug().Str("url", httpReq.URL.String()).Msg("Sending request to wopi server") + log.Debug().Str("url", httpReq.URL.String()).Msg("Sending request to wopiserver") // Call the WOPI server and parse the response (body will always contain a payload) openRes, err := p.wopiClient.Do(httpReq) if err != nil { - return nil, errors.Wrap(err, "wopi: error performing open request to WOPI server") + return nil, errors.Wrap(err, "wopi: error performing open request to wopiserver") } defer openRes.Body.Close() @@ -221,7 +215,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc if body != nil { sbody = string(body) } - log.Warn().Msg(fmt.Sprintf("wopi: WOPI server returned HTTP %s to request %s, error was: %s", openRes.Status, httpReq.URL.String(), sbody)) + log.Warn().Str("status", openRes.Status).Str("error", sbody).Msg("wopi: wopiserver returned error") return nil, errors.New(sbody) } @@ -251,7 +245,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc appFullURL = url.String() } - // Depending on whether wopi server returned any form parameters or not, + // Depending on whether the WOPI server returned any form parameters or not, // we decide whether the request method is POST or GET var formParams map[string]string method := "GET" @@ -265,7 +259,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc } } - log.Info().Msg(fmt.Sprintf("wopi: returning app URL %s", appFullURL)) + log.Info().Str("url", appFullURL).Str("resource", resource.Path).Msg("wopi: returning URL for file") return &appprovider.OpenInAppURL{ AppUrl: appFullURL, Method: method, @@ -296,27 +290,6 @@ func (p *wopiProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.Pro }, nil } -func registerMimeTypes(mappingFile string) error { - // TODO(lopresti) this function also exists in the storage provider, to be seen if we want to factor it out, though a - // fileext <-> mimetype "service" would have to be served by the gateway for it to be accessible both by storage providers and app providers. - if mappingFile != "" { - f, err := ioutil.ReadFile(mappingFile) - if err != nil { - return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err) - } - mimeTypes := map[string]string{} - err = json.Unmarshal(f, &mimeTypes) - if err != nil { - return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err) - } - // register all mime types that were read - for e, m := range mimeTypes { - mime.RegisterMime(e, m) - } - } - return nil -} - func getAppURLs(c *config) (map[string]map[string]string, error) { // Initialize WOPI URLs by discovery httpcl := rhttp.GetHTTPClient( @@ -367,18 +340,22 @@ func getAppURLs(c *config) (map[string]map[string]string, error) { // scrape app's home page to find the appname if !strings.Contains(buf.String(), c.AppName) { - return nil, errors.New("Application server at " + c.AppURL + " does not match this AppProvider for " + c.AppName) + return nil, fmt.Errorf("wopi: application server at %s does not match this AppProvider for %s", c.AppURL, c.AppName) } - // register the supported mimetypes in the AppRegistry: this is hardcoded for the time being - // TODO(lopresti) move to config - switch c.AppName { - case "CodiMD": - appURLs = getCodimdExtensions(c.AppURL) - case "Etherpad": - appURLs = getEtherpadExtensions(c.AppURL) - default: - return nil, errors.New("Application server " + c.AppName + " running at " + c.AppURL + " is unsupported") + // TODO(lopresti) we don't know if the app is not supported/configured in WOPI + // return nil, errors.New("Application server " + c.AppName + " running at " + c.AppURL + " is unsupported") + + // generate the map of supported extensions + appURLs = make(map[string]map[string]string) + appURLs["view"] = make(map[string]string) + appURLs["edit"] = make(map[string]string) + for _, m := range c.MimeTypes { + exts := mime.GetFileExts(m) + for _, e := range exts { + appURLs["view"]["."+e] = c.AppURL + appURLs["edit"]["."+e] = c.AppURL + } } } return appURLs, nil @@ -394,7 +371,7 @@ func (p *wopiProvider) getAccessTokenTTL(ctx context.Context) (string, error) { } if claims, ok := token.Claims.(*jwt.StandardClaims); ok && token.Valid { - // milliseconds since Jan 1, 1970 UTC as required in https://wopi.readthedocs.io/projects/wopirest/en/latest/concepts.html?highlight=access_token_ttl#term-access-token-ttl + // milliseconds since Jan 1, 1970 UTC as required in https://learn.microsoft.com/en-us/microsoft-365/cloud-storage-partner-program/rest/concepts#the-access_token_ttl-property return strconv.FormatInt(claims.ExpiresAt*1000, 10), nil } @@ -409,6 +386,9 @@ func parseWopiDiscovery(body io.Reader) (map[string]map[string]string, error) { return nil, err } root := doc.SelectElement("wopi-discovery") + if root == nil { + return nil, errors.New("wopi-discovery response malformed") + } for _, netzone := range root.SelectElements("net-zone") { @@ -453,24 +433,3 @@ func parseWopiDiscovery(body io.Reader) (map[string]map[string]string, error) { } return appURLs, nil } - -func getCodimdExtensions(appURL string) map[string]map[string]string { - // Register custom mime types - mime.RegisterMime(".zmd", "application/compressed-markdown") - - appURLs := make(map[string]map[string]string) - appURLs["edit"] = map[string]string{ - ".txt": appURL, - ".md": appURL, - ".zmd": appURL, - } - return appURLs -} - -func getEtherpadExtensions(appURL string) map[string]map[string]string { - appURLs := make(map[string]map[string]string) - appURLs["edit"] = map[string]string{ - ".epd": appURL, - } - return appURLs -} diff --git a/pkg/mime/mime.go b/pkg/mime/mime.go index 6908450308..50e9a3953f 100644 --- a/pkg/mime/mime.go +++ b/pkg/mime/mime.go @@ -20,9 +20,10 @@ package mime import ( "path" + "strings" "sync" - gomime "github.com/cubewise-code/go-mime" + gomime "github.com/glpatcern/go-mime" // hopefully temporary ) const defaultMimeDir = "httpd/unix-directory" @@ -47,11 +48,15 @@ func Detect(isDir bool, fn string) string { } ext := path.Ext(fn) + ext = strings.TrimPrefix(ext, ".") mimeType := getCustomMime(ext) if mimeType == "" { mimeType = gomime.TypeByExtension(ext) + if mimeType != "" { + mimes.Store(ext, mimeType) + } } if mimeType == "" { @@ -61,6 +66,24 @@ func Detect(isDir bool, fn string) string { return mimeType } +// GetFileExts performs the inverse resolution from mimetype to file extensions +func GetFileExts(mime string) []string { + var found []string + // first look in our cache + mimes.Range(func(e, m interface{}) bool { + if m.(string) == mime { + found = append(found, e.(string)) + } + return true + }) + if len(found) > 0 { + return found + } + + // then use the gomime package + return gomime.ExtensionsByType(mime) +} + func getCustomMime(ext string) string { if m, ok := mimes.Load(ext); ok { return m.(string)