From 0224d94d9a8d26c781d12cdd98909d3adf3d6025 Mon Sep 17 00:00:00 2001 From: Aleksandr Dubovikov Date: Wed, 9 Oct 2024 20:35:38 +0200 Subject: [PATCH] url_preview tests Signed-off-by: Aleksandr Dubovikov --- mediaapi/routing/url_preview.go | 17 +- mediaapi/routing/url_preview_test.go | 231 +++++++++++++++++++++++++++ setup/config/config_test.go | 2 +- 3 files changed, 244 insertions(+), 6 deletions(-) diff --git a/mediaapi/routing/url_preview.go b/mediaapi/routing/url_preview.go index 75150f440e..4723746bf2 100644 --- a/mediaapi/routing/url_preview.go +++ b/mediaapi/routing/url_preview.go @@ -88,11 +88,9 @@ func makeUrlPreviewHandler( } // Check if the url is in the blacklist - for _, pattern := range urlBlackList { - if pattern.MatchString(pUrl) { - logger.WithField("pattern", pattern.String()).Warn("the url is blacklisted") - return util.ErrorResponse(ErrorBlackListed) - } + if checkURLBlacklisted(urlBlackList, pUrl) { + logger.Debug("The url is in the blacklist") + return util.ErrorResponse(ErrorBlackListed) } urlParsed, perr := url.Parse(pUrl) @@ -668,5 +666,14 @@ func createUrlBlackList(cfg *config.MediaAPI) []*regexp.Regexp { blackList[i] = regexp.MustCompile(pattern) } return blackList +} +func checkURLBlacklisted(blacklist []*regexp.Regexp, url string) bool { + // Check if the url is in the blacklist + for _, pattern := range blacklist { + if pattern.MatchString(url) { + return true + } + } + return false } diff --git a/mediaapi/routing/url_preview_test.go b/mediaapi/routing/url_preview_test.go index f9def31ff6..08dc4da596 100644 --- a/mediaapi/routing/url_preview_test.go +++ b/mediaapi/routing/url_preview_test.go @@ -1,16 +1,22 @@ package routing import ( + "bytes" "context" "fmt" "io" "net/http" + "net/http/httptest" + "net/url" "os" "path/filepath" "reflect" "strings" + "sync" "testing" + "time" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" @@ -18,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) var tests = []map[string]interface{}{ @@ -151,3 +158,227 @@ func Test_LoadStorePreview(t *testing.T) { t.Errorf("Stored and loaded previews not equal: stored=%v, loaded=%v", testPreview, loadedPreview) } } + +func Test_Blacklist(t *testing.T) { + + tests := map[string]interface{}{ + "entrys": []string{ + "drive.google.com", + "https?://altavista.com/someurl", + "https?://(www.)?google.com", + "http://stackoverflow.com", + }, + "tests": map[string]bool{ + "https://drive.google.com/path": true, + "http://altavista.com": false, + "http://altavista.com/someurl": true, + "https://altavista.com/someurl": true, + "https://stackoverflow.com": false, + }, + } + + cfg := &config.MediaAPI{ + UrlPreviewBlacklist: tests["entrys"].([]string), + } + blacklist := createUrlBlackList(cfg) + + for url, expected := range tests["tests"].(map[string]bool) { + value := checkURLBlacklisted(blacklist, url) + if value != expected { + t.Errorf("Blacklist %v: expected=%v, got=%v", url, expected, value) + } + } +} + +func Test_ActiveRequestWaiting(t *testing.T) { + activeRequests := &types.ActiveUrlPreviewRequests{ + Url: map[string]*types.UrlPreviewResult{ + "someurl": &types.UrlPreviewResult{ + Cond: sync.NewCond(&sync.Mutex{}), + Preview: &types.UrlPreview{}, + Error: nil, + }, + }, + } + + successResults := 0 + + for i := 0; i < 3; i++ { + go func() { + if res, ok := checkActivePreviewResponse(activeRequests, "someurl"); ok { + if res.Code != 200 { + t.Errorf("Unsuccess result: %v", res) + } + successResults++ + return + } + t.Errorf("url %v not found in active requests", "someurl") + }() + } + + time.Sleep(time.Duration(1) * time.Second) + if successResults != 0 { + t.Error("Subroutines didn't wait") + } + activeRequests.Url["someurl"].Cond.Broadcast() + to := time.After(1 * time.Second) + for { + select { + case <-to: + t.Errorf("Test timed out, results=%v", successResults) + return + default: + } + if successResults == 3 { + break + } + } +} + +func Test_UrlPreviewHandler(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Errorf("failed to get current working directory: %v", err) + } + + maxSize := config.FileSizeBytes(1024 * 1024) + logger := log.New().WithField("mediaapi", "test") + logger.Debug("some") + testdataPath := filepath.Join(wd, "./testdata") + + g := &config.Global{} + g.Defaults(config.DefaultOpts{Generate: true}) + cfg := &config.MediaAPI{ + Matrix: g, + MaxFileSizeBytes: maxSize, + BasePath: config.Path(testdataPath), + AbsBasePath: config.Path(testdataPath), + DynamicThumbnails: false, + } + cfg2 := &config.MediaAPI{ + Matrix: g, + MaxFileSizeBytes: maxSize, + BasePath: config.Path(testdataPath), + AbsBasePath: config.Path(testdataPath), + UrlPreviewThumbnailSize: config.ThumbnailSize{ + Width: 10, + Height: 10, + }, + MaxThumbnailGenerators: 10, + DynamicThumbnails: false, + } + + // create testdata folder and remove when done + _ = os.Mkdir(testdataPath, os.ModePerm) + defer fileutils.RemoveDir(types.Path(testdataPath), nil) + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) + db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ + ConnectionString: "file::memory:?cache=shared", + MaxOpenConnections: 100, + MaxIdleConnections: 2, + ConnMaxLifetimeSeconds: -1, + }) + if err != nil { + t.Errorf("error opening mediaapi database: %v", err) + } + db2, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ + ConnectionString: "file::memory:", + MaxOpenConnections: 100, + MaxIdleConnections: 2, + ConnMaxLifetimeSeconds: -1, + }) + if err != nil { + t.Errorf("error opening mediaapi database: %v", err) + } + + activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ + PathToResult: map[string]*types.ThumbnailGenerationResult{}, + } + rateLimits := &httputil.RateLimits{} + device := userapi.Device{ + ID: "1", + UserID: "user", + } + + handler := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration) + // this handler is to test filecache + handler2 := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration) + // this handler is to test image resize + handler3 := makeUrlPreviewHandler(cfg2, rateLimits, db2, activeThumbnailGeneration) + + data := bytes.Buffer{} + responseBody := ` + + Title + + + + + + + ` + data.WriteString(responseBody) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/test.png" || r.RequestURI == "/test2.png" { + w.Header().Add("Content-Type", "image/jpeg") + http.ServeFile(w, r, "../bimg-96x96-crop.jpg") + return + } + w.Write([]byte(responseBody)) + })) + + ur, _ := url.Parse("/?url=" + srv.URL) + req := &http.Request{ + Method: "GET", + URL: ur, + } + result := handler(req, &device) + assert.Equal(t, result.Code, 200, "Response code mismatch") + assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title") + assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") + assert.Greater(t, result.JSON.(*types.UrlPreview).ImageSize, types.FileSizeBytes(0), "Image size missmatch") + + // Test only image response + ur2, _ := url.Parse("/?url=" + srv.URL + "/test.png") + result = handler(&http.Request{ + Method: "GET", + URL: ur2, + }, &device) + assert.Equal(t, result.Code, 200, "Response code mismatch") + assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "") + assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") + assert.Greater(t, result.JSON.(*types.UrlPreview).ImageHeight, int(0), "height missmatch") + assert.Greater(t, result.JSON.(*types.UrlPreview).ImageWidth, int(0), "width missmatch") + + srcSize := result.JSON.(*types.UrlPreview).ImageSize + srcHeight := result.JSON.(*types.UrlPreview).ImageHeight + srcWidth := result.JSON.(*types.UrlPreview).ImageWidth + + // Test image resize + ur3, _ := url.Parse("/?url=" + srv.URL + "/test2.png") + result = handler3(&http.Request{ + Method: "GET", + URL: ur3, + }, &device) + assert.Equal(t, result.Code, 200, "Response code mismatch") + assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") + assert.Less(t, result.JSON.(*types.UrlPreview).ImageSize, srcSize, "thumbnail file size missmatch") + assert.Less(t, result.JSON.(*types.UrlPreview).ImageHeight, srcHeight, "thumbnail height missmatch") + assert.Less(t, result.JSON.(*types.UrlPreview).ImageWidth, srcWidth, "thumbnail width missmatch") + + srv.Close() + + // Test in-memory cache + result = handler(req, &device) + assert.Equal(t, result.Code, 200, "Response code mismatch") + assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title") + assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") + + // Test response file cache + result = handler2(req, &device) + assert.Equal(t, result.Code, 200, "Response code mismatch") + assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title") + assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") + +} diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 14f4fffc48..a6f7590214 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -327,7 +327,7 @@ func Test_MediaAPIConfigVerify(t *testing.T) { config := &MediaAPI{ Matrix: &Global{DatabaseOptions: DatabaseOptions{}}, Database: DatabaseOptions{}, - MaxFileSizeBytes: FileSizeBytes(9223372036854775807), + MaxFileSizeBytes: FileSizeBytes(^int64(0)), } configErrs := &ConfigErrors{}