From 512c82201fd6c49b9cd3e7670219b8a3599341e5 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Fri, 3 Jan 2025 23:07:12 +0000 Subject: [PATCH 01/22] feat: better tests --- .github/workflows/go-test.yml | 2 + cmd/server/main.go | 2 + cmd/worker/main.go | 81 +----- go.mod | 21 ++ go.sum | 136 ++++++++++ internal/admin/admin_test.go | 23 +- internal/archive/archive_test.go | 383 +++++++++++++++++++++++++++ internal/channel/channel_test.go | 206 ++++++++++---- internal/config/env_test.go | 2 + internal/server/server.go | 5 + internal/transport/http/auth_test.go | 129 +++++++++ internal/transport/http/handler.go | 8 +- internal/worker/worker.go | 107 ++++++++ tests/setup.go | 213 ++++++++++++--- 14 files changed, 1146 insertions(+), 172 deletions(-) create mode 100644 internal/archive/archive_test.go create mode 100644 internal/transport/http/auth_test.go create mode 100644 internal/worker/worker.go diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 696317d..da59805 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -9,6 +9,8 @@ on: jobs: test: runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:dev steps: - name: Checkout repository diff --git a/cmd/server/main.go b/cmd/server/main.go index 45e8eb3..d9615e0 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,7 +17,9 @@ func main() { if os.Getenv("DEVELOPMENT") == "true" { log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) } + log.Info().Str("commit", utils.Commit).Str("build_time", utils.BuildTime).Msg("starting server") + if err := server.Run(ctx); err != nil { log.Fatal().Err(err).Msg("failed to run") } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 879e0b1..5480395 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -2,102 +2,29 @@ package main import ( "context" - "fmt" "os" "os/signal" "syscall" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/zibbp/ganymede/internal/archive" - "github.com/zibbp/ganymede/internal/blocked" - "github.com/zibbp/ganymede/internal/channel" - "github.com/zibbp/ganymede/internal/config" - serverConfig "github.com/zibbp/ganymede/internal/config" - "github.com/zibbp/ganymede/internal/database" - "github.com/zibbp/ganymede/internal/live" - "github.com/zibbp/ganymede/internal/platform" - "github.com/zibbp/ganymede/internal/queue" - tasks_client "github.com/zibbp/ganymede/internal/tasks/client" - tasks_worker "github.com/zibbp/ganymede/internal/tasks/worker" "github.com/zibbp/ganymede/internal/utils" - "github.com/zibbp/ganymede/internal/vod" + "github.com/zibbp/ganymede/internal/worker" ) func main() { ctx := context.Background() - envConfig := config.GetEnvConfig() - envAppConfig := config.GetEnvApplicationConfig() - _, err := serverConfig.Init() - if err != nil { - log.Panic().Err(err).Msg("Error initializing server config") - } - if os.Getenv("DEVELOPMENT") == "true" { log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) } log.Info().Str("commit", utils.Commit).Str("build_time", utils.BuildTime).Msg("starting worker") - dbString := fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s sslrootcert=%s", envAppConfig.DB_USER, envAppConfig.DB_PASS, envAppConfig.DB_HOST, envAppConfig.DB_PORT, envAppConfig.DB_NAME, envAppConfig.DB_SSL, envAppConfig.DB_SSL_ROOT_CERT) - - db := database.NewDatabase(ctx, database.DatabaseConnectionInput{ - DBString: dbString, - IsWorker: false, - }) - - riverClient, err := tasks_client.NewRiverClient(tasks_client.RiverClientInput{ - DB_URL: dbString, - }) - if err != nil { - log.Panic().Err(err).Msg("Error creating river worker") - } - - var platformTwitch platform.Platform - // setup twitch platform - if envConfig.TwitchClientId != "" && envConfig.TwitchClientSecret != "" { - platformTwitch = &platform.TwitchConnection{ - ClientId: envConfig.TwitchClientId, - ClientSecret: envConfig.TwitchClientSecret, - } - _, err = platformTwitch.Authenticate(ctx) - if err != nil { - log.Panic().Err(err).Msg("Error authenticating to Twitch") - } - } - - channelService := channel.NewService(db, platformTwitch) - vodService := vod.NewService(db, riverClient, platformTwitch) - queueService := queue.NewService(db, vodService, channelService, riverClient) - blockedVodsService := blocked.NewService(db) - // twitchService := twitch.NewService() - archiveService := archive.NewService(db, channelService, vodService, queueService, blockedVodsService, riverClient, platformTwitch) - liveService := live.NewService(db, archiveService, platformTwitch) - - // initialize river - riverWorkerClient, err := tasks_worker.NewRiverWorker(tasks_worker.RiverWorkerInput{ - DB_URL: dbString, - DB: db, - PlatformTwitch: platformTwitch, - VideoDownloadWorkers: envConfig.MaxVideoDownloadExecutions, - VideoPostProcessWorkers: envConfig.MaxVideoConvertExecutions, - ChatDownloadWorkers: envConfig.MaxChatDownloadExecutions, - ChatRenderWorkers: envConfig.MaxChatRenderExecutions, - SpriteThumbnailWorkers: envConfig.MaxVideoSpriteThumbnailExecutions, - }) + // Set up the worker + riverWorkerClient, err := worker.SetupWorker(ctx) if err != nil { - log.Panic().Err(err).Msg("Error creating river worker") - } - - // get periodic tasks - periodicTasks, err := riverWorkerClient.GetPeriodicTasks(liveService) - if err != nil { - log.Panic().Err(err).Msg("Error getting periodic tasks") - } - - for _, task := range periodicTasks { - riverWorkerClient.Client.PeriodicJobs().Add(task) + log.Panic().Err(err).Msg("Error setting up river worker") } // start worker in a goroutine diff --git a/go.mod b/go.mod index 65d3bed..ba49544 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,9 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -46,8 +49,11 @@ require ( github.com/docker/docker v27.4.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/gavv/httpexpect/v2 v2.16.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -56,11 +62,16 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/imkira/go-interpol v1.1.0 // indirect github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect @@ -79,6 +90,8 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/riverqueue/river/riverdriver v0.14.3 // indirect github.com/riverqueue/river/rivershared v0.14.3 // indirect + github.com/sanity-io/litter v1.5.5 // indirect + github.com/sergi/go-diff v1.0.0 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -89,6 +102,13 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.9.0 // indirect + github.com/valyala/fasthttp v1.34.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect + github.com/yudai/gojsondiff v1.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect @@ -101,6 +121,7 @@ require ( golang.org/x/sync v0.10.0 // indirect golang.org/x/tools v0.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + moul.io/http2curl/v2 v2.3.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 5fa8f33..5ac6719 100644 --- a/go.sum +++ b/go.sum @@ -14,12 +14,24 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexedwards/scs/pgxstore v0.0.0-20240316134038-7e11d57e8885 h1:I5Z6bSLjKuh99H9JLN35Ep9+GOYp2Cg0Jy+HhykoQf8= github.com/alexedwards/scs/pgxstore v0.0.0-20240316134038-7e11d57e8885/go.mod h1:hwveArYcjyOK66EViVgVU5Iqj7zyEsWjKXMQhDJrTLI= github.com/alexedwards/scs/v2 v2.8.0 h1:h31yUYoycPuL0zt14c0gd+oqxfRwIj6SOjHdKRZxhEw= github.com/alexedwards/scs/v2 v2.8.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -44,6 +56,7 @@ github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -56,10 +69,24 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= +github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= @@ -92,19 +119,68 @@ github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/hcl/v2 v2.23.0 h1:Fphj1/gCylPxHutVSEOf2fBOh1VE4AuLV7+kbJf3qos= github.com/hashicorp/hcl/v2 v2.23.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -118,10 +194,16 @@ github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsb github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -181,8 +263,14 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -216,6 +304,14 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= @@ -231,6 +327,8 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -244,6 +342,7 @@ github.com/swaggo/files/v2 v2.0.1 h1:XCVJO/i/VosCDsJu1YLpdejGsGnBE9deRMpjN4pJLHk github.com/swaggo/files/v2 v2.0.1/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM= github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0 h1:c51aBXT3v2HEBVarmaBnsKzvgZjC5amn0qsj8Naqi50= @@ -264,8 +363,24 @@ github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPD github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= +github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -299,11 +414,14 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= @@ -313,6 +431,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -333,8 +455,17 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -367,6 +498,7 @@ golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -396,5 +528,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= riverqueue.com/riverui v0.7.0 h1:K1maQJtgtL1FhE7vOFc3T56JHrKcibQBx3vhVbKc+pk= riverqueue.com/riverui v0.7.0/go.mod h1:5bpSwSmb3zQbvvRspuWn28LY58cds7NFP9r8yJ5pAfU= diff --git a/internal/admin/admin_test.go b/internal/admin/admin_test.go index ff0b3ab..cfb8e5e 100644 --- a/internal/admin/admin_test.go +++ b/internal/admin/admin_test.go @@ -6,16 +6,28 @@ import ( "github.com/stretchr/testify/assert" "github.com/zibbp/ganymede/internal/channel" + "github.com/zibbp/ganymede/internal/server" "github.com/zibbp/ganymede/tests" ) -func TestGetStats(t *testing.T) { - ctx := context.Background() +type AdminTest struct { + App *server.Application +} + +// TestAdmin tests the admin service. This function runs all the tests to avoid spinning up multiple containers. +func TestAdmin(t *testing.T) { app, err := tests.Setup(t) assert.NoError(t, err) - // create a channel for to test\ - _, err = app.ChannelService.CreateChannel(channel.Channel{ + adminTest := AdminTest{App: app} + + t.Run("TestGetStats", adminTest.GetStatsTest) + +} + +// GetStatsTest tests the GetStats function +func (s *AdminTest) GetStatsTest(t *testing.T) { + _, err := s.App.ChannelService.CreateChannel(channel.Channel{ ExtID: "123456789", Name: "test_channel", DisplayName: "Test Channel", @@ -23,8 +35,7 @@ func TestGetStats(t *testing.T) { }) assert.NoError(t, err) - // test GetStats - stats, err := app.AdminService.GetStats(ctx) + stats, err := s.App.AdminService.GetStats(context.Background()) assert.NoError(t, err) assert.Equal(t, 0, stats.VodCount) assert.Equal(t, 1, stats.ChannelCount) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..21925ee --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,383 @@ +package archive_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/zibbp/ganymede/ent/queue" + "github.com/zibbp/ganymede/ent/vod" + "github.com/zibbp/ganymede/internal/archive" + "github.com/zibbp/ganymede/internal/config" + "github.com/zibbp/ganymede/internal/server" + "github.com/zibbp/ganymede/internal/utils" + "github.com/zibbp/ganymede/tests" +) + +type ArchiveTest struct { + App *server.Application +} + +var ( + TestTwitchChannelName = "sodapoppin" + TestTwitchChannelDisplayName = "sodapoppin" + TestTwitchChannelExtId = "26301881" + TestTwitchVideoId = "1989753443" + TestArchiveTimeout = 300 * time.Second +) + +// TestAdmin tests the admin service. This function runs all the tests to avoid spinning up multiple containers. +func TestArchive(t *testing.T) { + app, err := tests.Setup(t) + assert.NoError(t, err) + + archiveTest := ArchiveTest{App: app} + + t.Run("TestArchiveChannel", archiveTest.ArchiveChannelTest) + +} + +// ArchiveChannelTest tests the ArchiveChannel function +func (s *ArchiveTest) ArchiveChannelTest(t *testing.T) { + archivedPlatformChannel, err := s.App.ArchiveService.ArchiveChannel(context.Background(), TestTwitchChannelName) + assert.NoError(t, err) + + assert.Equal(t, TestTwitchChannelName, archivedPlatformChannel.Name) + assert.Equal(t, TestTwitchChannelDisplayName, archivedPlatformChannel.DisplayName) + assert.Equal(t, TestTwitchChannelExtId, archivedPlatformChannel.ExtID) + + // Check if profile image was download + assert.FileExists(t, archivedPlatformChannel.ImagePath) + + // Check if profile image is not empty + fileInfo, err := os.Stat(archivedPlatformChannel.ImagePath) + assert.NoError(t, err) + assert.NotEqual(t, 0, fileInfo.Size()) +} + +// ArchiveVideo tests the full archive process for a video with chat downloading, processing, and rendering +func TestArchiveVideo(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Archive the video + err = app.ArchiveService.ArchiveVideo(context.Background(), archive.ArchiveVideoInput{ + VideoId: TestTwitchVideoId, + Quality: utils.R720P60, + ArchiveChat: true, + RenderChat: true, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchVideoId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, true, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, true, q.RenderChat) + assert.Equal(t, true, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Pending, q.TaskChatDownload) + assert.Equal(t, utils.Pending, q.TaskChatRender) + assert.Equal(t, utils.Pending, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.FileExists(t, v.VideoPath) + assert.FileExists(t, v.ChatPath) + assert.FileExists(t, v.ChatVideoPath) +} + +// ArchiveVideo tests the full archive process for a video without chat downloading, processing, and rendering +func TestArchiveVideoNoChat(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Archive the video + err = app.ArchiveService.ArchiveVideo(context.Background(), archive.ArchiveVideoInput{ + VideoId: TestTwitchVideoId, + Quality: utils.R720P60, + ArchiveChat: false, + RenderChat: false, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchVideoId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, false, q.RenderChat) + assert.Equal(t, false, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.FileExists(t, v.VideoPath) + assert.NoFileExists(t, v.ChatPath) + assert.NoFileExists(t, v.ChatVideoPath) +} + +// ArchiveVideo tests the full archive process for a video without chat rendering +func TestArchiveVideoNoChatRender(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Archive the video + err = app.ArchiveService.ArchiveVideo(context.Background(), archive.ArchiveVideoInput{ + VideoId: TestTwitchVideoId, + Quality: utils.R720P60, + ArchiveChat: true, + RenderChat: false, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchVideoId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, true, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, false, q.RenderChat) + assert.Equal(t, true, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Pending, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Pending, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.FileExists(t, v.VideoPath) + assert.FileExists(t, v.ChatPath) + assert.NoFileExists(t, v.ChatVideoPath) +} + +// TestArchiveVideoHLS tests the full archive process for a video without chat downloading, processing, and rendering converting to HLS +func TestArchiveVideoHLS(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Update config to save as HLS + c := config.Get() + c.Archive.SaveAsHls = true + assert.NoError(t, config.UpdateConfig(c)) + + // Archive the video + err = app.ArchiveService.ArchiveVideo(context.Background(), archive.ArchiveVideoInput{ + VideoId: TestTwitchVideoId, + Quality: utils.R720P60, + ArchiveChat: false, + RenderChat: false, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchVideoId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.NotNil(t, v.TmpVideoHlsPath) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, false, q.RenderChat) + assert.Equal(t, false, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.NoFileExists(t, v.ChatPath) + assert.NoFileExists(t, v.ChatVideoPath) + + assert.DirExists(t, v.VideoHlsPath) + + // Assert number of files in HLS directory is greater than 0 + files, err := os.ReadDir(v.VideoHlsPath) + assert.NoError(t, err) + assert.Greater(t, len(files), 0) +} diff --git a/internal/channel/channel_test.go b/internal/channel/channel_test.go index 4a2d1cc..ea13588 100644 --- a/internal/channel/channel_test.go +++ b/internal/channel/channel_test.go @@ -1,51 +1,143 @@ package channel_test import ( - "context" - "errors" - "os" "testing" "github.com/stretchr/testify/assert" "github.com/zibbp/ganymede/internal/channel" + "github.com/zibbp/ganymede/internal/server" "github.com/zibbp/ganymede/tests" ) -func TestChannelCRUD(t *testing.T) { +type ChannelTest struct { + App *server.Application +} + +var ( + TestChannelName = "test_channel" + TestChannelExtID = "123456789" + TestChannelDisplayName = "Test Channel" + TestChannelImagePath = "/vods/test_channel/test_channel.jpg" +) + +// TestChannel tests the channel service. This function runs all the tests to avoid spinning up multiple containers. +func TestChannel(t *testing.T) { app, err := tests.Setup(t) assert.NoError(t, err) - // test CreateChannel - chann, err := app.ChannelService.CreateChannel(channel.Channel{ - ExtID: "123456789", - Name: "test_channel", - DisplayName: "Test Channel", - ImagePath: "/vods/test_channel/test_channel.jpg", + channelTest := ChannelTest{App: app} + + t.Run("TestCreateChannel", channelTest.CreateChannelTest) + t.Run("TestCreateChannelInvalid", channelTest.CreateChannelInvalidTest) + t.Run("TestGetChannels", channelTest.GetChannelsTest) + t.Run("TestGetChannel", channelTest.GetChannelTest) + t.Run("TestGetChannelByName", channelTest.GetChannelByNameTest) + t.Run("TestGetChannelByExtId", channelTest.GetChannelByExtIdTest) + t.Run("TestDeleteChannel", channelTest.DeleteChannelTest) + t.Run("TestUpdateChannel", channelTest.UpdateChannelTest) + t.Run("TestCheckChannelExists", channelTest.CheckChannelExistsTest) + t.Run("TestCheckChannelExistsByExtId", channelTest.CheckChannelExistsByExtIdTest) + +} + +// CreateChannelTest tests the CreateChannel function +func (s *ChannelTest) CreateChannelTest(t *testing.T) { + channel, err := s.App.ChannelService.CreateChannel(channel.Channel{ + ExtID: TestChannelExtID, + Name: TestChannelName, + DisplayName: TestChannelDisplayName, + ImagePath: TestChannelImagePath, }) + assert.NoError(t, err) + assert.Equal(t, "123456789", channel.ExtID) + assert.Equal(t, "test_channel", channel.Name) + assert.Equal(t, "Test Channel", channel.DisplayName) + assert.Equal(t, "/vods/test_channel/test_channel.jpg", channel.ImagePath) +} +// CreateChannelInvalid tests the CreateChannel function with invalid data +func (s *ChannelTest) CreateChannelInvalidTest(t *testing.T) { + _, err := s.App.ChannelService.CreateChannel(channel.Channel{ + ExtID: "123456789", // duplicate of the previous test + Name: "duplicate channel", + DisplayName: "Duplicate Channel", + ImagePath: "/vods/test_channel/duplicate_channel.jpg", + }) + assert.Error(t, err) +} + +// GetChannelsTest tests the GetChannels function +func (s *ChannelTest) GetChannelsTest(t *testing.T) { + channels, err := s.App.ChannelService.GetChannels() assert.NoError(t, err) - assert.Equal(t, "123456789", chann.ExtID) - assert.Equal(t, "test_channel", chann.Name) - assert.Equal(t, "Test Channel", chann.DisplayName) - assert.Equal(t, "/vods/test_channel/test_channel.jpg", chann.ImagePath) + assert.Equal(t, 1, len(channels)) +} - // test GetChannel - getChannel, err := app.ChannelService.GetChannel(chann.ID) +// GetChannelTest tests the GetChannel function +func (s *ChannelTest) GetChannelTest(t *testing.T) { + channel, err := s.App.ChannelService.CreateChannel(channel.Channel{ + ExtID: "12345", + Name: "get_channel_test", + DisplayName: "Get Channel Test", + ImagePath: "/vods/get_channel_test/get_channel_test.jpg", + }) assert.NoError(t, err) - assert.Equal(t, chann.ID, getChannel.ID) - // test GetChannelByName - getChannelByName, err := app.ChannelService.GetChannelByName(chann.Name) + getChannel, err := s.App.ChannelService.GetChannel(channel.ID) assert.NoError(t, err) - assert.Equal(t, chann.ID, getChannelByName.ID) + assert.Equal(t, channel.ID, getChannel.ID) + assert.Equal(t, "get_channel_test", getChannel.Name) + assert.Equal(t, "Get Channel Test", getChannel.DisplayName) + assert.Equal(t, "/vods/get_channel_test/get_channel_test.jpg", getChannel.ImagePath) +} - // test GetChannels - channels, err := app.ChannelService.GetChannels() +// GetChannelByNameTest tests the GetChannelByName function +func (s *ChannelTest) GetChannelByNameTest(t *testing.T) { + channel, err := s.App.ChannelService.GetChannelByName(TestChannelName) assert.NoError(t, err) - assert.Equal(t, 1, len(channels)) + assert.Equal(t, TestChannelName, channel.Name) + assert.Equal(t, TestChannelExtID, channel.ExtID) + assert.Equal(t, TestChannelDisplayName, channel.DisplayName) + assert.Equal(t, TestChannelImagePath, channel.ImagePath) +} - // test UpdateChannel - updatedChannel, err := app.ChannelService.UpdateChannel(chann.ID, channel.Channel{ +// GetChannelByExtIdTest tests the GetChannelByExtId function +func (s *ChannelTest) GetChannelByExtIdTest(t *testing.T) { + channel, err := s.App.ChannelService.GetChannelByExtId(TestChannelExtID) + assert.NoError(t, err) + assert.Equal(t, TestChannelName, channel.Name) + assert.Equal(t, TestChannelExtID, channel.ExtID) + assert.Equal(t, TestChannelDisplayName, channel.DisplayName) + assert.Equal(t, TestChannelImagePath, channel.ImagePath) +} + +// DeleteChannelTest tests the DeleteChannel function +func (s *ChannelTest) DeleteChannelTest(t *testing.T) { + channel, err := s.App.ChannelService.CreateChannel(channel.Channel{ + ExtID: "1234", + Name: "delete_me", + DisplayName: "Delete Me", + ImagePath: "/vods/delete_me/delete_me.jpg", + }) + assert.NoError(t, err) + + err = s.App.ChannelService.DeleteChannel(channel.ID) + assert.NoError(t, err) + _, err = s.App.ChannelService.GetChannel(channel.ID) + assert.Error(t, err) +} + +// UpdateChannelTest tests the UpdateChannel function +func (s *ChannelTest) UpdateChannelTest(t *testing.T) { + createdChannel, err := s.App.ChannelService.CreateChannel(channel.Channel{ + ExtID: "1234", + Name: "update_me", + DisplayName: "Update Me", + ImagePath: "/vods/update_me/update_me.jpg", + }) + assert.NoError(t, err) + + updatedChannel, err := s.App.ChannelService.UpdateChannel(createdChannel.ID, channel.Channel{ Name: "updated_channel", DisplayName: "Updated Channel", ImagePath: "/vods/updated_channel/updated_channel.jpg", @@ -58,39 +150,49 @@ func TestChannelCRUD(t *testing.T) { assert.Equal(t, "/vods/updated_channel/updated_channel.jpg", updatedChannel.ImagePath) assert.Equal(t, true, updatedChannel.Retention) assert.Equal(t, int64(30), updatedChannel.RetentionDays) +} - // test CheckChannelExists - assert.True(t, app.ChannelService.CheckChannelExists(updatedChannel.Name)) +// CheckChannelExistsTest tests the CheckChannelExists function +func (s *ChannelTest) CheckChannelExistsTest(t *testing.T) { + exists := s.App.ChannelService.CheckChannelExists(TestChannelName) + assert.True(t, exists) - // test DeleteChannel - err = app.ChannelService.DeleteChannel(updatedChannel.ID) - assert.NoError(t, err) - assert.False(t, app.ChannelService.CheckChannelExists(updatedChannel.Name)) + exists = s.App.ChannelService.CheckChannelExists("non_existent_channel") + assert.False(t, exists) } -func TestPlatformTwitchChannel(t *testing.T) { - ctx := context.Background() - app, err := tests.Setup(t) - assert.NoError(t, err) +// CheckChannelExistsByExtIdTest tests the CheckChannelExistsByExtId function +func (s *ChannelTest) CheckChannelExistsByExtIdTest(t *testing.T) { + exists := s.App.ChannelService.CheckChannelExistsByExtId(TestChannelExtID) + assert.True(t, exists) - // test ArchiveChannel - chann, err := app.ArchiveService.ArchiveChannel(ctx, "sodapoppin") - assert.NoError(t, err) - assert.Equal(t, "sodapoppin", chann.Name) + exists = s.App.ChannelService.CheckChannelExistsByExtId("123") + assert.False(t, exists) +} - if _, err := os.Stat(chann.ImagePath); errors.Is(err, os.ErrNotExist) { - t.Errorf("image not found: %s", chann.ImagePath) - } +// func TestPlatformTwitchChannel(t *testing.T) { +// ctx := context.Background() +// app, err := tests.Setup(t) +// assert.NoError(t, err) - // remove image - err = os.Remove(chann.ImagePath) - assert.NoError(t, err) +// // test ArchiveChannel +// chann, err := app.ArchiveService.ArchiveChannel(ctx, "sodapoppin") +// assert.NoError(t, err) +// assert.Equal(t, "sodapoppin", chann.Name) - // test UpdateChannelImage - assert.NoError(t, app.ChannelService.UpdateChannelImage(ctx, chann.ID)) +// if _, err := os.Stat(chann.ImagePath); errors.Is(err, os.ErrNotExist) { +// t.Errorf("image not found: %s", chann.ImagePath) +// } - // ensure image exists - if _, err := os.Stat(chann.ImagePath); errors.Is(err, os.ErrNotExist) { - t.Errorf("image not found: %s", chann.ImagePath) - } -} +// // remove image +// err = os.Remove(chann.ImagePath) +// assert.NoError(t, err) + +// // test UpdateChannelImage +// assert.NoError(t, app.ChannelService.UpdateChannelImage(ctx, chann.ID)) + +// // ensure image exists +// if _, err := os.Stat(chann.ImagePath); errors.Is(err, os.ErrNotExist) { +// t.Errorf("image not found: %s", chann.ImagePath) +// } +// } diff --git a/internal/config/env_test.go b/internal/config/env_test.go index 0bcdf5e..c4aee78 100644 --- a/internal/config/env_test.go +++ b/internal/config/env_test.go @@ -9,6 +9,8 @@ import ( func TestGetEnvConfig(t *testing.T) { os.Setenv("VIDEOS_DIR", "/custom/videos") + os.Setenv("TWITCH_CLIENT_ID", "client_id") + os.Setenv("TWITCH_CLIENT_SECRET", "client_secret") env := GetEnvConfig() diff --git a/internal/server/server.go b/internal/server/server.go index 53f96d6..8e09a7d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -72,6 +72,11 @@ func SetupApplication(ctx context.Context) (*Application, error) { zerolog.SetGlobalLevel(zerolog.InfoLevel) } + // Disable logging for tests + if os.Getenv("TESTS_LOGGING") == "false" { + zerolog.SetGlobalLevel(zerolog.Disabled) + } + dbString := fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s sslrootcert=%s", envAppConfig.DB_USER, envAppConfig.DB_PASS, envAppConfig.DB_HOST, envAppConfig.DB_PORT, envAppConfig.DB_NAME, envAppConfig.DB_SSL, envAppConfig.DB_SSL_ROOT_CERT) db := database.NewDatabase(ctx, database.DatabaseConnectionInput{ diff --git a/internal/transport/http/auth_test.go b/internal/transport/http/auth_test.go new file mode 100644 index 0000000..bc6665b --- /dev/null +++ b/internal/transport/http/auth_test.go @@ -0,0 +1,129 @@ +package http_test + +import ( + "net/http" + "testing" + + "github.com/gavv/httpexpect/v2" + "github.com/stretchr/testify/assert" + "github.com/zibbp/ganymede/ent" + internalHttp "github.com/zibbp/ganymede/internal/transport/http" + "github.com/zibbp/ganymede/internal/utils" + "github.com/zibbp/ganymede/tests" +) + +type HttpTestService struct { + E *httpexpect.Expect +} + +// AuthTest runs all the auth tests +func TestAuth(t *testing.T) { + e, err := tests.SetupHTTP(t) + assert.NoError(t, err) + + s := &HttpTestService{E: e} + + t.Run("AuthLoginInvalid", s.AuthLoginInvalid) + t.Run("AuthLogin", s.AuthLogin) + t.Run("AuthGetUser", s.AuthGetUser) + t.Run("AuthChangePasswordInvalid", s.AuthChangePasswordInvalid) + t.Run("AuthChangePasswordNoMatch", s.AuthChangePasswordNoMatch) + t.Run("AuthChangePassword", s.AuthChangePassword) + t.Run("AuthChangePasswordBack", s.AuthChangePasswordBack) + t.Run("AuthRegisterInvalid", s.AuthRegisterInvalid) + t.Run("AuthRegister", s.AuthRegister) + t.Run("AuthLogout", s.AuthLogout) + t.Run("AuthLogin", s.AuthLogin) +} + +// AuthLoginInvalid tests the /auth/login endpoint with invalid credentials +func (s *HttpTestService) AuthLoginInvalid(t *testing.T) { + obj := s.E.POST("/auth/login").WithJSON(internalHttp.LoginRequest{Username: "foo", Password: "foobar123"}).Expect().Status(http.StatusBadRequest).JSON().Object() + + obj.Value("success").IsEqual(false) +} + +// AuthLogin tests the /auth/login endpoint with valid credentials +func (s *HttpTestService) AuthLogin(t *testing.T) { + var user *ent.User + + obj := s.E.POST("/auth/login").WithJSON(internalHttp.LoginRequest{Username: "admin", Password: "ganymede"}).Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) + obj.Value("data").Decode(&user) + + assert.Equal(t, "admin", user.Username) + assert.Equal(t, utils.AdminRole, user.Role) + assert.NotNil(t, user.ID) +} + +// AuthGetUser tests the /auth/me endpoint +func (s *HttpTestService) AuthGetUser(t *testing.T) { + var user *ent.User + + obj := s.E.GET("/auth/me").Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) + obj.Value("data").Decode(&user) + + assert.Equal(t, "admin", user.Username) + assert.Equal(t, utils.AdminRole, user.Role) + assert.NotNil(t, user.ID) +} + +// AuthChangePasswordInvalid tests the /auth/change-password endpoint with invalid credentials +func (s *HttpTestService) AuthChangePasswordInvalid(t *testing.T) { + obj := s.E.POST("/auth/change-password").WithJSON(internalHttp.ChangePasswordRequest{OldPassword: "ganymede1", NewPassword: "ganymede1", ConfirmNewPassword: "ganymede1"}).Expect().Status(http.StatusInternalServerError).JSON().Object() + + obj.Value("success").IsEqual(false) +} + +// AuthChangePasswordNoMatch tests the /auth/change-password endpoint with non-matching new passwords +func (s *HttpTestService) AuthChangePasswordNoMatch(t *testing.T) { + obj := s.E.POST("/auth/change-password").WithJSON(internalHttp.ChangePasswordRequest{OldPassword: "ganymede", NewPassword: "ganymede1", ConfirmNewPassword: "ganymede2"}).Expect().Status(http.StatusBadRequest).JSON().Object() + + obj.Value("success").IsEqual(false) +} + +// AuthChangePassword tests the /auth/change-password endpoint with valid credentials +func (s *HttpTestService) AuthChangePassword(t *testing.T) { + obj := s.E.POST("/auth/change-password").WithJSON(internalHttp.ChangePasswordRequest{OldPassword: "ganymede", NewPassword: "ganymede1", ConfirmNewPassword: "ganymede1"}).Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) +} + +// AuthChangePassword tests the /auth/change-password endpoint changing the password back to the original +func (s *HttpTestService) AuthChangePasswordBack(t *testing.T) { + obj := s.E.POST("/auth/change-password").WithJSON(internalHttp.ChangePasswordRequest{OldPassword: "ganymede1", NewPassword: "ganymede", ConfirmNewPassword: "ganymede"}).Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) +} + +// AuthRegisterInvalid tests the /auth/register endpoint with invalid credentials +func (s *HttpTestService) AuthRegisterInvalid(t *testing.T) { + obj := s.E.POST("/auth/register").WithJSON(internalHttp.RegisterRequest{Username: "t", Password: "short"}).Expect().Status(http.StatusBadRequest).JSON().Object() + + obj.Value("success").IsEqual(false) +} + +// AuthRegister tests the /auth/register endpoint with valid credentials +func (s *HttpTestService) AuthRegister(t *testing.T) { + var user *ent.User + + obj := s.E.POST("/auth/register").WithJSON(internalHttp.RegisterRequest{Username: "testing", Password: "testing123"}).Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) + obj.Value("data").Decode(&user) + + assert.Equal(t, "testing", user.Username) + assert.Equal(t, utils.UserRole, user.Role) + assert.NotNil(t, user.ID) +} + +// AuthLogout tests the /auth/logout endpoint +func (s *HttpTestService) AuthLogout(t *testing.T) { + + obj := s.E.POST("/auth/logout").Expect().Status(http.StatusOK).JSON().Object() + + obj.Value("success").IsEqual(true) +} diff --git a/internal/transport/http/handler.go b/internal/transport/http/handler.go index 2c3d258..6d22390 100644 --- a/internal/transport/http/handler.go +++ b/internal/transport/http/handler.go @@ -2,9 +2,11 @@ package http import ( "context" + "fmt" "net/http" "net/http/httputil" "net/url" + "os" "strings" "time" @@ -321,10 +323,14 @@ func groupV1Routes(e *echo.Group, h *Handler) { } func (h *Handler) Serve(ctx context.Context) error { + appPort := os.Getenv("APP_PORT") + if appPort == "" { + appPort = "4000" + } // Run the server in a goroutine serverErrCh := make(chan error, 1) go func() { - if err := h.Server.Start(":4000"); err != nil && err != http.ErrServerClosed { + if err := h.Server.Start(fmt.Sprintf(":%s", appPort)); err != nil && err != http.ErrServerClosed { serverErrCh <- err } close(serverErrCh) diff --git a/internal/worker/worker.go b/internal/worker/worker.go new file mode 100644 index 0000000..0e6cec6 --- /dev/null +++ b/internal/worker/worker.go @@ -0,0 +1,107 @@ +package worker + +import ( + "context" + "fmt" + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/rs/zerolog/pkgerrors" + "github.com/zibbp/ganymede/internal/archive" + "github.com/zibbp/ganymede/internal/blocked" + "github.com/zibbp/ganymede/internal/channel" + "github.com/zibbp/ganymede/internal/config" + serverConfig "github.com/zibbp/ganymede/internal/config" + "github.com/zibbp/ganymede/internal/database" + "github.com/zibbp/ganymede/internal/live" + "github.com/zibbp/ganymede/internal/platform" + "github.com/zibbp/ganymede/internal/queue" + tasks_client "github.com/zibbp/ganymede/internal/tasks/client" + tasks_worker "github.com/zibbp/ganymede/internal/tasks/worker" + "github.com/zibbp/ganymede/internal/vod" +) + +// SetupWorker sets up the worker +func SetupWorker(ctx context.Context) (*tasks_worker.RiverWorkerClient, error) { + envConfig := config.GetEnvConfig() + envAppConfig := config.GetEnvApplicationConfig() + _, err := serverConfig.Init() + if err != nil { + return nil, err + } + + zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack + if envConfig.DEBUG { + log.Info().Msg("debug mode enabled") + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } else { + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } + + // Disable logging for tests + if os.Getenv("TESTS_LOGGING") == "false" { + zerolog.SetGlobalLevel(zerolog.Disabled) + } + + dbString := fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s sslrootcert=%s", envAppConfig.DB_USER, envAppConfig.DB_PASS, envAppConfig.DB_HOST, envAppConfig.DB_PORT, envAppConfig.DB_NAME, envAppConfig.DB_SSL, envAppConfig.DB_SSL_ROOT_CERT) + + db := database.NewDatabase(ctx, database.DatabaseConnectionInput{ + DBString: dbString, + IsWorker: false, + }) + + riverClient, err := tasks_client.NewRiverClient(tasks_client.RiverClientInput{ + DB_URL: dbString, + }) + if err != nil { + return nil, err + } + + var platformTwitch platform.Platform + // setup twitch platform + if envConfig.TwitchClientId != "" && envConfig.TwitchClientSecret != "" { + platformTwitch = &platform.TwitchConnection{ + ClientId: envConfig.TwitchClientId, + ClientSecret: envConfig.TwitchClientSecret, + } + _, err = platformTwitch.Authenticate(ctx) + if err != nil { + return nil, err + } + } + + channelService := channel.NewService(db, platformTwitch) + vodService := vod.NewService(db, riverClient, platformTwitch) + queueService := queue.NewService(db, vodService, channelService, riverClient) + blockedVodsService := blocked.NewService(db) + // twitchService := twitch.NewService() + archiveService := archive.NewService(db, channelService, vodService, queueService, blockedVodsService, riverClient, platformTwitch) + liveService := live.NewService(db, archiveService, platformTwitch) + + // initialize river + riverWorkerClient, err := tasks_worker.NewRiverWorker(tasks_worker.RiverWorkerInput{ + DB_URL: dbString, + DB: db, + PlatformTwitch: platformTwitch, + VideoDownloadWorkers: envConfig.MaxVideoDownloadExecutions, + VideoPostProcessWorkers: envConfig.MaxVideoConvertExecutions, + ChatDownloadWorkers: envConfig.MaxChatDownloadExecutions, + ChatRenderWorkers: envConfig.MaxChatRenderExecutions, + }) + if err != nil { + return nil, err + } + + // get periodic tasks + periodicTasks, err := riverWorkerClient.GetPeriodicTasks(liveService) + if err != nil { + log.Panic().Err(err).Msg("Error getting periodic tasks") + } + + for _, task := range periodicTasks { + riverWorkerClient.Client.PeriodicJobs().Add(task) + } + + return riverWorkerClient, nil +} diff --git a/tests/setup.go b/tests/setup.go index e6f5f0d..f840215 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -2,87 +2,228 @@ package tests import ( "context" + "fmt" + "net" + "net/http" "os" + osExec "os/exec" + "path/filepath" + "strings" "testing" "time" + "github.com/gavv/httpexpect/v2" + "github.com/joho/godotenv" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" "github.com/zibbp/ganymede/internal/server" + "github.com/zibbp/ganymede/internal/worker" ) -// Setup initializes the integration test environment. -// It setups up the entire application and returns the various services for testing. -// A Postgres Testcontainer is used to provide a real database for further tersting. -func Setup(t *testing.T) (*server.Application, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +var ( + TestPostgresDatabase = "ganymede" + TestPostgresUser = "ganymede" + TestPostgresPassword = "ganymede" +) - // create temporary postgres container to run the tests - postgresContainer, err := postgres.Run(ctx, - "postgres:14-alpine", - postgres.WithDatabase("test"), - postgres.WithUsername("user"), - postgres.WithPassword("password"), +// setupPostgresTestContainer sets up a postgres container for testing +func setupPostgresTestContainer(ctx context.Context) (*postgres.PostgresContainer, error) { + pgContainer, err := postgres.Run(ctx, "postgres:17-alpine", + postgres.WithDatabase(TestPostgresDatabase), + postgres.WithUsername(TestPostgresUser), + postgres.WithPassword(TestPostgresPassword), testcontainers.WithWaitStrategy( wait.ForLog("database system is ready to accept connections"). WithOccurrence(2). WithStartupTimeout(5*time.Second)), ) + if err != nil { return nil, err } - port, err := postgresContainer.MappedPort(ctx, "5432") + return pgContainer, nil +} + +// getRootPath returns the root path of the project +// Git is the simplest way to get the root path of the project +func getRootPath(t *testing.T) string { + e := osExec.Command("git", "rev-parse", "--show-toplevel") + out, err := e.Output() if err != nil { - return nil, err + t.Errorf("Could not get root path: %v", err) } + return strings.Trim(string(out), "\n") +} +func setupEnvironment(t *testing.T, postgresHost string, postgresPort string) { + // Load .env file if available (for local development) + envPath := filepath.Join(getRootPath(t), ".env") + _ = godotenv.Load(envPath) + + t.Log(envPath) - // set environment variables - os.Setenv("DB_HOST", "localhost") - os.Setenv("DB_PORT", port.Port()) - os.Setenv("DB_USER", "user") - os.Setenv("DB_PASS", "password") - os.Setenv("DB_NAME", "test") - os.Setenv("JWT_SECRET", "secret") - os.Setenv("JWT_REFRESH_SECRET", "refresh_secret") - os.Setenv("FRONTEND_HOST", "http://localhost:1234") + // Set the environment variables specific to the test + os.Setenv("TESTS_LOGGING", "false") // Disable logging for tests + os.Setenv("DEBUG", "false") + os.Setenv("DB_HOST", postgresHost) + os.Setenv("DB_PORT", postgresPort) + os.Setenv("DB_NAME", TestPostgresDatabase) + os.Setenv("DB_USER", TestPostgresUser) + os.Setenv("DB_PASS", TestPostgresPassword) + // Set paths // set temporary directories videosDir, err := os.MkdirTemp("/tmp", "ganymede-tests") - if err != nil { - return nil, err - } + assert.NoError(t, err) os.Setenv("VIDEOS_DIR", videosDir) t.Log("VIDEOS_DIR", videosDir) tempDir, err := os.MkdirTemp("/tmp", "ganymede-tests") - if err != nil { - return nil, err - } + assert.NoError(t, err) os.Setenv("TEMP_DIR", tempDir) t.Log("TEMP_DIR", tempDir) configDir, err := os.MkdirTemp("/tmp", "ganymede-tests") - if err != nil { - return nil, err - } + assert.NoError(t, err) os.Setenv("CONFIG_DIR", configDir) t.Log("CONFIG_DIR", configDir) logsDir, err := os.MkdirTemp("/tmp", "ganymede-tests") - if err != nil { - return nil, err - } + assert.NoError(t, err) os.Setenv("LOGS_DIR", logsDir) t.Log("LOGS_DIR", logsDir) +} - // create the application. this does not start the HTTP server +// Setup initializes the integration test environment. +// It setups up the entire application and returns the various services for testing. +// A Postgres Testcontainer is used to provide a real database for further tersting. +// Used for service tests in internal//_test.go +func Setup(t *testing.T) (*server.Application, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer, err := setupPostgresTestContainer(ctx) + if err != nil { + t.Fatalf("Could not start postgres container: %v", err) + } + + // Register cleanup + t.Cleanup(func() { + Cleanup(t) + if postgresContainer != nil { + _ = postgresContainer.Terminate(ctx) + } + }) + + postgresPort, err := postgresContainer.MappedPort(ctx, "5432") + assert.NoError(t, err) + + // Setup env vars + setupEnvironment(t, "localhost", postgresPort.Port()) + + // Create the application. This does not start the HTTP server app, err := server.SetupApplication(ctx) if err != nil { return nil, err } + // Start worker + workerClient, err := worker.SetupWorker(ctx) + if err != nil { + return nil, err + } + + go func() { + if err := workerClient.Start(); err != nil { + log.Panic().Err(err).Msg("Error running river worker") + } + }() + return app, nil + +} + +// SetupHTTP is similar to Setup but starts the HTTP server for testing end-to-end http requests. +// Used for tests in internal/transport/http +func SetupHTTP(t *testing.T) (*httpexpect.Expect, error) { + ctx := context.Background() + + postgresContainer, err := setupPostgresTestContainer(ctx) + if err != nil { + t.Fatalf("Could not start postgres container: %v", err) + } + + // Register cleanup + t.Cleanup(func() { + Cleanup(t) + if postgresContainer != nil { + _ = postgresContainer.Terminate(ctx) + } + }) + + postgresPort, err := postgresContainer.MappedPort(ctx, "5432") + assert.NoError(t, err) + + // Setup env vars + setupEnvironment(t, "localhost", postgresPort.Port()) + + // Get free port for Ganymede to run on + port, err := getFreePort() + assert.NoError(t, err) + os.Setenv("APP_PORT", fmt.Sprintf("%d", port)) + + // Start the application + go func() { + err = server.Run(ctx) + assert.NoError(t, err) + }() + + // Wait for the application to start + time.Sleep(5 * time.Second) + + e := httpexpect.WithConfig(httpexpect.Config{ + BaseURL: fmt.Sprintf("http://localhost:%d/api/v1", port), + Reporter: httpexpect.NewAssertReporter(t), + Client: &http.Client{ + Jar: httpexpect.NewCookieJar(), + }, + }) + + // // Start worker + // workerClient, err := worker.SetupWorker(ctx) + // if err != nil { + // return nil, err + // } + + // go func() { + // if err := workerClient.Start(); err != nil { + // log.Panic().Err(err).Msg("Error running river worker") + // } + // }() + + return e, nil +} + +// Cleanup cleans up the test environment +func Cleanup(t *testing.T) { + // Cleanup temporary directories + assert.NoError(t, os.RemoveAll(os.Getenv("VIDEOS_DIR"))) + assert.NoError(t, os.RemoveAll(os.Getenv("TEMP_DIR"))) + assert.NoError(t, os.RemoveAll(os.Getenv("CONFIG_DIR"))) + assert.NoError(t, os.RemoveAll(os.Getenv("LOGS_DIR"))) +} + +// getFreePort returns a free port on the host +func getFreePort() (int, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer listener.Close() + + addr := listener.Addr().(*net.TCPAddr) + return addr.Port, nil } From 1f52916497578a502a27c1d9752116474133a26f Mon Sep 17 00:00:00 2001 From: Zibbp Date: Fri, 3 Jan 2025 23:34:13 +0000 Subject: [PATCH 02/22] hardcode image for now --- .github/workflows/go-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index da59805..4211a54 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -10,7 +10,7 @@ jobs: test: runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}:dev + image: ghcr.io/zibbp/ganymede:dev steps: - name: Checkout repository From 6b7db6cf54911cec2cb0e0185871da3645b1c79b Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 18:24:28 +0000 Subject: [PATCH 03/22] add required deps --- .github/workflows/go-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 4211a54..1cc23fc 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -28,4 +28,6 @@ jobs: env: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} - run: make test + run: | + apt update && apt install -y make git + make test From b9f607aa1551116273c04d42eb587593b6e3c625 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 18:28:42 +0000 Subject: [PATCH 04/22] error not required --- tests/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.go b/tests/setup.go index f840215..0993523 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -54,7 +54,7 @@ func getRootPath(t *testing.T) string { e := osExec.Command("git", "rev-parse", "--show-toplevel") out, err := e.Output() if err != nil { - t.Errorf("Could not get root path: %v", err) + t.Logf("Could not get root path: %v", err) } return strings.Trim(string(out), "\n") } From 4e62bfd2b25cc4d115275cacee0ad98fd09f5700 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 18:32:58 +0000 Subject: [PATCH 05/22] enable logging --- tests/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.go b/tests/setup.go index 0993523..ad6b8d7 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -66,7 +66,7 @@ func setupEnvironment(t *testing.T, postgresHost string, postgresPort string) { t.Log(envPath) // Set the environment variables specific to the test - os.Setenv("TESTS_LOGGING", "false") // Disable logging for tests + os.Setenv("TESTS_LOGGING", "true") // Disable logging for tests os.Setenv("DEBUG", "false") os.Setenv("DB_HOST", postgresHost) os.Setenv("DB_PORT", postgresPort) From b53bf669f00a7f4e65af3363abd0a25b34af9ad9 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 18:42:43 +0000 Subject: [PATCH 06/22] test --- .github/workflows/go-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 1cc23fc..10d61bc 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -9,8 +9,8 @@ on: jobs: test: runs-on: ubuntu-latest - container: - image: ghcr.io/zibbp/ganymede:dev + # container: + # image: ghcr.io/zibbp/ganymede:dev steps: - name: Checkout repository From ec0abd0786eae81f331b108137d723f42a90036c Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 21:24:17 +0000 Subject: [PATCH 07/22] dind? --- .github/workflows/go-test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 10d61bc..2887866 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -9,8 +9,10 @@ on: jobs: test: runs-on: ubuntu-latest - # container: - # image: ghcr.io/zibbp/ganymede:dev + container: + image: ghcr.io/zibbp/ganymede:dev + volumes: + - /var/run/docker.sock:/var/run/docker.sock steps: - name: Checkout repository @@ -29,5 +31,5 @@ jobs: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} run: | - apt update && apt install -y make git + apt update && apt install -y docker make git make test From bbb9e6d4dfd7e9e7315fa5f28a7ac093dc086c70 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 21:34:01 +0000 Subject: [PATCH 08/22] test --- .github/workflows/go-test.yml | 4 +++- tests/setup.go | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 2887866..b490b76 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -31,5 +31,7 @@ jobs: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} run: | - apt update && apt install -y docker make git + apt update && apt install -y make git + docker ps make test + docker ps diff --git a/tests/setup.go b/tests/setup.go index ad6b8d7..7f2d51f 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -121,8 +121,11 @@ func Setup(t *testing.T) (*server.Application, error) { postgresPort, err := postgresContainer.MappedPort(ctx, "5432") assert.NoError(t, err) + postgresIp, err := postgresContainer.ContainerIP(ctx) + assert.NoError(t, err) + // Setup env vars - setupEnvironment(t, "localhost", postgresPort.Port()) + setupEnvironment(t, postgresIp, postgresPort.Port()) // Create the application. This does not start the HTTP server app, err := server.SetupApplication(ctx) From 7f558602744db15b4a3dfff57bdcd78f4d044c1b Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 21:39:04 +0000 Subject: [PATCH 09/22] test --- .github/workflows/go-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index b490b76..dbc094e 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -31,7 +31,7 @@ jobs: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} run: | - apt update && apt install -y make git + apt update && apt install -y docker make git docker ps make test docker ps From 0d395a339e2dfc6bca2ae8858293c7366a2b97a0 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 21:52:09 +0000 Subject: [PATCH 10/22] test --- .github/workflows/go-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index dbc094e..f8452b0 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -31,7 +31,5 @@ jobs: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} run: | - apt update && apt install -y docker make git - docker ps + apt update && apt install -y make git make test - docker ps From 55b2235273cfd45333b04d2127ce6ff22cf99411 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 22:00:20 +0000 Subject: [PATCH 11/22] test --- tests/setup.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/setup.go b/tests/setup.go index 7f2d51f..64327a1 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -124,6 +124,8 @@ func Setup(t *testing.T) (*server.Application, error) { postgresIp, err := postgresContainer.ContainerIP(ctx) assert.NoError(t, err) + t.Log("Postgres IP:", postgresIp) + // Setup env vars setupEnvironment(t, postgresIp, postgresPort.Port()) From a71d57c141629ee5d2ab9395870d4e3b57f92593 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 22:17:27 +0000 Subject: [PATCH 12/22] test --- .github/workflows/go-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index f8452b0..096c7a4 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -11,13 +11,15 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/zibbp/ganymede:dev - volumes: - - /var/run/docker.sock:/var/run/docker.sock + options: --privileged steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + - name: Set up Go uses: actions/setup-go@v5 with: From 2484199a1541a1530d1cbcedb571f0eda11d1ce7 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 4 Jan 2025 22:25:40 +0000 Subject: [PATCH 13/22] test --- .github/workflows/go-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 096c7a4..6e14e6e 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -33,5 +33,7 @@ jobs: TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} run: | + docker ps apt update && apt install -y make git make test + docker ps From 06de43822287e6b54c949cedcf0e3e07909fe173 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sun, 5 Jan 2025 15:41:54 +0000 Subject: [PATCH 14/22] test --- .github/workflows/go-test.yml | 42 ++++++++++++++++++++--------------- Dockerfile | 20 +++++++++++++++++ tests/setup.go | 2 +- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 6e14e6e..0dd5f7f 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -6,34 +6,40 @@ on: pull_request: branches: [main] +env: + TESTCONTAINER_DOCKER_NETWORK: ganymede-tests + jobs: test: - runs-on: ubuntu-latest - container: - image: ghcr.io/zibbp/ganymede:dev - options: --privileged + runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Docker + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Set up Go - uses: actions/setup-go@v5 + - name: Build test docker image + id: docker_build_test_image + uses: docker/build-push-action@v6 with: - go-version: 1.x + platforms: linux/amd64 + push: false + load: true + target: tests + + - name: get-name + run: | + echo "REPO_NAME=$(basename ${{ github.repository }})" >> $GITHUB_ENV - - name: Install dependencies - run: go mod download + - name: Create docker network + run: docker network create ${{ env.TESTCONTAINER_DOCKER_NETWORK }} - - name: Run Tests - env: - TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} - TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} + - name: Run tests run: | - docker ps - apt update && apt install -y make git - make test - docker ps + docker run \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /home/runner/work/${{ env.REPO_NAME }}/${{ env.REPO_NAME }}:/opt/app + "${{ steps.docker_build_test_image.outputs.imageid }}" \ + bash -c "cd /opt/app && make test" diff --git a/Dockerfile b/Dockerfile index 883363a..c6c90a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,26 @@ RUN \ else echo "Lockfile not found." && exit 1; \ fi +# +# Tests stage. Inclues depedencies required for tests +# +FROM golang:1.23-bookworm AS tests + +RUN apt-get update && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg make git + +RUN pip3 install --upgrade pip streamlink --break-system-packages + +# Copy and install chat-downloader +COPY --from=tools /tmp/chat-downloader /tmp/chat-downloader +RUN cd /tmp/chat-downloader && python3 setup.py install && cd .. && rm -rf chat-downloader + +# Setup fonts +RUN chmod 644 /usr/share/fonts/* && chmod -R a+rX /usr/share/fonts + +# Copy TwitchDownloaderCLI +COPY --from=tools /tmp/TwitchDownloaderCLI /usr/local/bin/ +RUN chmod +x /usr/local/bin/TwitchDownloaderCLI + # Production stage FROM debian:bookworm-slim WORKDIR /opt/app diff --git a/tests/setup.go b/tests/setup.go index 64327a1..518104f 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -121,7 +121,7 @@ func Setup(t *testing.T) (*server.Application, error) { postgresPort, err := postgresContainer.MappedPort(ctx, "5432") assert.NoError(t, err) - postgresIp, err := postgresContainer.ContainerIP(ctx) + postgresIp, err := postgresContainer.Host(ctx) assert.NoError(t, err) t.Log("Postgres IP:", postgresIp) From 69e2e1b6bd576f47d79c0a783406884614f042ce Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sun, 5 Jan 2025 15:47:37 +0000 Subject: [PATCH 15/22] fix --- .github/workflows/go-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 0dd5f7f..b4d9ffa 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -40,6 +40,6 @@ jobs: run: | docker run \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v /home/runner/work/${{ env.REPO_NAME }}/${{ env.REPO_NAME }}:/opt/app + -v /home/runner/work/${{ env.REPO_NAME }}/${{ env.REPO_NAME }}:/opt/app \ "${{ steps.docker_build_test_image.outputs.imageid }}" \ bash -c "cd /opt/app && make test" From 55067de42fa5b0ea5d20dee1baa642c7d8a3da9b Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sun, 5 Jan 2025 15:52:27 +0000 Subject: [PATCH 16/22] fix --- .github/workflows/go-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index b4d9ffa..02cbf44 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -41,5 +41,7 @@ jobs: docker run \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /home/runner/work/${{ env.REPO_NAME }}/${{ env.REPO_NAME }}:/opt/app \ + -e TWITCH_CLIENT_ID=${{ secrets.TWITCH_CLIENT_ID }} \ + -e TWITCH_CLIENT_SECRET=${{ secrets.TWITCH_CLIENT_SECRET }} \ "${{ steps.docker_build_test_image.outputs.imageid }}" \ bash -c "cd /opt/app && make test" From 3b634ade802b7519b86f378b58a5b3a9ba160b37 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sun, 5 Jan 2025 16:02:21 +0000 Subject: [PATCH 17/22] fix http test setup --- tests/setup.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/setup.go b/tests/setup.go index 518104f..78e97e3 100644 --- a/tests/setup.go +++ b/tests/setup.go @@ -172,8 +172,13 @@ func SetupHTTP(t *testing.T) (*httpexpect.Expect, error) { postgresPort, err := postgresContainer.MappedPort(ctx, "5432") assert.NoError(t, err) + postgresIp, err := postgresContainer.Host(ctx) + assert.NoError(t, err) + + t.Log("Postgres IP:", postgresIp) + // Setup env vars - setupEnvironment(t, "localhost", postgresPort.Port()) + setupEnvironment(t, postgresIp, postgresPort.Port()) // Get free port for Ganymede to run on port, err := getFreePort() From e6e93e3ac52a030068705b274f693b532d97e258 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 11 Jan 2025 15:11:23 +0000 Subject: [PATCH 18/22] add note about platform missing --- internal/tasks/shared.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/tasks/shared.go b/internal/tasks/shared.go index 5ae5d8d..9379883 100644 --- a/internal/tasks/shared.go +++ b/internal/tasks/shared.go @@ -80,6 +80,7 @@ func StoreFromContext(ctx context.Context) (*database.Database, error) { func PlatformFromContext(ctx context.Context) (platform.Platform, error) { platform, exists := ctx.Value(tasks_shared.PlatformTwitchKey).(platform.Platform) if !exists || platform == nil { + log.Error().Msg("platform not found in context, this usually means the platform authentication failed, check your platform client_id and client_secret.") return nil, errors.New("platform not found in context") } From f64ad2b2389c004ce2b76227e67205788548783c Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 11 Jan 2025 15:33:06 +0000 Subject: [PATCH 19/22] use environment secrets for PRs --- .github/workflows/{go-test.yml => test.yml} | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename .github/workflows/{go-test.yml => test.yml} (89%) diff --git a/.github/workflows/go-test.yml b/.github/workflows/test.yml similarity index 89% rename from .github/workflows/go-test.yml rename to .github/workflows/test.yml index 02cbf44..0e80684 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Go Test +name: Tests on: push: @@ -12,6 +12,7 @@ env: jobs: test: runs-on: ubuntu-22.04 + environment: Tests steps: - name: Checkout repository @@ -36,6 +37,7 @@ jobs: - name: Create docker network run: docker network create ${{ env.TESTCONTAINER_DOCKER_NETWORK }} + # Instead of having to manually install all dependencies, build the docker container and run tests in there - name: Run tests run: | docker run \ From ced7675b38f47aefa5e17773d3a70a01c587854f Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 11 Jan 2025 15:42:49 +0000 Subject: [PATCH 20/22] use pull_request_target --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e80684..fadee62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,8 +3,8 @@ name: Tests on: push: branches: [main] - pull_request: - branches: [main] + pull_request_target: # run in the context of base branch for secrets + types: [opened, synchronize, reopened] env: TESTCONTAINER_DOCKER_NETWORK: ganymede-tests @@ -12,11 +12,12 @@ env: jobs: test: runs-on: ubuntu-22.04 - environment: Tests steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From fdc2370a103845a60621dbfa3970ec77df9f0398 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 11 Jan 2025 20:34:02 +0000 Subject: [PATCH 21/22] fix rebase --- internal/worker/worker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 0e6cec6..828d6d7 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -88,6 +88,7 @@ func SetupWorker(ctx context.Context) (*tasks_worker.RiverWorkerClient, error) { VideoPostProcessWorkers: envConfig.MaxVideoConvertExecutions, ChatDownloadWorkers: envConfig.MaxChatDownloadExecutions, ChatRenderWorkers: envConfig.MaxChatRenderExecutions, + SpriteThumbnailWorkers: envConfig.MaxVideoSpriteThumbnailExecutions, }) if err != nil { return nil, err From 216370923957bdb4edce708b73ff743297fb9e77 Mon Sep 17 00:00:00 2001 From: Zibbp Date: Sat, 11 Jan 2025 22:46:26 +0000 Subject: [PATCH 22/22] add clip and sprite thumbnail tests --- internal/archive/archive_test.go | 176 +++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 21925ee..033f16b 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -25,6 +25,7 @@ var ( TestTwitchChannelDisplayName = "sodapoppin" TestTwitchChannelExtId = "26301881" TestTwitchVideoId = "1989753443" + TestTwitchClipId = "SarcasticDarkPanCoolCat-rgyYByzzfGqIwbWd" TestArchiveTimeout = 300 * time.Second ) @@ -381,3 +382,178 @@ func TestArchiveVideoHLS(t *testing.T) { assert.NoError(t, err) assert.Greater(t, len(files), 0) } + +// ArchiveVideo tests the full archive process for a video with chat downloading, processing, and rendering +func TestArchiveClip(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Archive the video + err = app.ArchiveService.ArchiveClip(context.Background(), archive.ArchiveClipInput{ + ID: TestTwitchClipId, + Quality: utils.R720P60, + ArchiveChat: true, + RenderChat: true, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchClipId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, true, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, true, q.RenderChat) + assert.Equal(t, true, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Pending, q.TaskChatDownload) + assert.Equal(t, utils.Pending, q.TaskChatRender) + assert.Equal(t, utils.Pending, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.FileExists(t, v.VideoPath) + assert.FileExists(t, v.ChatPath) + assert.FileExists(t, v.ChatVideoPath) +} + +// TestArchiveVideoWithSpriteThumbnails tests generate sprite thumbnails after a video is archived. +func TestArchiveVideoWithSpriteThumbnails(t *testing.T) { + // Setup the application + app, err := tests.Setup(t) + assert.NoError(t, err) + + // Archive the video + err = app.ArchiveService.ArchiveVideo(context.Background(), archive.ArchiveVideoInput{ + VideoId: TestTwitchVideoId, + Quality: utils.R720P60, + ArchiveChat: false, + RenderChat: false, + }) + assert.NoError(t, err) + + // Assert video was created + v, err := app.Database.Client.Vod.Query().Where(vod.ExtID(TestTwitchVideoId)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + // Assert queue item was created + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, true, q.VideoProcessing) + assert.Equal(t, false, q.RenderChat) + assert.Equal(t, false, q.ArchiveChat) + assert.NotNil(t, q.WorkflowID) + assert.NotNil(t, q.WorkflowRunID) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Pending, q.TaskVideoDownload) + assert.Equal(t, utils.Pending, q.TaskVideoConvert) + assert.Equal(t, utils.Pending, q.TaskVideoMove) + + // Wait for the video to be archived + startTime := time.Now() + for { + if time.Since(startTime) >= TestArchiveTimeout { + t.Errorf("Timeout reached while waiting for video to be archived") + } + + q, err := app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + if err != nil { + t.Errorf("Error querying queue item: %v", err) + } + + if !q.Processing { + break + } + + time.Sleep(5 * time.Second) + } + + // Assert queue item was updated + q, err = app.Database.Client.Queue.Query().Where(queue.HasVodWith(vod.ID(v.ID))).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.Equal(t, false, q.ChatProcessing) + assert.Equal(t, false, q.VideoProcessing) + assert.Equal(t, utils.Success, q.TaskChatDownload) + assert.Equal(t, utils.Success, q.TaskChatRender) + assert.Equal(t, utils.Success, q.TaskChatMove) + assert.Equal(t, utils.Success, q.TaskVideoDownload) + assert.Equal(t, utils.Success, q.TaskVideoConvert) + assert.Equal(t, utils.Success, q.TaskVideoMove) + + // Assert files exist + assert.FileExists(t, v.ThumbnailPath) + assert.FileExists(t, v.WebThumbnailPath) + assert.FileExists(t, v.VideoPath) + assert.NoFileExists(t, v.ChatPath) + assert.NoFileExists(t, v.ChatVideoPath) + + // Wait 5 seconds for sprite thumbnails + time.Sleep(5 * time.Second) + + // Assert sprite thumbnail facts + v, err = app.Database.Client.Vod.Query().Where(vod.ID(v.ID)).Only(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, v.SpriteThumbnailsColumns) + assert.NotNil(t, v.SpriteThumbnailsRows) + assert.NotNil(t, v.SpriteThumbnailsHeight) + assert.NotNil(t, v.SpriteThumbnailsWidth) + assert.NotNil(t, v.SpriteThumbnailsInterval) + if len(v.SpriteThumbnailsImages) == 0 { + t.Errorf("expected more than 0 sprite thumbnails") + } + + for _, spriteThumbnailPath := range v.SpriteThumbnailsImages { + assert.FileExists(t, spriteThumbnailPath) + } +}