Skip to content

Commit

Permalink
tests: add test minio
Browse files Browse the repository at this point in the history
  • Loading branch information
Mic92 committed Oct 27, 2024
1 parent 6bbe643 commit 680f33b
Show file tree
Hide file tree
Showing 10 changed files with 248 additions and 8 deletions.
17 changes: 14 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,31 @@ module github.com/Mic92/niks3
go 1.22.7

require (
github.com/jackc/pgx/v5 v5.7.1
github.com/jmoiron/sqlx v1.4.0
github.com/lib/pq v1.10.9
github.com/minio/minio-go/v7 v7.0.79
github.com/pressly/goose/v3 v3.22.1
)

require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.7.1 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
)
24 changes: 24 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
Expand All @@ -21,6 +25,11 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
Expand All @@ -29,6 +38,10 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.79 h1:SvJZpj3hT0RN+4KiuX/FxLfPZdsuegy6d/2PiemM/bM=
github.com/minio/minio-go/v7 v7.0.79/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -37,6 +50,8 @@ github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuio
github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand All @@ -48,12 +63,21 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
13 changes: 13 additions & 0 deletions main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,25 @@ import (

func innerTestMain(m *testing.M) int {
var err error

// unload environment variables from the devenv
os.Unsetenv("DATABASE_URL")
os.Unsetenv("PGDATABASE")
os.Unsetenv("PGUSER")
os.Unsetenv("PGHOST")

testPostgresServer, err = startPostgresServer()
defer testPostgresServer.Cleanup()
if err != nil {
slog.Error("failed to start postgres", "error", err)
return 1
}
testMinioServer, err = startMinioServer()
defer testMinioServer.Cleanup()
if err != nil {
slog.Error("failed to start minio", "error", err)
os.Exit(1)
}
return m.Run()
}

Expand Down
149 changes: 149 additions & 0 deletions minio_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package main

import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"log/slog"
"net"
"os"
"os/exec"
"path/filepath"
"sync/atomic"
"syscall"
"testing"
"time"

minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)

var (
testMinioServer *minioServer
testBucketCount atomic.Int32
)

type minioServer struct {
cmd *exec.Cmd
tempDir string
secret string
port uint16
}

func randToken(n int) (string, error) {
bytes := make([]byte, n)
if _, err := rand.Read(bytes); err != nil {
return "", err
}
return hex.EncodeToString(bytes), nil
}

func randPort() (uint16, error) {
ln, err := net.Listen("tcp", "localhost:0")
if err != nil {
return 0, err
}
ln.Close()
time.Sleep(1 * time.Second)
return (uint16)(ln.Addr().(*net.TCPAddr).Port), nil
}

func (s *minioServer) Client(t *testing.T) *minio.Client {
endpoint := fmt.Sprintf("localhost:%d", s.port)
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4("minioadmin", s.secret, ""),
Secure: false,
})
ok(t, err)
return minioClient
}

func (s *minioServer) Cleanup() {
err := syscall.Kill(s.cmd.Process.Pid, syscall.SIGKILL)
if err != nil {
slog.Error("failed to kill postgres", "error", err)
}
err = s.cmd.Wait()
if err != nil {
slog.Error("failed to wait for postgres", "error", err)
}

os.RemoveAll(s.tempDir)
}

func startMinioServer() (*minioServer, error) {
tempDir, err := os.MkdirTemp("", "minio")
if err != nil {
return nil, fmt.Errorf("failed to create temp dir: %w", err)
}
defer func() {
if err != nil {
os.RemoveAll(tempDir)
}
}()

port, err := randPort()
if err != nil {
return nil, fmt.Errorf("failed to find free port: %w", err)
}

fmt.Printf("####################### Command: minio server --address :%d %s\n", port, filepath.Join(tempDir, "data"))

minioProc := exec.Command("minio", "server", "--address", fmt.Sprintf(":%d", port), filepath.Join(tempDir, "data"))
minioProc.Stdout = os.Stdout
minioProc.Stderr = os.Stderr

// random hex string
secret, err := randToken(20)
if err != nil {
return nil, fmt.Errorf("failed to generate access key: %w", err)
}

env := os.Environ()
env = append(env, "MINIO_ROOT_USER=minioadmin")
env = append(env, fmt.Sprintf("MINIO_ROOT_PASSWORD=%s", secret))
env = append(env, "AWS_ACCESS_KEY_ID=minioadmin")
env = append(env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", secret))
minioProc.Env = env

if err = minioProc.Start(); err != nil {
return nil, fmt.Errorf("failed to start postgres: %w", err)
}

// wait for server to start
for i := 0; i < 200; i++ {
var conn net.Conn
conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", port))
if err == nil {
conn.Close()
break
}
time.Sleep(100 * time.Millisecond)
}

if err != nil {
return nil, fmt.Errorf("failed to connect to minio server: %w", err)
}
server := &minioServer{
cmd: minioProc,
tempDir: tempDir,
secret: secret,
port: port,
}
defer func() {
if err != nil {
server.Cleanup()
}
}()

return server, nil
}

// TODO: remove this test once we use minio in actual code
func TestServer_Miniotest(t *testing.T) {
server := createTestServer(t)
defer server.Close()
_, err := server.minioClient.BucketExists(context.Background(), server.bucketName)
ok(t, err)
}
3 changes: 3 additions & 0 deletions nix-cache-info
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
StoreDir: /nix/store
WantMassQuery: 1
Priority: 50
1 change: 1 addition & 0 deletions nix/devshells/flake-module.nix
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
pkgs.sqlc # type safe querying
pkgs.minio-client
pkgs.awscli
pkgs.minio
];

shellHook = ''
Expand Down
3 changes: 2 additions & 1 deletion nix/packages/flake-module.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@
name = "niks3";
src = ../..;

vendorHash = "sha256-PX0MYvoyZYYHYV7sMMXVbzDl+TpQjIJpAr4RBFxSmuQ=";
vendorHash = "sha256-Vqll61QhSmpN6GdL7L2ghUHtzpT9mhxfhyRgTNFVQyo=";

doCheck = true;
nativeCheckInputs = [
pkgs.postgresql
pkgs.minio-client
pkgs.minio
];
};
packages.default = config.packages.niks3;
Expand Down
18 changes: 16 additions & 2 deletions request_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,19 @@ import (
"os/exec"
"strconv"
"testing"
"time"

"github.com/Mic92/niks3/pg"
minio "github.com/minio/minio-go/v7"
)

func createTestServer(t *testing.T) *Server {
if testPostgresServer == nil {
t.Fatal("postgres server not started")
}
if testMinioServer == nil {
t.Fatal("minio server not started")
}

// create database for test
dbName := "db" + strconv.Itoa(int(testDbCount.Add(1)))
Expand All @@ -29,15 +34,24 @@ func createTestServer(t *testing.T) *Server {

connectionString := fmt.Sprintf("postgres://?dbname=%s&user=postgres&host=%s", dbName, testPostgresServer.tempDir)

ctx, cancel := context.WithTimeout(context.Background(), 10)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()

pool, err := pg.Connect(ctx, connectionString)
if err != nil {
ok(t, err)
}
// create bucket for test
bucketName := "bucket" + strconv.Itoa(int(testBucketCount.Add(1)))
minioClient := testMinioServer.Client(t)

err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
ok(t, err)

return &Server{
pool: pool,
pool: pool,
bucketName: bucketName,
minioClient: minioClient,
}
}

Expand Down
4 changes: 4 additions & 0 deletions reset-db.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -eux
sudo -u postgres dropdb niks3 || true
sudo -u postgres createdb niks3 -O "$USER"
Loading

0 comments on commit 680f33b

Please sign in to comment.