diff --git a/go.mod b/go.mod index b51dcf0..a06d425 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,31 @@ module github.com/Mic92/niks3 go 1.22.7 require ( + github.com/jackc/pgx/v5 v5.7.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 + github.com/minio/minio-go/v7 v7.0.79 github.com/pressly/goose/v3 v3.22.1 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mfridman/interpolate v0.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 5ff7883..c00ed61 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -21,6 +25,11 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -29,6 +38,10 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.79 h1:SvJZpj3hT0RN+4KiuX/FxLfPZdsuegy6d/2PiemM/bM= +github.com/minio/minio-go/v7 v7.0.79/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -37,6 +50,8 @@ github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuio github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -48,12 +63,21 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main_test.go b/main_test.go index c053c7a..7ac0b3c 100644 --- a/main_test.go +++ b/main_test.go @@ -8,12 +8,25 @@ import ( func innerTestMain(m *testing.M) int { var err error + + // unload environment variables from the devenv + os.Unsetenv("DATABASE_URL") + os.Unsetenv("PGDATABASE") + os.Unsetenv("PGUSER") + os.Unsetenv("PGHOST") + testPostgresServer, err = startPostgresServer() defer testPostgresServer.Cleanup() if err != nil { slog.Error("failed to start postgres", "error", err) return 1 } + testMinioServer, err = startMinioServer() + defer testMinioServer.Cleanup() + if err != nil { + slog.Error("failed to start minio", "error", err) + os.Exit(1) + } return m.Run() } diff --git a/minio_test.go b/minio_test.go new file mode 100644 index 0000000..f82db64 --- /dev/null +++ b/minio_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "net" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "syscall" + "testing" + "time" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + testMinioServer *minioServer + testBucketCount atomic.Int32 +) + +type minioServer struct { + cmd *exec.Cmd + tempDir string + secret string + port uint16 +} + +func randToken(n int) (string, error) { + bytes := make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes), nil +} + +func randPort() (uint16, error) { + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + return 0, err + } + ln.Close() + time.Sleep(1 * time.Second) + return (uint16)(ln.Addr().(*net.TCPAddr).Port), nil +} + +func (s *minioServer) Client(t *testing.T) *minio.Client { + endpoint := fmt.Sprintf("localhost:%d", s.port) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4("minioadmin", s.secret, ""), + Secure: false, + }) + ok(t, err) + return minioClient +} + +func (s *minioServer) Cleanup() { + err := syscall.Kill(s.cmd.Process.Pid, syscall.SIGKILL) + if err != nil { + slog.Error("failed to kill postgres", "error", err) + } + err = s.cmd.Wait() + if err != nil { + slog.Error("failed to wait for postgres", "error", err) + } + + os.RemoveAll(s.tempDir) +} + +func startMinioServer() (*minioServer, error) { + tempDir, err := os.MkdirTemp("", "minio") + if err != nil { + return nil, fmt.Errorf("failed to create temp dir: %w", err) + } + defer func() { + if err != nil { + os.RemoveAll(tempDir) + } + }() + + port, err := randPort() + if err != nil { + return nil, fmt.Errorf("failed to find free port: %w", err) + } + + fmt.Printf("####################### Command: minio server --address :%d %s\n", port, filepath.Join(tempDir, "data")) + + minioProc := exec.Command("minio", "server", "--address", fmt.Sprintf(":%d", port), filepath.Join(tempDir, "data")) + minioProc.Stdout = os.Stdout + minioProc.Stderr = os.Stderr + + // random hex string + secret, err := randToken(20) + if err != nil { + return nil, fmt.Errorf("failed to generate access key: %w", err) + } + + env := os.Environ() + env = append(env, "MINIO_ROOT_USER=minioadmin") + env = append(env, fmt.Sprintf("MINIO_ROOT_PASSWORD=%s", secret)) + env = append(env, "AWS_ACCESS_KEY_ID=minioadmin") + env = append(env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", secret)) + minioProc.Env = env + + if err = minioProc.Start(); err != nil { + return nil, fmt.Errorf("failed to start postgres: %w", err) + } + + // wait for server to start + for i := 0; i < 200; i++ { + var conn net.Conn + conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", port)) + if err == nil { + conn.Close() + break + } + time.Sleep(100 * time.Millisecond) + } + + if err != nil { + return nil, fmt.Errorf("failed to connect to minio server: %w", err) + } + server := &minioServer{ + cmd: minioProc, + tempDir: tempDir, + secret: secret, + port: port, + } + defer func() { + if err != nil { + server.Cleanup() + } + }() + + return server, nil +} + +// TODO: remove this test once we use minio in actual code +func TestServer_Miniotest(t *testing.T) { + server := createTestServer(t) + defer server.Close() + _, err := server.minioClient.BucketExists(context.Background(), server.bucketName) + ok(t, err) +} diff --git a/nix-cache-info b/nix-cache-info new file mode 100644 index 0000000..f6b8231 --- /dev/null +++ b/nix-cache-info @@ -0,0 +1,3 @@ +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 50 diff --git a/nix/devshells/flake-module.nix b/nix/devshells/flake-module.nix index 1fa9778..e264c1e 100644 --- a/nix/devshells/flake-module.nix +++ b/nix/devshells/flake-module.nix @@ -23,6 +23,7 @@ pkgs.sqlc # type safe querying pkgs.minio-client pkgs.awscli + pkgs.minio ]; shellHook = '' diff --git a/nix/packages/flake-module.nix b/nix/packages/flake-module.nix index 56978db..a1e2705 100644 --- a/nix/packages/flake-module.nix +++ b/nix/packages/flake-module.nix @@ -6,12 +6,13 @@ name = "niks3"; src = ../..; - vendorHash = "sha256-PX0MYvoyZYYHYV7sMMXVbzDl+TpQjIJpAr4RBFxSmuQ="; + vendorHash = "sha256-Vqll61QhSmpN6GdL7L2ghUHtzpT9mhxfhyRgTNFVQyo="; doCheck = true; nativeCheckInputs = [ pkgs.postgresql pkgs.minio-client + pkgs.minio ]; }; packages.default = config.packages.niks3; diff --git a/request_test.go b/request_test.go index 936b5c4..3413fe0 100644 --- a/request_test.go +++ b/request_test.go @@ -10,14 +10,19 @@ import ( "os/exec" "strconv" "testing" + "time" "github.com/Mic92/niks3/pg" + minio "github.com/minio/minio-go/v7" ) func createTestServer(t *testing.T) *Server { if testPostgresServer == nil { t.Fatal("postgres server not started") } + if testMinioServer == nil { + t.Fatal("minio server not started") + } // create database for test dbName := "db" + strconv.Itoa(int(testDbCount.Add(1))) @@ -29,15 +34,24 @@ func createTestServer(t *testing.T) *Server { connectionString := fmt.Sprintf("postgres://?dbname=%s&user=postgres&host=%s", dbName, testPostgresServer.tempDir) - ctx, cancel := context.WithTimeout(context.Background(), 10) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() pool, err := pg.Connect(ctx, connectionString) if err != nil { ok(t, err) } + // create bucket for test + bucketName := "bucket" + strconv.Itoa(int(testBucketCount.Add(1))) + minioClient := testMinioServer.Client(t) + + err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) + ok(t, err) + return &Server{ - pool: pool, + pool: pool, + bucketName: bucketName, + minioClient: minioClient, } } diff --git a/reset-db.sh b/reset-db.sh new file mode 100755 index 0000000..4838b36 --- /dev/null +++ b/reset-db.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -eux +sudo -u postgres dropdb niks3 || true +sudo -u postgres createdb niks3 -O "$USER" diff --git a/server.go b/server.go index 45401da..c7d198a 100644 --- a/server.go +++ b/server.go @@ -9,16 +9,28 @@ import ( "github.com/Mic92/niks3/pg" "github.com/jackc/pgx/v5/pgxpool" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) type Options struct { DBConnectionString string HTTPAddr string MigrateDB bool + + // TODO: Document how to use this with AWS. + S3Endpoint string + S3AccessKey string + S3SecretKey string + S3UseSSL bool + S3BucketName string } type Server struct { - pool *pgxpool.Pool + pool *pgxpool.Pool + minioClient *minio.Client + bucketName string } func RunServer(opts *Options) error { @@ -31,7 +43,15 @@ func RunServer(opts *Options) error { } defer pool.Close() - service := &Server{pool: pool} + minioClient, err := minio.New(opts.S3Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(opts.S3AccessKey, opts.S3SecretKey, ""), + Secure: opts.S3UseSSL, + }) + if err != nil { + return fmt.Errorf("failed to create minio s3 client: %w", err) + } + + service := &Server{pool: pool, minioClient: minioClient, bucketName: opts.S3BucketName} mux := http.NewServeMux() mux.HandleFunc("/health", service.healthCheckHandler)