Skip to content

Commit

Permalink
PostgreSQL storage backend (#51)
Browse files Browse the repository at this point in the history
  • Loading branch information
sheshenia authored Jul 15, 2022
1 parent 3368d22 commit 9ca3cac
Show file tree
Hide file tree
Showing 15 changed files with 1,216 additions and 2 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ A brief overview of the various command-line switches and HTTP endpoints and API

## Features

- Horizontal scaling: zero/minimal local state. Persistence in storage layers. MySQL backend provided in the box.
- Horizontal scaling: zero/minimal local state. Persistence in storage layers. MySQL and PostgreSQL backends provided in the box.
- Multiple APNs topics: potentially multi-tenant.
- Multi-command targeting: send the same command (or pushes) to multiple enrollments without individually queuing commands.
- Migration endpoint: allow migrating MDM enrollments between storage backends or (supported) MDM servers
Expand Down
32 changes: 32 additions & 0 deletions cmd/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ import (
"github.com/micromdm/nanomdm/storage/allmulti"
"github.com/micromdm/nanomdm/storage/file"
"github.com/micromdm/nanomdm/storage/mysql"
"github.com/micromdm/nanomdm/storage/pgsql"

_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
)

type StringAccumulator []string
Expand Down Expand Up @@ -72,6 +74,12 @@ func (s *Storage) Parse(logger log.Logger) (storage.AllStorage, error) {
return nil, err
}
mdmStorage = append(mdmStorage, mysqlStorage)
case "pgsql":
pgsqlStorage, err := pgsqlStorageConfig(dsn, options, logger)
if err != nil {
return nil, err
}
mdmStorage = append(mdmStorage, pgsqlStorage)
default:
return nil, fmt.Errorf("unknown storage: %s", storage)
}
Expand Down Expand Up @@ -134,3 +142,27 @@ func splitOptions(s string) map[string]string {
}
return out
}

func pgsqlStorageConfig(dsn, options string, logger log.Logger) (*pgsql.PgSQLStorage, error) {
logger = logger.With("storage", "pgsql")
opts := []pgsql.Option{
pgsql.WithDSN(dsn),
pgsql.WithLogger(logger),
}
if options != "" {
for k, v := range splitOptions(options) {
switch k {
case "delete":
if v == "1" {
opts = append(opts, pgsql.WithDeleteCommands())
logger.Debug("msg", "deleting commands")
} else if v != "0" {
return nil, fmt.Errorf("invalid value for delete option: %q", v)
}
default:
return nil, fmt.Errorf("invalid option: %q", k)
}
}
}
return pgsql.New(opts...)
}
14 changes: 14 additions & 0 deletions docs/operations-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,20 @@ Options are specified as a comma-separated list of "key=value" pairs. The mysql

*Example:* `-storage mysql -dsn nanomdm:nanomdm/mymdmdb -storage-options delete=1`

#### pgsql storage backend

* `-storage pgsql`

Configures the PostgreSQL storage backend. The `-dsn` flag should be in the [format the SQL driver expects](https://pkg.go.dev/github.com/lib/pq#pkg-overview). Be sure to create your tables with the [schema.sql](../storage/pgsql/schema.sql) file that corresponds to your NanoMDM version. Also make sure you apply any schema changes for each updated version (i.e. execute the numbered schema change files). PostgreSQL 9.5 or later is required.

*Example:* `-storage pgsql -dsn postgres://postgres:toor@localhost:5432/nanomdm?sslmode=disable`

Options are specified as a comma-separated list of "key=value" pairs. The pgsql backend supports these options:
* `delete=1`, `delete=0`
* This option turns on or off the command and response deleter. It is disabled by default. When enabled (with `delete=1`) command responses, queued commands, and commands themselves will be deleted from the database after enrollments have responded to a command.

*Example:* `-storage pgsql -dsn postgres://postgres:toor@localhost/nanomdm -storage-options delete=1`

#### multi-storage backend

You can configure multiple storage backends to be used simultaneously. Specifying multiple sets of `-storage`, `-dsn`, & `-storage-options` flags will configure the "multi-storage" adapter. The flags must be specified in sets and are related to each other in the order they're specified: for example the first `-storage` flag corresponds to the first `-dsn` flag and so forth.
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ require (
github.com/RobotsAndPencils/buford v0.14.0
github.com/go-sql-driver/mysql v1.6.0
github.com/groob/plist v0.0.0-20220217120414-63fa881b19a5
github.com/lib/pq v1.10.6
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/groob/plist v0.0.0-20220217120414-63fa881b19a5 h1:saaSiB25B1wgaxrshQhurfPKUGJ4It3OxNJUy0rdOjU=
github.com/groob/plist v0.0.0-20220217120414-63fa881b19a5/go.mod h1:itkABA+w2cw7x5nYUS/pLRef6ludkZKOigbROmCTaFw=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
2 changes: 1 addition & 1 deletion storage/mysql/mysql.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Pacakge mysql stores and retrieves MDM data from SQL
// Package mysql stores and retrieves MDM data from MySQL
package mysql

import (
Expand Down
36 changes: 36 additions & 0 deletions storage/pgsql/bstoken.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package pgsql

import (
"github.com/micromdm/nanomdm/mdm"
)

func (s *PgSQLStorage) StoreBootstrapToken(r *mdm.Request, msg *mdm.SetBootstrapToken) error {
_, err := s.db.ExecContext(
r.Context,
`UPDATE devices SET bootstrap_token_b64 = $1, bootstrap_token_at = CURRENT_TIMESTAMP WHERE id = $2;`,
nullEmptyString(msg.BootstrapToken.BootstrapToken.String()),
r.ID,
)
if err != nil {
return err
}
return s.updateLastSeen(r)
}

func (s *PgSQLStorage) RetrieveBootstrapToken(r *mdm.Request, _ *mdm.GetBootstrapToken) (*mdm.BootstrapToken, error) {
var tokenB64 string
err := s.db.QueryRowContext(
r.Context,
`SELECT bootstrap_token_b64 FROM devices WHERE id = $1;`,
r.ID,
).Scan(&tokenB64)
if err != nil {
return nil, err
}
bsToken := new(mdm.BootstrapToken)
err = bsToken.SetTokenString(tokenB64)
if err == nil {
err = s.updateLastSeen(r)
}
return bsToken, err
}
52 changes: 52 additions & 0 deletions storage/pgsql/certauth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package pgsql

import (
"context"
"strings"

"github.com/micromdm/nanomdm/mdm"
)

// Executes SQL statements that return a single COUNT(*) of rows.
func (s *PgSQLStorage) queryRowContextRowExists(ctx context.Context, query string, args ...interface{}) (bool, error) {
var ct int
err := s.db.QueryRowContext(ctx, query, args...).Scan(&ct)
return ct > 0, err
}

func (s *PgSQLStorage) EnrollmentHasCertHash(r *mdm.Request, _ string) (bool, error) {
return s.queryRowContextRowExists(
r.Context,
`SELECT COUNT(*) FROM cert_auth_associations WHERE id = $1;`,
r.ID,
)
}

func (s *PgSQLStorage) HasCertHash(r *mdm.Request, hash string) (bool, error) {
return s.queryRowContextRowExists(
r.Context,
`SELECT COUNT(*) FROM cert_auth_associations WHERE sha256 = $1;`,
strings.ToLower(hash),
)
}

func (s *PgSQLStorage) IsCertHashAssociated(r *mdm.Request, hash string) (bool, error) {
return s.queryRowContextRowExists(
r.Context,
`SELECT COUNT(*) FROM cert_auth_associations WHERE id = $1 AND sha256 = $2;`,
r.ID, strings.ToLower(hash),
)
}

// AssociateCertHash "DO NOTHING" on duplicated keys
func (s *PgSQLStorage) AssociateCertHash(r *mdm.Request, hash string) error {
_, err := s.db.ExecContext(
r.Context, `
INSERT INTO cert_auth_associations (id, sha256)
VALUES ($1, $2)
ON CONFLICT ON CONSTRAINT cert_auth_associations_pkey DO UPDATE SET updated_at=now();`,
r.ID,
strings.ToLower(hash),
)
return err
}
61 changes: 61 additions & 0 deletions storage/pgsql/migrate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package pgsql

import (
"context"

"github.com/micromdm/nanomdm/mdm"
)

func (s *PgSQLStorage) RetrieveMigrationCheckins(ctx context.Context, c chan<- interface{}) error {
// TODO: if a TokenUpdate does not include the latest UnlockToken
// then we should synthesize a TokenUpdate to transfer it over.
deviceRows, err := s.db.QueryContext(
ctx,
`SELECT authenticate, token_update FROM devices;`,
)
if err != nil {
return err
}
defer deviceRows.Close()
for deviceRows.Next() {
var authBytes, tokenBytes []byte
if err := deviceRows.Scan(&authBytes, &tokenBytes); err != nil {
return err
}
for _, msgBytes := range [][]byte{authBytes, tokenBytes} {
msg, err := mdm.DecodeCheckin(msgBytes)
if err != nil {
c <- err
} else {
c <- msg
}
}
}
if err = deviceRows.Err(); err != nil {
return err
}
userRows, err := s.db.QueryContext(
ctx,
`SELECT token_update FROM users;`,
)
if err != nil {
return err
}
defer userRows.Close()
for userRows.Next() {
var msgBytes []byte
if err := userRows.Scan(&msgBytes); err != nil {
return err
}
msg, err := mdm.DecodeCheckin(msgBytes)
if err != nil {
c <- err
} else {
c <- msg
}
}
if err = userRows.Err(); err != nil {
return err
}
return nil
}
Loading

0 comments on commit 9ca3cac

Please sign in to comment.