added enrollment migration (nano2nano)

.gitignore

@@ -1 +1,2 @@
 /nanomdm-*
+/nano2nano-*

Dockerfile

@@ -1,6 +1,7 @@
 FROM gcr.io/distroless/static
 
 COPY nanomdm-linux-amd64 /nanomdm
+COPY nano2nano-linux-amd64 /nano2nano
 
 EXPOSE 9000
 

Makefile

@@ -7,13 +7,21 @@ NANOMDM=\
 	nanomdm-darwin-arm64 \
 	nanomdm-linux-amd64
 
-my: nanomdm-$(OSARCH)
+NANO2NANO=\
+	nano2nano-darwin-amd64 \
+	nano2nano-darwin-arm64 \
+	nano2nano-linux-amd64
 
-docker: nanomdm-linux-amd64
+my: nanomdm-$(OSARCH) nano2nano-$(OSARCH)
+
+docker: nanomdm-linux-amd64 nano2nano-linux-amd64
 
 $(NANOMDM): cmd/nanomdm
 	GOOS=$(word 2,$(subst -, ,$@)) GOARCH=$(word 3,$(subst -, ,$(subst .exe,,$@))) go build $(LDFLAGS) -o $@ ./$<
 
+$(NANO2NANO): cmd/nano2nano
+	GOOS=$(word 2,$(subst -, ,$@)) GOARCH=$(word 3,$(subst -, ,$(subst .exe,,$@))) go build $(LDFLAGS) -o $@ ./$<
+
 %-$(VERSION).zip: %.exe
 	rm -f $@
 	zip $@ $<
@@ -23,11 +31,13 @@ $(NANOMDM): cmd/nanomdm
 	zip $@ $<
 
 clean:
-	rm -f nanomdm-*
+	rm -f nanomdm-* nano2nano-*
 
-release: $(foreach bin,$(NANOMDM),$(subst .exe,,$(bin))-$(VERSION).zip)
+release: \
+	$(foreach bin,$(NANOMDM),$(subst .exe,,$(bin))-$(VERSION).zip) \
+	$(foreach bin,$(NANO2NANO),$(subst .exe,,$(bin))-$(VERSION).zip)
 
 test:
 	go test -v -cover -race ./...
 
-.PHONY: my docker $(NANOMDM) clean release test
+.PHONY: my docker $(NANOMDM) $(NANO2NANO) clean release test

cmd/nano2nano/main.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	stdlog "log"
+	"net/http"
+
+	"github.com/jessepeterson/nanomdm/cmd/cli"
+	"github.com/jessepeterson/nanomdm/log/stdlogfmt"
+	"github.com/jessepeterson/nanomdm/mdm"
+)
+
+// overridden by -ldflags -X
+var version = "unknown"
+
+func main() {
+	cliStorage := cli.NewStorage()
+	flag.Var(&cliStorage.Storage, "storage", "name of storage system")
+	flag.Var(&cliStorage.DSN, "dsn", "data source name (e.g. connection string or path)")
+	var (
+		flVersion = flag.Bool("version", false, "print version")
+		flDebug   = flag.Bool("debug", false, "log debug messages")
+		flURL     = flag.String("url", "", "NanoMDM migration URL")
+		flAPIKey  = flag.String("key", "", "NanoMDM API Key")
+	)
+	flag.Parse()
+
+	if *flVersion {
+		fmt.Println(version)
+		return
+	}
+
+	logger := stdlogfmt.New(stdlog.Default(), *flDebug)
+
+	var skipServer bool
+	if *flURL == "" || *flAPIKey == "" {
+		logger.Info("msg", "URL or API key not set; not sending server requests")
+		skipServer = true
+	}
+	client := http.DefaultClient
+
+	mdmStorage, err := cliStorage.Parse(logger)
+	if err != nil {
+		stdlog.Fatal(err)
+	}
+
+	checkins := make(chan interface{})
+	ctx := context.Background()
+	go func() {
+		// dispatch to our storage backend to start sending the checkins
+		// channel our MDM check-in messages.
+		if err := mdmStorage.RetrieveMigrationCheckins(ctx, checkins); err != nil {
+			logger.Info(
+				"msg", "retrieving migration checkins",
+				"err", err,
+			)
+		}
+		close(checkins)
+	}()
+
+	// because order matters (a lot) we are purposefully single threaded for now.
+	for checkin := range checkins {
+		switch v := checkin.(type) {
+		case *mdm.Authenticate:
+			logger.Info(logsFromEnrollment("Authenticate", &v.Enrollment)...)
+			if !skipServer {
+				if err := httpPut(client, *flURL, *flAPIKey, v.Raw); err != nil {
+					logger.Info("msg", "sending to migration endpoint", "err", err)
+				}
+			}
+		case *mdm.TokenUpdate:
+			logger.Info(logsFromEnrollment("TokenUpdate", &v.Enrollment)...)
+			if !skipServer {
+				if err := httpPut(client, *flURL, *flAPIKey, v.Raw); err != nil {
+					logger.Info("msg", "sending to migration endpoint", "err", err)
+				}
+			}
+		case error:
+			logger.Info("msg", "receiving checkin", "err", v)
+		default:
+			logger.Info("msg", "invalid type provided")
+		}
+	}
+}
+
+func logsFromEnrollment(checkin string, e *mdm.Enrollment) []interface{} {
+	r := e.Resolved()
+	logs := []interface{}{
+		"checkin", checkin,
+		"device_id", r.DeviceChannelID,
+	}
+	if r.UserChannelID != "" {
+		logs = append(logs, "user_id", r.UserChannelID)
+	}
+	if e.UserShortName != "" {
+		logs = append(logs, "user_short_name", e.UserShortName)
+	}
+	logs = append(logs, "type", r.Type.String())
+	return logs
+}
+
+func httpPut(client *http.Client, url string, key string, sendBytes []byte) error {
+	if url == "" || key == "" {
+		return errors.New("no URL or API key")
+	}
+	req, err := http.NewRequest("PUT", url, bytes.NewReader(sendBytes))
+	if err != nil {
+		return err
+	}
+	req.SetBasicAuth("nanomdm", key)
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	_, err = ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != 200 {
+		return fmt.Errorf("Check-in Request failed with HTTP status: %d", res.StatusCode)
+	}
+	return nil
+}

docs/operations-guide.md

@@ -246,3 +246,41 @@ Of course the device won't check-in to retrieve this command, it will just sit i
 * Endpoint: `/migration`
 
 The migration endpoint (as talked about above under the `-migration` switch) is an API endpoint that allows sending raw `TokenUpdate` and `Authenticate` messages to establish an enrollment — in particular the APNs push topic, token, and push magic. This endpoint bypasses certificate validation and certificate authentication (though still requires API HTTP authentication). In this way we enable a way to "migrate" MDM enrollments from another MDM. This is how the `llorne` tool of [the micro2nano project](https://github.com/micromdm/micro2nano) works, for example.
+
+# Enrollment Migration (nano2nano)
+
+The `nano2nano` tool extracts migration enrollment data from a given storage backend and sends it to a NanoMDM migration endpoint. In this way you can effectively migrate between database backends. For example if you started with a `file` backend you could migrate to a `mysql` backend and vice versa. Note that MDM servers must have *exactly* the same server URL for migrations to operate.
+
+*Note:* Enrollment migration is **lossy**. It is not intended to bring over all data related to an enrollment — just the absolute bare minimum of data to support a migrated device being able to operate with MDM. For example previous commands & responses and even inventory data will be missing.
+
+*Note: There are some edge cases around enrollment migration. One such case is iOS unlock tokens. If the latest `TokenUpdate` did not contain the enroll-time unlock token for iOS then this information is probably lost in the migration. Again this feature is only meant to migrate the absolute minimum of information to allow for a device to be sent APNs push requests and have an operational command-queue.
+
+## Switches
+
+### -debug
+
+* log debug messages
+
+Enable additional debug logging.
+
+### -storage & -dsn
+
+See the "-storage & -dsn" section, above, for NanoMDM. The syntax and capabilities are the same.
+
+### -key string
+
+* NanoMDM API Key
+
+The NanoMDM API key used to authenticate to the migration endpoint.
+
+### -url string
+
+* NanoMDM migration URL
+
+The URL of the NanoMDM migration endpoint. For example "http://127.0.0.1:9000/migration".
+
+### -version
+
+* print version
+
+Print version and exit.

storage/all.go

@@ -7,4 +7,5 @@ type AllStorage interface {
 	PushCertStore
 	CommandEnqueuer
 	CertAuthStore
+	StoreMigrator
 }

storage/allmulti/migrate.go

@@ -0,0 +1,8 @@
+package allmulti
+
+import "context"
+
+func (ms *MultiAllStorage) RetrieveMigrationCheckins(ctx context.Context, c chan<- interface{}) error {
+	ms.logger.Info("msg", "only using first store for migration")
+	return ms.stores[0].RetrieveMigrationCheckins(ctx, c)
+}

storage/file/file.go

@@ -80,6 +80,16 @@ func (e *enrollment) readFile(name string) ([]byte, error) {
 	return ioutil.ReadFile(e.dirPrefix(name))
 }
 
+func (e *enrollment) fileExists(name string) (bool, error) {
+	if _, err := os.Stat(e.dirPrefix(name)); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
 // assocSubEnrollment writes an empty file of the sub (user) enrollment for tracking.
 func (e *enrollment) assocSubEnrollment(id string) error {
 	subPath := e.dirPrefix(SubEnrollmentPathname)
@@ -149,7 +159,7 @@ func (s *FileStorage) StoreTokenUpdate(r *mdm.Request, msg *mdm.TokenUpdate) err
 		return err
 	}
 	// delete the disabled flag to let signify this enrollment is enabled
-	if err := os.Remove(e.dirPrefix(DisabledFilename)); err != nil && !errors.Is(err, os.ErrExist) {
+	if err := os.Remove(e.dirPrefix(DisabledFilename)); err != nil && !errors.Is(err, os.ErrNotExist) {
 		return err
 	}
 	return nil

storage/file/migrate.go

@@ -0,0 +1,63 @@
+package file
+
+import (
+	"context"
+	"os"
+
+	"github.com/jessepeterson/nanomdm/mdm"
+)
+
+func sendCheckinMessage(e *enrollment, filename string, c chan<- interface{}) {
+	msgBytes, err := e.readFile(filename)
+	if err != nil {
+		c <- err
+		return
+	}
+	msg, err := mdm.DecodeCheckin(msgBytes)
+	if err != nil {
+		c <- err
+		return
+	}
+	c <- msg
+}
+
+func (s *FileStorage) RetrieveMigrationCheckins(_ context.Context, c chan<- interface{}) error {
+	for _, userLoop := range []bool{false, true} {
+		entries, err := os.ReadDir(s.path)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			if !entry.IsDir() {
+				continue
+			}
+			e := s.newEnrollment(entry.Name())
+			authExists, err := e.fileExists(AuthenticateFilename)
+			if err != nil {
+				c <- err
+			}
+			// if an Authenticate doesn't exist then this is a
+			// user-channel enrollment. skip it for this loop
+			if !userLoop && !authExists {
+				continue
+			}
+			if !userLoop {
+				sendCheckinMessage(e, AuthenticateFilename, c)
+			}
+			tokExists, err := e.fileExists(TokenUpdateFilename)
+			if err != nil {
+				c <- err
+			}
+			// if neither an authenticate nor tokenupdate exists then
+			// this is an invalid enrollment and we should skip it
+			if !tokExists && !authExists {
+				continue
+			}
+			// TODO: if we have an UnlockToken for a device we
+			// should synthesize it into a TokenUpdate message because
+			// they are saved out-of-band.
+			sendCheckinMessage(e, TokenUpdateFilename, c)
+		}
+	}
+	return nil
+}

storage/mysql/migrate.go

@@ -0,0 +1,61 @@
+package mysql
+
+import (
+	"context"
+
+	"github.com/jessepeterson/nanomdm/mdm"
+)
+
+func (s *MySQLStorage) RetrieveMigrationCheckins(ctx context.Context, c chan<- interface{}) error {
+	// TODO: if a TokenUpdate does not include the latest UnlockToken
+	// then we should synthesize a TokenUpdate to transfer it over.
+	deviceRows, err := s.db.QueryContext(
+		ctx,
+		`SELECT authenticate, token_update FROM devices;`,
+	)
+	if err != nil {
+		return err
+	}
+	defer deviceRows.Close()
+	for deviceRows.Next() {
+		var authBytes, tokenBytes []byte
+		if err := deviceRows.Scan(&authBytes, &tokenBytes); err != nil {
+			return err
+		}
+		for _, msgBytes := range [][]byte{authBytes, tokenBytes} {
+			msg, err := mdm.DecodeCheckin(msgBytes)
+			if err != nil {
+				c <- err
+			} else {
+				c <- msg
+			}
+		}
+	}
+	if err = deviceRows.Err(); err != nil {
+		return err
+	}
+	userRows, err := s.db.QueryContext(
+		ctx,
+		`SELECT token_update FROM users;`,
+	)
+	if err != nil {
+		return err
+	}
+	defer userRows.Close()
+	for userRows.Next() {
+		var msgBytes []byte
+		if err := userRows.Scan(&msgBytes); err != nil {
+			return err
+		}
+		msg, err := mdm.DecodeCheckin(msgBytes)
+		if err != nil {
+			c <- err
+		} else {
+			c <- msg
+		}
+	}
+	if err = userRows.Err(); err != nil {
+		return err
+	}
+	return nil
+}