diff --git a/cmd/rekor-cli/app/get.go b/cmd/rekor-cli/app/get.go index ee1b1085a..440b10a18 100644 --- a/cmd/rekor-cli/app/get.go +++ b/cmd/rekor-cli/app/get.go @@ -112,22 +112,28 @@ var getCmd = &cobra.Command{ } } + // Note: this UUID may be an EntryID if uuid != "" { params := entries.NewGetLogEntryByUUIDParams() params.SetTimeout(viper.GetDuration("timeout")) - params.EntryUUID, err = sharding.GetUUIDFromIDString(uuid) + // NOTE: This undoes the change that let people pass in longer UUIDs without + // trouble even if their client is old, a.k.a. it will be able to use the TreeID + // (if present) for routing in the GetLogEntryByUUIDHandler + params.EntryUUID = uuid + + resp, err := rekorClient.Entries.GetLogEntryByUUID(params) if err != nil { - return nil, fmt.Errorf("unable to parse uuid: %w", err) + return nil, err } - resp, err := rekorClient.Entries.GetLogEntryByUUID(params) + u, err := sharding.GetUUIDFromIDString(params.EntryUUID) if err != nil { return nil, err } for k, entry := range resp.Payload { - if k != params.EntryUUID { + if k != u { continue } diff --git a/cmd/rekor-cli/app/log_info.go b/cmd/rekor-cli/app/log_info.go index 0bfc92530..53fcfac89 100644 --- a/cmd/rekor-cli/app/log_info.go +++ b/cmd/rekor-cli/app/log_info.go @@ -43,6 +43,7 @@ type logInfoCmdOutput struct { TreeSize int64 RootHash string TimestampNanos uint64 + TreeID int64 } func (l *logInfoCmdOutput) String() string { @@ -52,7 +53,8 @@ func (l *logInfoCmdOutput) String() string { Tree Size: %v Root Hash: %s Timestamp: %s -`, l.TreeSize, l.RootHash, ts) +TreeID: %v +`, l.TreeSize, l.RootHash, ts, l.TreeID) } // logInfoCmd represents the current information about the transparency log @@ -114,6 +116,7 @@ var logInfoCmd = &cobra.Command{ TreeSize: *logInfo.TreeSize, RootHash: *logInfo.RootHash, TimestampNanos: sth.GetTimestamp(), + TreeID: *logInfo.TreeID, } oldState := state.Load(serverURL) diff --git a/cmd/rekor-server/app/flags.go b/cmd/rekor-server/app/flags.go index 7cb29a6ac..ba6d748ed 100644 --- a/cmd/rekor-server/app/flags.go +++ b/cmd/rekor-server/app/flags.go @@ -41,11 +41,11 @@ func (l *LogRangesFlag) Set(s string) error { return fmt.Errorf("invalid range flag, expected two parts separated by an =, got %s", r) } lr := sharding.LogRange{} - lr.TreeID, err = strconv.ParseUint(split[0], 10, 64) + lr.TreeID, err = strconv.ParseInt(split[0], 10, 64) if err != nil { return err } - lr.TreeLength, err = strconv.ParseUint(split[1], 10, 64) + lr.TreeLength, err = strconv.ParseInt(split[1], 10, 64) if err != nil { return err } @@ -55,7 +55,7 @@ func (l *LogRangesFlag) Set(s string) error { // The last entry is special and should not have a terminating range, because this is active. lastRangeStr := ranges[len(ranges)-1] - lastTreeID, err := strconv.ParseUint(lastRangeStr, 10, 64) + lastTreeID, err := strconv.ParseInt(lastRangeStr, 10, 64) if err != nil { return err } @@ -65,7 +65,7 @@ func (l *LogRangesFlag) Set(s string) error { }) // Look for duplicate tree ids - TreeIDs := map[uint64]struct{}{} + TreeIDs := map[int64]struct{}{} for _, lr := range inputRanges { if _, ok := TreeIDs[lr.TreeID]; ok { return fmt.Errorf("duplicate tree id: %d", lr.TreeID) diff --git a/cmd/rekor-server/app/flags_test.go b/cmd/rekor-server/app/flags_test.go index 60faba544..e9028759b 100644 --- a/cmd/rekor-server/app/flags_test.go +++ b/cmd/rekor-server/app/flags_test.go @@ -27,7 +27,7 @@ func TestLogRanges_Set(t *testing.T) { name string arg string want []sharding.LogRange - active uint64 + active int64 }{ { name: "one, no length", diff --git a/cmd/rekor-server/app/root.go b/cmd/rekor-server/app/root.go index 27f4b1017..f22ef417d 100644 --- a/cmd/rekor-server/app/root.go +++ b/cmd/rekor-server/app/root.go @@ -65,7 +65,7 @@ func init() { rootCmd.PersistentFlags().String("trillian_log_server.address", "127.0.0.1", "Trillian log server address") rootCmd.PersistentFlags().Uint16("trillian_log_server.port", 8090, "Trillian log server port") rootCmd.PersistentFlags().Uint("trillian_log_server.tlog_id", 0, "Trillian tree id") - rootCmd.PersistentFlags().Var(&logRangeMap, "trillian_log_server.log_id_ranges", "ordered list of tree ids and ranges") + rootCmd.PersistentFlags().String("trillian_log_server.log_id_ranges", "", "ordered list of tree ids and ranges") rootCmd.PersistentFlags().String("rekor_server.hostname", "rekor.sigstore.dev", "public hostname of instance") rootCmd.PersistentFlags().String("rekor_server.address", "127.0.0.1", "Address to bind to") diff --git a/cmd/rekor-server/app/serve.go b/cmd/rekor-server/app/serve.go index 4cde11b85..44cae749a 100644 --- a/cmd/rekor-server/app/serve.go +++ b/cmd/rekor-server/app/serve.go @@ -103,6 +103,14 @@ var serveCmd = &cobra.Command{ server.Port = int(viper.GetUint("port")) server.EnabledListeners = []string{"http"} + // Update logRangeMap if flag was passed in + rangeMap := viper.GetString("trillian_log_server.log_id_ranges") + if rangeMap != "" { + if err := logRangeMap.Set(rangeMap); err != nil { + log.Logger.Fatal("unable to set logRangeMap from flag: %v", err) + } + } + api.ConfigureAPI(logRangeMap.Ranges) server.ConfigureAPI() diff --git a/openapi.yaml b/openapi.yaml index 05e55dd62..28ab38c3d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -581,10 +581,14 @@ definitions: type: string format: signedCheckpoint description: The current signed tree head + treeID: + type: integer + description: The current treeID required: - rootHash - treeSize - signedTreeHead + - treeID ConsistencyProof: type: object diff --git a/pkg/api/api.go b/pkg/api/api.go index ae98b678d..0b464800b 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -85,7 +85,10 @@ func NewAPI(ranges sharding.LogRanges) (*API, error) { return nil, errors.Wrap(err, "create and init tree") } tLogID = t.TreeId + log.Logger.Infof("Creating new tree with ID: %v", t.TreeId) } + // append the active treeID to the API's logRangeMap for lookups + ranges.Ranges = append(ranges.Ranges, sharding.LogRange{TreeID: tLogID}) rekorSigner, err := signer.New(ctx, viper.GetString("rekor_server.signer")) if err != nil { diff --git a/pkg/api/entries.go b/pkg/api/entries.go index 892c9a8d1..4be145b8b 100644 --- a/pkg/api/entries.go +++ b/pkg/api/entries.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer" "github.com/go-openapi/runtime/middleware" @@ -117,9 +118,11 @@ func logEntryFromLeaf(ctx context.Context, signer signature.Signer, tc TrillianC // GetLogEntryAndProofByIndexHandler returns the entry and inclusion proof for a specified log index func GetLogEntryByIndexHandler(params entries.GetLogEntryByIndexParams) middleware.Responder { ctx := params.HTTPRequest.Context() - tc := NewTrillianClient(ctx) + tid, resolvedIndex := api.logRanges.ResolveVirtualIndex(int(params.LogIndex)) + tc := NewTrillianClientFromTreeID(ctx, tid) + log.RequestIDLogger(params.HTTPRequest).Debugf("Retrieving resolved index %v from TreeID %v", resolvedIndex, tid) - resp := tc.getLeafAndProofByIndex(params.LogIndex) + resp := tc.getLeafAndProofByIndex(resolvedIndex) switch resp.status { case codes.OK: case codes.NotFound, codes.OutOfRange, codes.InvalidArgument: @@ -268,12 +271,30 @@ func getEntryURL(locationURL url.URL, uuid string) strfmt.URI { func GetLogEntryByUUIDHandler(params entries.GetLogEntryByUUIDParams) middleware.Responder { ctx := params.HTTPRequest.Context() - entryUUID, err := sharding.GetUUIDFromIDString(params.EntryUUID) + uuid, err := sharding.GetUUIDFromIDString(params.EntryUUID) if err != nil { return handleRekorAPIError(params, http.StatusBadRequest, err, "") } - hashValue, _ := hex.DecodeString(entryUUID) - tc := NewTrillianClient(params.HTTPRequest.Context()) + var tid int64 + tidString, err := sharding.GetTreeIDFromIDString(params.EntryUUID) + if err != nil { + // If EntryID is plain UUID, assume no sharding and use ActiveIndex. The ActiveIndex + // will == the tlog_id if a tlog_id is passed in at server startup. + if err.Error() == "cannot get treeID from plain UUID" { + tid = api.logRanges.ActiveIndex() + } else { + return handleRekorAPIError(params, http.StatusBadRequest, err, "") + } + } else { + tid, err = strconv.ParseInt(tidString, 16, 64) + if err != nil { + return handleRekorAPIError(params, http.StatusBadRequest, err, "") + } + } + hashValue, _ := hex.DecodeString(uuid) + + tc := NewTrillianClientFromTreeID(params.HTTPRequest.Context(), tid) + log.RequestIDLogger(params.HTTPRequest).Debugf("Retrieving UUID %v from TreeID %v", uuid, tid) resp := tc.getLeafAndProofByHash(hashValue) switch resp.status { diff --git a/pkg/api/tlog.go b/pkg/api/tlog.go index c3fa516e7..6ad2fe813 100644 --- a/pkg/api/tlog.go +++ b/pkg/api/tlog.go @@ -76,7 +76,9 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder { RootHash: &hashString, TreeSize: &treeSize, SignedTreeHead: &scString, + TreeID: &tc.logID, } + return tlog.NewGetLogInfoOK().WithPayload(&logInfo) } diff --git a/pkg/api/trillian_client.go b/pkg/api/trillian_client.go index 6cc1ca0c0..cb64c0eee 100644 --- a/pkg/api/trillian_client.go +++ b/pkg/api/trillian_client.go @@ -48,6 +48,14 @@ func NewTrillianClient(ctx context.Context) TrillianClient { } } +func NewTrillianClientFromTreeID(ctx context.Context, tid int64) TrillianClient { + return TrillianClient{ + client: api.logClient, + logID: tid, + context: ctx, + } +} + type Response struct { status codes.Code err error diff --git a/pkg/generated/models/log_info.go b/pkg/generated/models/log_info.go index b576bf015..15eae37fe 100644 --- a/pkg/generated/models/log_info.go +++ b/pkg/generated/models/log_info.go @@ -44,6 +44,10 @@ type LogInfo struct { // Required: true SignedTreeHead *string `json:"signedTreeHead"` + // The current treeID + // Required: true + TreeID *int64 `json:"treeID"` + // The current number of nodes in the merkle tree // Required: true // Minimum: 1 @@ -62,6 +66,10 @@ func (m *LogInfo) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateTreeID(formats); err != nil { + res = append(res, err) + } + if err := m.validateTreeSize(formats); err != nil { res = append(res, err) } @@ -94,6 +102,15 @@ func (m *LogInfo) validateSignedTreeHead(formats strfmt.Registry) error { return nil } +func (m *LogInfo) validateTreeID(formats strfmt.Registry) error { + + if err := validate.Required("treeID", "body", m.TreeID); err != nil { + return err + } + + return nil +} + func (m *LogInfo) validateTreeSize(formats strfmt.Registry) error { if err := validate.Required("treeSize", "body", m.TreeSize); err != nil { diff --git a/pkg/sharding/ranges.go b/pkg/sharding/ranges.go index 44301ec93..3358855b0 100644 --- a/pkg/sharding/ranges.go +++ b/pkg/sharding/ranges.go @@ -20,24 +20,24 @@ type LogRanges struct { } type LogRange struct { - TreeID uint64 - TreeLength uint64 + TreeID int64 + TreeLength int64 } -func (l *LogRanges) ResolveVirtualIndex(index int) (uint64, uint64) { +func (l *LogRanges) ResolveVirtualIndex(index int) (int64, int64) { indexLeft := index for _, l := range l.Ranges { if indexLeft < int(l.TreeLength) { - return l.TreeID, uint64(indexLeft) + return l.TreeID, int64(indexLeft) } indexLeft -= int(l.TreeLength) } // Return the last one! - return l.Ranges[len(l.Ranges)-1].TreeID, uint64(indexLeft) + return l.Ranges[len(l.Ranges)-1].TreeID, int64(indexLeft) } // ActiveIndex returns the active shard index, always the last shard in the range -func (l *LogRanges) ActiveIndex() uint64 { +func (l *LogRanges) ActiveIndex() int64 { return l.Ranges[len(l.Ranges)-1].TreeID } diff --git a/pkg/sharding/ranges_test.go b/pkg/sharding/ranges_test.go index d8fe1b705..2249ea3c2 100644 --- a/pkg/sharding/ranges_test.go +++ b/pkg/sharding/ranges_test.go @@ -29,8 +29,8 @@ func TestLogRanges_ResolveVirtualIndex(t *testing.T) { for _, tt := range []struct { Index int - WantTreeID uint64 - WantIndex uint64 + WantTreeID int64 + WantIndex int64 }{ { Index: 3, diff --git a/pkg/sharding/sharding.go b/pkg/sharding/sharding.go index 503b4e117..8a7063faa 100644 --- a/pkg/sharding/sharding.go +++ b/pkg/sharding/sharding.go @@ -38,20 +38,12 @@ const TreeIDHexStringLen = 16 const UUIDHexStringLen = 64 const EntryIDHexStringLen = TreeIDHexStringLen + UUIDHexStringLen -// TODO: replace this with the actual LogRanges struct when logic is hooked up -var dummyLogRanges = LogRanges{ - Ranges: []LogRange{ - { - TreeID: 0, - TreeLength: 0}}, -} - type EntryID struct { TreeID string UUID string } -// This function can take a TreeID of equal or greater length than TreeIDHexStringLen. In +// This function can take a TreeID of equal or lesser length than TreeIDHexStringLen. In // case the TreeID length is less than TreeIDHexStringLen, it will be padded to the correct // length. func CreateEntryIDFromParts(treeid string, uuid string) (EntryID, error) { @@ -70,13 +62,7 @@ func CreateEntryIDFromParts(treeid string, uuid string) (EntryID, error) { return createEmptyEntryID(), err } - if _, err := hex.DecodeString(treeidFormatted); err != nil { - err := fmt.Errorf("treeid %v is not a valid hex string: %v", treeidFormatted, err) - return createEmptyEntryID(), err - } - - if _, err := hex.DecodeString(uuid); err != nil { - err := fmt.Errorf("uuid %v is not a valid hex string: %v", uuid, err) + if err := ValidateEntryID(treeidFormatted + uuid); err != nil { return createEmptyEntryID(), err } @@ -91,12 +77,6 @@ func createEmptyEntryID() EntryID { UUID: ""} } -func CreateEntryIDWithActiveTreeID(uuid string) (EntryID, error) { - // TODO: Update this to be the global LogRanges struct - treeid := strconv.FormatUint(dummyLogRanges.ActiveIndex(), 10) - return CreateEntryIDFromParts(treeid, uuid) -} - func (e EntryID) ReturnEntryIDString() string { return e.TreeID + e.UUID } @@ -112,15 +92,106 @@ func PadToTreeIDLen(t string) (string, error) { } } -// Returns UUID (with no prepended TreeID) from a UUID or EntryID string +// Returns UUID (with no prepended TreeID) from a UUID or EntryID string. +// Validates UUID and also TreeID if present. func GetUUIDFromIDString(id string) (string, error) { - if len(id) != UUIDHexStringLen && len(id) != EntryIDHexStringLen { + switch len(id) { + case UUIDHexStringLen: + if err := ValidateUUID(id); err != nil { + return "", err + } + return id, nil + + case EntryIDHexStringLen: + if err := ValidateEntryID(id); err != nil { + if err.Error() == "0 is not a valid TreeID" { + return id[len(id)-UUIDHexStringLen:], nil + } + return "", err + } + return id[len(id)-UUIDHexStringLen:], nil + + default: return "", fmt.Errorf("invalid ID len %v for %v", len(id), id) } +} - if _, err := hex.DecodeString(id); err != nil { - return "", fmt.Errorf("id %v is not a valid hex string: %v", id, err) +// This is permissive in that if passed an EntryID, it will find the UUID and validate it. +func ValidateUUID(u string) error { + switch len(u) { + // If u is an EntryID, call validate on just the UUID + case EntryIDHexStringLen: + uid := u[len(u)-UUIDHexStringLen:] + if err := ValidateUUID(uid); err != nil { + return err + } + return nil + case UUIDHexStringLen: + if _, err := hex.DecodeString(u); err != nil { + return fmt.Errorf("id %v is not a valid hex string: %v", u, err) + } + return nil + default: + return fmt.Errorf("invalid ID len %v for %v", len(u), u) } +} - return id[len(id)-UUIDHexStringLen:], nil +// This is permissive in that if passed an EntryID, it will find the TreeID and validate it. +func ValidateTreeID(t string) error { + switch len(t) { + // If t is an EntryID, call validate on just the TreeID + case EntryIDHexStringLen: + tid := t[:TreeIDHexStringLen] + err := ValidateTreeID(tid) + if err != nil { + return err + } + return nil + case TreeIDHexStringLen: + // Check that it's a valid int64 in hex (base 16) + i, err := strconv.ParseInt(t, 16, 64) + if err != nil { + return fmt.Errorf("could not convert treeID %v to int64: %v", t, err) + } + + // Check for invalid TreeID values + // TODO: test for more of these + if i == 0 { + return fmt.Errorf("0 is not a valid TreeID") + } + + return nil + default: + return fmt.Errorf("TreeID len expected to be %v but got %v", TreeIDHexStringLen, len(t)) + } +} + +func ValidateEntryID(id string) error { + UUIDErr := ValidateUUID(id) + if UUIDErr != nil { + return UUIDErr + } + + treeIDErr := ValidateTreeID(id) + if treeIDErr != nil { + return treeIDErr + } + + return nil +} + +// Returns TreeID (with no appended UUID) from a TreeID or EntryID string. +// Validates TreeID and also UUID if present. +func GetTreeIDFromIDString(id string) (string, error) { + switch len(id) { + case UUIDHexStringLen: + return "", fmt.Errorf("cannot get treeID from plain UUID") + case EntryIDHexStringLen, TreeIDHexStringLen: + if err := ValidateEntryID(id); err != nil { + return "", err + } + return id[:TreeIDHexStringLen], nil + default: + return "", fmt.Errorf("invalid ID len %v for %v", len(id), id) + } } diff --git a/pkg/sharding/sharding_test.go b/pkg/sharding/sharding_test.go index 61f115f11..eaf6ca999 100644 --- a/pkg/sharding/sharding_test.go +++ b/pkg/sharding/sharding_test.go @@ -15,15 +15,12 @@ package sharding -import ( - "strconv" - "testing" -) +import "testing" // Create some test data // Good data -const validTreeID1 = "FFFFFFFFFFFFFFFF" -const validTreeID2 = "0000000000000000" +const validTreeID1 = "0FFFFFFFFFFFFFFF" +const validTreeID2 = "3315648d077a9f02" const validTreeID3 = "7241b7903737211c" const shortTreeID = "12345" @@ -37,6 +34,10 @@ var validTreeIDs = []string{validTreeID1, validTreeID2, validTreeID3, shortTreeI var validEntryIDs = []string{validEntryID1, validEntryID2, validEntryID3} // Bad data +// Unknown TreeID +const invalidTreeID = "0000000000000000" +const invalidEntryID = invalidTreeID + validUUID + // Wrong length const tooLongTreeID = validTreeID1 + "e" @@ -70,7 +71,7 @@ var notHexUUIDs = []string{notHexUUID1, notHexUUID2, notHexUUID3} var notHexEntryandUUIDs = []string{notHexEntryID1, notHexEntryID2, notHexEntryID3, notHexUUID1, notHexUUID2, notHexUUID3} // Test functions -func TestCreateEntryID(t *testing.T) { +func TestCreateEntryIDFromParts(t *testing.T) { for _, s := range wrongLengthTreeIDs { if _, err := CreateEntryIDFromParts(s, validUUID); err == nil { t.Errorf("expected length error for wrong TreeID of invalid len: %v", s) @@ -124,27 +125,6 @@ func TestCreateEmptyEntryID(t *testing.T) { } } -func TestCreateEntryIDWithActiveTreeID(t *testing.T) { - entryID, err := CreateEntryIDWithActiveTreeID(validUUID) - if err != nil { - t.Errorf("unable to create entryID: %v", err) - } - - // TODO: Update dummy to be the global LogRanges struct - activeIndexString := strconv.FormatUint(dummyLogRanges.ActiveIndex(), 10) - expectedTreeID, err := PadToTreeIDLen(activeIndexString) - if err != nil { - t.Errorf("unable to pad %v to treeIDLen: %v", activeIndexString, err) - } - if entryID.TreeID != expectedTreeID { - t.Errorf("expected entryID.TreeID %v but got %v", dummyLogRanges.ActiveIndex(), entryID.TreeID) - } - - if entryID.UUID != validUUID { - t.Errorf("expected entryID.TreeID %v but got %v", validUUID, entryID.UUID) - } -} - func TestPadToTreeIDLen(t *testing.T) { short := "12345678" shortPadded := "0000000012345678" @@ -212,3 +192,31 @@ func TestGetUUIDFromIDString(t *testing.T) { } } } + +func TestGetTreeIDFromIDString(t *testing.T) { + valid1, err := GetTreeIDFromIDString(validEntryID1) + if valid1 != validTreeID1 || err != nil { + t.Errorf("expected TreeID %v with nil error, got: %v with error %v", validTreeID1, valid1, err) + } + valid2, err := GetTreeIDFromIDString(validEntryID2) + if valid2 != validTreeID2 || err != nil { + t.Errorf("expected TreeID %v with nil error, got: %v with error %v", validTreeID2, valid2, err) + } + valid3, err := GetTreeIDFromIDString(validEntryID3) + if valid3 != validTreeID3 || err != nil { + t.Errorf("expected TreeID %v with nil error, got: %v with error %v", validTreeID3, valid3, err) + } + + // tree IDs of zero should return an error + invalid, err := GetTreeIDFromIDString(invalidEntryID) + if invalid != "" || err.Error() != "0 is not a valid TreeID" { + t.Errorf("expected err 'unknown treeID', got: %v with error %v", invalid, err) + } + + // invalid UUID should also return an error because we test inclusively for + // malformed parts of EntryID + _, e := GetTreeIDFromIDString(notHexEntryID2) + if e == nil { + t.Errorf("expected error for invalid UUID, but got none") + } +} diff --git a/tests/e2e_test.go b/tests/e2e_test.go index 5412c5a9a..c96aa9df9 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -258,7 +258,13 @@ func TestGet(t *testing.T) { outputContains(t, out, uuid) // Exercise GET with the new EntryID (TreeID + UUID) - entryID, err := sharding.CreateEntryIDFromParts("0", uuid) + out = runCli(t, "loginfo") + tidStr := strings.TrimSpace(strings.Split(out, "TreeID: ")[1]) + tid, err := strconv.ParseInt(tidStr, 10, 64) + if err != nil { + t.Errorf(err.Error()) + } + entryID, err := sharding.CreateEntryIDFromParts(fmt.Sprintf("%x", tid), uuid) if err != nil { t.Error(err) }