From 3541ef98ac1d232afe1db22bdbf29c92b72c9352 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 5 Jul 2022 15:47:12 +0100 Subject: [PATCH] Check inactive shards for UUID for /retrieve endpoint Signed-off-by: Priya Wadhwa --- pkg/api/entries.go | 94 +++++++++++++++++--------------------- tests/sharding-e2e-test.sh | 7 ++- 2 files changed, 49 insertions(+), 52 deletions(-) diff --git a/pkg/api/entries.go b/pkg/api/entries.go index 6189e6a2f..159f51f57 100644 --- a/pkg/api/entries.go +++ b/pkg/api/entries.go @@ -19,11 +19,9 @@ import ( "bytes" "context" "encoding/hex" - "errors" "fmt" "net/http" "net/url" - "strconv" "github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer" "github.com/go-openapi/runtime" @@ -32,6 +30,7 @@ import ( "github.com/go-openapi/swag" "github.com/google/trillian" ttypes "github.com/google/trillian/types" + "github.com/pkg/errors" "github.com/spf13/viper" "github.com/transparency-dev/merkle/rfc6962" "golang.org/x/sync/errgroup" @@ -312,43 +311,11 @@ func getEntryURL(locationURL url.URL, uuid string) strfmt.URI { // GetLogEntryByUUIDHandler gets log entry and inclusion proof for specified UUID aka merkle leaf hash func GetLogEntryByUUIDHandler(params entries.GetLogEntryByUUIDParams) middleware.Responder { - uuid, err := sharding.GetUUIDFromIDString(params.EntryUUID) + logEntry, err := retrieveLogEntry(params.HTTPRequest.Context(), params.EntryUUID) if err != nil { - return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not get UUID from ID string %v", params.EntryUUID)) + return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not get log entry with ID %v", params.EntryUUID)) } - tidString, err := sharding.GetTreeIDFromIDString(params.EntryUUID) - - // If treeID is found in EntryID, route to correct tree - if err == nil { - tid, err := strconv.ParseInt(tidString, 16, 64) - if err != nil { - return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not convert treeID %v to int", tidString)) - } - logEntry, err := RetrieveUUID(params, uuid, tid) - if err != nil { - if errors.Is(err, ErrNotFound) { - return handleRekorAPIError(params, http.StatusNotFound, err, "") - } - return handleRekorAPIError(params, http.StatusInternalServerError, err, "") - } - return entries.NewGetLogEntryByUUIDOK().WithPayload(logEntry) - } - - // If EntryID is plain UUID (ex. from client v0.5), check all trees - if errors.Is(err, sharding.ErrPlainUUID) { - trees := []sharding.LogRange{{TreeID: api.logRanges.ActiveTreeID()}} - trees = append(trees, api.logRanges.GetInactive()...) - - for _, t := range trees { - logEntry, err := RetrieveUUID(params, uuid, t.TreeID) - if err != nil { - continue - } - return entries.NewGetLogEntryByUUIDOK().WithPayload(logEntry) - } - return handleRekorAPIError(params, http.StatusNotFound, err, "UUID not found in any known trees") - } - return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not get treeID from ID string %v", params.EntryUUID)) + return entries.NewGetLogEntryByUUIDOK().WithPayload(logEntry) } // SearchLogQueryHandler searches log by index, UUID, or proposed entry and returns array of entries found with inclusion proofs @@ -366,17 +333,12 @@ func SearchLogQueryHandler(params entries.SearchLogQueryParams) middleware.Respo if err != nil { return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not get UUID from ID string %v", entryID)) } - if tid, err := sharding.TreeID(entryID); err == nil { - entry, err := RetrieveUUID(entries.GetLogEntryByUUIDParams{ - EntryUUID: entryID, - HTTPRequest: params.HTTPRequest, - }, uuid, tid) - if err != nil { - return handleRekorAPIError(params, http.StatusBadRequest, err, fmt.Sprintf("could not get uuid from %v", entryID)) - } - resultPayload = append(resultPayload, entry) + logEntry, err := retrieveLogEntry(httpReqCtx, entryID) + if err == nil { + resultPayload = append(resultPayload, logEntry) continue } + // If we couldn't get the entry, search for the hash later hash, err := hex.DecodeString(uuid) if err != nil { return handleRekorAPIError(params, http.StatusBadRequest, err, malformedUUID) @@ -487,16 +449,46 @@ func SearchLogQueryHandler(params entries.SearchLogQueryParams) middleware.Respo var ErrNotFound = errors.New("grpc returned 0 leaves with success code") -// Attempt to retrieve a UUID from a backend tree -func RetrieveUUID(params entries.GetLogEntryByUUIDParams, uuid string, tid int64) (models.LogEntry, error) { - ctx := params.HTTPRequest.Context() +// Retrieve a Log Entry +// If a tree ID is specified, look in that tree +// Otherwise, look through all inactive and active shards +func retrieveLogEntry(ctx context.Context, entryUUID string) (models.LogEntry, error) { + uuid, err := sharding.GetUUIDFromIDString(entryUUID) + if err != nil { + return models.LogEntry{}, errors.Wrap(err, "getting uuid from id string") + } + + // Get the tree ID and check that shard for the entry + tid, err := sharding.TreeID(entryUUID) + if err == nil { + return retrieveUUIDFromTree(ctx, uuid, tid) + } + + // If we got a UUID instead of an EntryID, search all shards + if errors.Is(err, sharding.ErrPlainUUID) { + trees := []sharding.LogRange{{TreeID: api.logRanges.ActiveTreeID()}} + trees = append(trees, api.logRanges.GetInactive()...) + + for _, t := range trees { + logEntry, err := retrieveUUIDFromTree(ctx, uuid, t.TreeID) + if err != nil { + continue + } + return logEntry, nil + } + } + + return models.LogEntry{}, err +} + +func retrieveUUIDFromTree(ctx context.Context, uuid string, tid int64) (models.LogEntry, error) { hashValue, err := hex.DecodeString(uuid) if err != nil { return models.LogEntry{}, err } - tc := NewTrillianClientFromTreeID(params.HTTPRequest.Context(), tid) - log.RequestIDLogger(params.HTTPRequest).Debugf("Attempting to retrieve UUID %v from TreeID %v", uuid, tid) + tc := NewTrillianClientFromTreeID(ctx, tid) + log.Logger.Debugf("Attempting to retrieve UUID %v from TreeID %v", uuid, tid) resp := tc.getLeafAndProofByHash(hashValue) switch resp.status { diff --git a/tests/sharding-e2e-test.sh b/tests/sharding-e2e-test.sh index de3c7b368..cf24c98c6 100755 --- a/tests/sharding-e2e-test.sh +++ b/tests/sharding-e2e-test.sh @@ -243,9 +243,14 @@ fi echo echo "Testing /api/v1/log/entries/retrieve endpoint..." -UUID1=$($REKOR_CLI get --log-index 0 --rekor_server http://localhost:3000 --format json | jq -r .UUID) +UUID1=$($REKOR_CLI get --log-index 1 --rekor_server http://localhost:3000 --format json | jq -r .UUID) UUID2=$($REKOR_CLI get --log-index 3 --rekor_server http://localhost:3000 --format json | jq -r .UUID) + +# Make sure retrieve by UUID in the inactive shard works +NUM_ELEMENTS=$(curl -f http://localhost:3000/api/v1/log/entries/retrieve -H "Content-Type: application/json" -H "Accept: application/json" -d "{ \"entryUUIDs\": [\"$UUID1\"]}" | jq '. | length') +stringsMatch $NUM_ELEMENTS "1" + HEX_INITIAL_TREE_ID=$(printf "%x" $INITIAL_TREE_ID | awk '{ for(c = 0; c < 16 ; c++) s = s"0"; s = s$1; print substr(s, 1 + length(s) - 16);}') HEX_INITIAL_SHARD_ID=$(printf "%x" $SHARD_TREE_ID | awk '{ for(c = 0; c < 16 ; c++) s = s"0"; s = s$1; print substr(s, 1 + length(s) - 16);}')