(PXP-6815): update MDS object after file upload. #29
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,47 +2,68 @@ package handlers | |
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "log" | ||
| "net/http" | ||
| "net/url" | ||
| "os" | ||
| "strings" | ||
| ) | ||
|
|
||
| // MaxRetries maximum number of retries | ||
| const MaxRetries = 5 | ||
|
|
||
| type ConfigInfo struct { | ||
| Indexd IndexdInfo `indexd` | ||
| MetadataService MetadataServiceInfo `metadata_service` | ||
| } | ||
|
|
||
| type IndexdInfo struct { | ||
| URL string `url` | ||
| Username string `username` | ||
| Password string `password` | ||
| } | ||
|
|
||
| type MetadataServiceInfo struct { | ||
| URL string `url` | ||
| Username string `username` | ||
| Password string `password` | ||
| } | ||
|
|
||
| // Read Indexd and Metadata Service config info from CONFIG_FILE into | ||
| // ConfigInfo struct. Panic if Indexd creds could not be found | ||
| func getConfigInfo() *ConfigInfo { | ||
| configInfo := new(ConfigInfo) | ||
| configBytes := []byte(os.Getenv("CONFIG_FILE")) | ||
| log.Printf("Attempting to unmarshal both Indexd and Metadata Service configs from JSON in CONFIG_FILE env variable") | ||
| if err := json.Unmarshal(configBytes, configInfo); err != nil { | ||
| log.Panicf("Could not unmarshal JSON in CONFIG_FILE env variable: %s", err) | ||
| } | ||
| if configInfo.Indexd == (IndexdInfo{}) { | ||
| log.Printf("Could not find required Indexd config when unmarshalling both Indexd and Metadata Service configs. Trying again to only unmarshal Indexd config") | ||
| if err := json.Unmarshal(configBytes, &configInfo.Indexd); err != nil { | ||
| log.Panicf("Could not unmarshal JSON in CONFIG_FILE env variable: %s", err) | ||
| } | ||
| if configInfo.Indexd == (IndexdInfo{}) { | ||
| log.Panicf("Could not find required Indexd config in JSON in CONFIG_FILE env variable") | ||
| } | ||
| } | ||
| log.Printf("Indexd config was unmarshalled") | ||
| if configInfo.MetadataService != (MetadataServiceInfo{}) { | ||
| log.Printf("Metadata Service config was unmarshalled") | ||
| } else { | ||
| log.Printf("Metadata Service config was not unmarshalled") | ||
| } | ||
|
|
||
| return configInfo | ||
| } | ||
|
|
||
| // IndexS3Object indexes s3 object The fuction does several things. It | ||
| // downloads the object from S3, computes size and hashes, and updates Indexd | ||
| // and potentially Metadata Service | ||
| func IndexS3Object(s3objectURL string) { | ||
| configInfo := getConfigInfo() | ||
|
|
||
| s3objectURL, _ = url.QueryUnescape(s3objectURL) | ||
| u, err := url.Parse(s3objectURL) | ||
| if err != nil { | ||
|
|
@@ -63,6 +84,21 @@ func IndexS3Object(s3objectURL string) { | |
| } else { | ||
| uuid = strings.Join(split_key[:len(split_key)-1], "/") | ||
| } | ||
| filename := split_key[len(split_key)-1] | ||
|
|
||
| log.Printf("Attempting to get rev for record %s in Indexd", uuid) | ||
| rev, err := GetIndexdRecordRev(uuid, configInfo.Indexd.URL) | ||
| mdsUploadedBody := fmt.Sprintf(`{"_upload_status": "uploaded", "_filename": "%s"}`, filename) | ||
| if err != nil { | ||
| log.Panicf("Can not get record %s from Indexd. Error message %s", uuid, err) | ||
| } else if rev == "" { | ||
| log.Printf("Indexd record with guid %s already has size and hashes", uuid) | ||
| updateMetadataObjectWrapper(uuid, configInfo, mdsUploadedBody) | ||
| return | ||
| } | ||
| log.Printf("Got rev %s from Indexd for record %s", rev, uuid) | ||
|
|
||
| updateMetadataObjectWrapper(uuid, configInfo, `{"_upload_status": "indexs3client job calculating hashes and size"}`) | ||
|
There was a problem hiding this comment. all the others statuses in the design doc are a single word (or underscore-separated words) you might want to check with @Avantol13 if this is an acceptable status. if not, maybe There was a problem hiding this comment. Yeah just one thing to keep in mind is what There was a problem hiding this comment. yeah, that's not detailed in the design doc, i'll let you and Alex make that call |
||
|
|
||
| client, err := CreateNewAwsClient() | ||
| if err != nil { | ||
|
|
@@ -77,54 +113,16 @@ func IndexS3Object(s3objectURL string) { | |
| } | ||
| log.Printf("Finish to compute hashes for %s", key) | ||
|
|
||
| indexdHashesBody := fmt.Sprintf(`{"size": %d, "urls": ["%s"], "hashes": {"md5": "%s", "sha1":"%s", "sha256": "%s", "sha512": "%s", "crc": "%s"}}`, | ||
| objectSize, s3objectURL, hashes.Md5, hashes.Sha1, hashes.Sha256, hashes.Sha512, hashes.Crc32c) | ||
| log.Printf("Attempting to update Indexd record %s. Request Body: %s", uuid, indexdHashesBody) | ||
| resp, err := UpdateIndexdRecord(uuid, rev, &configInfo.Indexd, []byte(indexdHashesBody)) | ||
| if err != nil { | ||
| log.Panicf("Could not update Indexd record %s. Error: %s", uuid, err) | ||
| } else if resp.StatusCode != http.StatusOK { | ||
| log.Panicf("Could not update Indexd record %s. Response Status Code: %d", uuid, resp.StatusCode) | ||
| } | ||
| log.Printf("Updated Indexd record %s with hash info. Response Status Code: %d", uuid, resp.StatusCode) | ||
|
|
||
| updateMetadataObjectWrapper(uuid, configInfo, mdsUploadedBody) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the ticket and design doc mention
_file_typebut not_filenameThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I talked with @Avantol13 about this. We thought
_filenamecould be preferable. I just updated the design doc to reflect this.