From 6bf65dc0d788bb223b07be4cdd7dd7c269385d7c Mon Sep 17 00:00:00 2001 From: John McCann Date: Sun, 18 Oct 2020 12:56:01 -0700 Subject: [PATCH 1/4] chore(indexing job): require mds creds for startup --- main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/main.go b/main.go index 1d342f6..f5ef3d2 100755 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ package main import ( "encoding/json" + "errors" "flag" "fmt" "log" @@ -37,6 +38,27 @@ func init() { flag.Parse() } +func checkIndexingJobsImageConfig(jobConfigs []handlers.JobConfig) error { + for _, jobConfig := range jobConfigs { + if jobConfig.Name == "indexing" { + imageConfig := jobConfig.ImageConfig.(map[string]interface{}) + if imageConfig["url"].(string) == "" || imageConfig["username"].(string) == "" || imageConfig["password"].(string) == "" { + return errors.New("indexing job imageConfig section missing indexd url and/or creds") + } + mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds" + if mdsConfig, ok := imageConfig["metadataService"]; ok { + mdsConfig := mdsConfig.(map[string]interface{}) + if mdsConfig["url"].(string) == "" || mdsConfig["username"].(string) == "" || mdsConfig["password"].(string) == "" { + return errors.New(mdsErrorMessage) + } + } else { + return errors.New(mdsErrorMessage) + } + } + } + return nil +} + func main() { jsonBytes, err := handlers.ReadFile(handlers.LookupCredFile()) if err != nil { @@ -61,6 +83,11 @@ func main() { jobConfigs := make([]handlers.JobConfig, 0) json.Unmarshal(b, &jobConfigs) + if err := checkIndexingJobsImageConfig(jobConfigs); err != nil { + glog.Error(err) + return + } + // start an SQSHandler instance SQSHandler := handlers.NewSQSHandler(sqsURL) From 65e2c53fa036ac503e76b553961fab1d7f372866 Mon Sep 17 00:00:00 2001 From: John McCann Date: Sun, 18 Oct 2020 13:30:27 -0700 Subject: [PATCH 2/4] chore(startup): os.Exit(1) on errors with creds --- main.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index f5ef3d2..48e6b40 100755 --- a/main.go +++ b/main.go @@ -38,14 +38,16 @@ func init() { flag.Parse() } +// Check that all "indexing" jobs have both Indexd and Metadata Service creds +// configured. If not, return an error. func checkIndexingJobsImageConfig(jobConfigs []handlers.JobConfig) error { for _, jobConfig := range jobConfigs { if jobConfig.Name == "indexing" { imageConfig := jobConfig.ImageConfig.(map[string]interface{}) if imageConfig["url"].(string) == "" || imageConfig["username"].(string) == "" || imageConfig["password"].(string) == "" { - return errors.New("indexing job imageConfig section missing indexd url and/or creds") + return errors.New("indexing job imageConfig section missing indexd url and/or creds!") } - mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds" + mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds!" if mdsConfig, ok := imageConfig["metadataService"]; ok { mdsConfig := mdsConfig.(map[string]interface{}) if mdsConfig["url"].(string) == "" || mdsConfig["username"].(string) == "" || mdsConfig["password"].(string) == "" { @@ -63,13 +65,13 @@ func main() { jsonBytes, err := handlers.ReadFile(handlers.LookupCredFile()) if err != nil { glog.Errorln("Can not read credential file!") - return + os.Exit(1) } var sqsURL string if sqs, err := handlers.GetValueFromJSON(jsonBytes, []string{"SQS", "url"}); err != nil { glog.Errorln("Can not read SQS url from credential file!") - return + os.Exit(1) } else { sqsURL = sqs.(string) } @@ -85,7 +87,7 @@ func main() { if err := checkIndexingJobsImageConfig(jobConfigs); err != nil { glog.Error(err) - return + os.Exit(1) } // start an SQSHandler instance From c0c2a97e0033edbbbd91c7e25fff662df0722b73 Mon Sep 17 00:00:00 2001 From: John McCann Date: Sun, 18 Oct 2020 18:20:50 -0700 Subject: [PATCH 3/4] test(CheckIndexingJobsImageConfig): add cases --- .secrets.baseline | 546 +---------------------------------------- handlers/utils.go | 23 ++ handlers/utils_test.go | 240 ++++++++++++++++++ main.go | 26 +- 4 files changed, 274 insertions(+), 561 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 27617f3..f4a60ff 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": null, + "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2020-07-20T21:13:15Z", + "generated_at": "2020-10-19T01:14:54Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -58,563 +58,37 @@ } ], "results": { - "Gopkg.lock": [ - { - "hashed_secret": "5bcb4792a577d84bcc5c03f120c98d012977eff5", - "is_verified": false, - "line_number": 47, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "97708b4fb4ba244db393f689ece7b0778d34f972", - "is_verified": false, - "line_number": 55, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "cab278f4e017ea11207f869826126e5bf70940c1", - "is_verified": false, - "line_number": 63, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "862ad72a7b52d559c74dbb3e8299ae7b604ca82d", - "is_verified": false, - "line_number": 74, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "459775e92adef405f9000b5d4c6f5a7c27708dbb", - "is_verified": false, - "line_number": 83, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "fe7a325542ce6b24d2cb3ea32e86bb544b45fd99", - "is_verified": false, - "line_number": 96, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "da776fabc0e998bbaf6ca8b2b4c7b839c61091cb", - "is_verified": false, - "line_number": 104, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "5589045af31aab32ac8e3eba58f70039312ff58c", - "is_verified": false, - "line_number": 116, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "16d8992ece9d81057cbc5fd4826e06c4c0735736", - "is_verified": false, - "line_number": 131, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "a9374baa0793fc4d432f379f01f92d6dd6d0c800", - "is_verified": false, - "line_number": 139, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "86fb8619c03cfeaed659d3eea06eacacb14dfcf5", - "is_verified": false, - "line_number": 147, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "a52646dc3d9e7c9dc362f9918688f751ccb4ba38", - "is_verified": false, - "line_number": 155, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "f472444fb1c41568eb4ef2f7a8439ef79c319f1f", - "is_verified": false, - "line_number": 163, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "311f3476943bf082a712d8dfc154bcd649bf88e3", - "is_verified": false, - "line_number": 172, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "844fa71533cca869ef1613c47d3336e1491e3545", - "is_verified": false, - "line_number": 187, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "81f8f0c01ee9ff8f5746e649be750f62dc5721ea", - "is_verified": false, - "line_number": 198, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "e1fdfe29fe7831b5e66a97584fb0e5b954f7f17a", - "is_verified": false, - "line_number": 210, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "1ab1f0600c7683af76c089feb9c0b3053b81d80c", - "is_verified": false, - "line_number": 234, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "70d4c589bec9b128879d5818e7d792a16974bf4c", - "is_verified": false, - "line_number": 243, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "d28b5c47e85b30236fd855a38af40bfb6897bc96", - "is_verified": false, - "line_number": 258, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "832a596b1a8952f501898738bf41d5353854804a", - "is_verified": false, - "line_number": 297, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "1c451055a43728b0fc1289be1ab535e8afa1b569", - "is_verified": false, - "line_number": 305, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "2e53577f67d4769d889b62e597f2b22e16cd1aca", - "is_verified": false, - "line_number": 313, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "04ce762bc053f7e5788141ed1df4357e73abf0c6", - "is_verified": false, - "line_number": 322, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "824f2a7c7dbf64415abe27e25eb572c34939bda8", - "is_verified": false, - "line_number": 366, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "5ca36e6e9bfd4bb7fbe06fab24522787a9af56a8", - "is_verified": false, - "line_number": 409, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "7a5027e323128e4378a087d25b41198bbb3fbafb", - "is_verified": false, - "line_number": 472, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "7628e8a78c177bd31ce5579f4be13dd5da86a678", - "is_verified": false, - "line_number": 480, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "bad46afaeb4dc23fa33743c8c6f1c760f68950b9", - "is_verified": false, - "line_number": 489, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "8223fb2d3ec973c465a1a07bd70cdea378e039c1", - "is_verified": false, - "line_number": 496, - "type": "Hex High Entropy String" - } - ], "handlers/handler.go": [ { "hashed_secret": "faccff18e52db0547225f3e23b4aa9a0bfec3a50", + "is_secret": false, "is_verified": false, - "line_number": 187, + "line_number": 188, "type": "Hex High Entropy String" }, { "hashed_secret": "bad1b4a10ed82d1dc1628ff569d1fbebc0b686a5", + "is_secret": false, "is_verified": false, - "line_number": 188, + "line_number": 189, "type": "Hex High Entropy String" } ], "handlers/utils_test.go": [ { "hashed_secret": "a41a00b5ae33ea952f92a9fabc58d0a018813df7", + "is_secret": false, "is_verified": false, "line_number": 16, "type": "Secret Keyword" - } - ], - "vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go": [ - { - "hashed_secret": "84b5be4b49d590b4f58de237bd143a9ab493981e", - "is_verified": false, - "line_number": 129, - "type": "Secret Keyword" - } - ], - "vendor/github.com/aws/aws-sdk-go/service/sts/api.go": [ - { - "hashed_secret": "25910f981e85ca04baf359199dd0bd4a3ae738b6", - "is_verified": false, - "line_number": 880, - "type": "AWS Access Key" - } - ], - "vendor/github.com/jmespath/go-jmespath/astnodetype_string.go": [ - { - "hashed_secret": "7d488916eb97aa479b5a45b3c7ddcc5a9cee4b3c", - "is_verified": false, - "line_number": 7, - "type": "Base64 High Entropy String" - } - ], - "vendor/github.com/json-iterator/go/Gopkg.lock": [ - { - "hashed_secret": "64710601da7f8963323561eb3bc45559b8f878a5", - "is_verified": false, - "line_number": 7, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "86fb8619c03cfeaed659d3eea06eacacb14dfcf5", - "is_verified": false, - "line_number": 13, - "type": "Hex High Entropy String" }, { - "hashed_secret": "ddb81deb09d5bad1fbe25a801ee10336a8061509", + "hashed_secret": "1ec252abcbf0dca36f5083d209e3b4e065ad5778", + "is_secret": false, "is_verified": false, - "line_number": 19, - "type": "Hex High Entropy String" - } - ], - "vendor/github.com/json-iterator/go/go.sum": [ - { - "hashed_secret": "e7d2b52449737744aa82f66895db6409dd7efc89", - "is_verified": false, - "line_number": 2, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "4dbbe18874344f8a812138c0c9b17841a5fab11d", - "is_verified": false, - "line_number": 3, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "408fac9dd47100fe196085eeee01e0d0b95a2836", - "is_verified": false, - "line_number": 4, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "b66d571566471f99fc35c72f3a7d69e04bdd0a55", - "is_verified": false, - "line_number": 5, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "476e8d4d81ad73ee40bee4f96ea5f8f72a73b2c0", - "is_verified": false, - "line_number": 6, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "b38699d8705643a8973709b7e5ceeb6bb7a14761", - "is_verified": false, - "line_number": 7, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "c262613ca55ef20e7a25cdfb32397652f083982d", - "is_verified": false, - "line_number": 8, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "6bba0354504f72e495c0cbeda4fb278b6dcefe10", - "is_verified": false, - "line_number": 9, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "6a4a1948d9e9978e41ee2d466928b91e8ac5d54b", - "is_verified": false, - "line_number": 10, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "c382813ad978b45fb54312f408087e014b4374eb", - "is_verified": false, - "line_number": 11, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "f16e20544c05634663c76b999b6707da9f0ecbcc", - "is_verified": false, - "line_number": 12, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "6befd736c9d2fdf056c960bb1469537e0588cda6", - "is_verified": false, - "line_number": 13, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "3765eda8e7c792ffa4b4cfca6385122b32fcb9e0", - "is_verified": false, - "line_number": 14, - "type": "Base64 High Entropy String" - } - ], - "vendor/github.com/modern-go/reflect2/Gopkg.lock": [ - { - "hashed_secret": "64710601da7f8963323561eb3bc45559b8f878a5", - "is_verified": false, - "line_number": 7, - "type": "Hex High Entropy String" - }, - { - "hashed_secret": "4cd8b5cbde688c38dc3876e2c2fef2fcd35f7c91", - "is_verified": false, - "line_number": 13, - "type": "Hex High Entropy String" - } - ], - "vendor/golang.org/x/oauth2/go.sum": [ - { - "hashed_secret": "5dd4005344ba24bac2dc5f741dca03ba07330bcd", - "is_verified": false, - "line_number": 1, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "805060f42af90533927009784f99152611f70431", - "is_verified": false, - "line_number": 2, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "a06f5b4f6487e191c59158ea61f382bd0a2686bf", - "is_verified": false, - "line_number": 3, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "4fb18fc74dca10136741c8fae75f110b36547ecb", - "is_verified": false, - "line_number": 4, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "646da3588de8f223d38f0a86f07980bb0259575d", - "is_verified": false, - "line_number": 6, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "bf5b914454bf2d472be60acb49a2e646cb1139a7", - "is_verified": false, - "line_number": 7, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "ae140ecff72ca042ad3042b33c376e86dcb60bed", - "is_verified": false, - "line_number": 8, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "af58e693e1b2d980ed923a3a65bacd88033d1c9d", - "is_verified": false, - "line_number": 9, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "6c76abe4bb26384b9ff06c9ef66352708b9368f1", - "is_verified": false, - "line_number": 10, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "ee4f7ae71e147f86b403ccf339f56d5183539691", - "is_verified": false, - "line_number": 11, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "d01387bed984468530fa7f981c05fc231fdefd00", - "is_verified": false, - "line_number": 12, - "type": "Base64 High Entropy String" - } - ], - "vendor/golang.org/x/oauth2/oauth2.go": [ - { - "hashed_secret": "b836e3dcfb134e8d43f3c5f29133c542afbd052c", - "is_verified": false, - "line_number": 193, + "line_number": 244, "type": "Secret Keyword" } - ], - "vendor/k8s.io/api/core/v1/generated.pb.go": [ - { - "hashed_secret": "7e15bb5c01e7dd56499e37c634cf791d3a519aee", - "is_verified": false, - "line_number": 26852, - "type": "Secret Keyword" - }, - { - "hashed_secret": "b6077f3155cee2f4f132751fa3c8d1da529c5e9f", - "is_verified": false, - "line_number": 59357, - "type": "Secret Keyword" - }, - { - "hashed_secret": "848c4bd26ce5574c1a4f39f7eb3490fc32e31196", - "is_verified": false, - "line_number": 59360, - "type": "Secret Keyword" - }, - { - "hashed_secret": "37df8ed8a1528e018b6ca0aa33a986f7ce453c5f", - "is_verified": false, - "line_number": 64885, - "type": "Secret Keyword" - }, - { - "hashed_secret": "ec1d28180d8883ff1af3df4eb37fcaea1c3291d9", - "is_verified": false, - "line_number": 65262, - "type": "Secret Keyword" - } - ], - "vendor/k8s.io/api/core/v1/generated.proto": [ - { - "hashed_secret": "1dcd8fc1a0be55707e0a434392312f2a5e1c3700", - "is_verified": false, - "line_number": 5129, - "type": "Secret Keyword" - }, - { - "hashed_secret": "f1456c0d675980582dda9b8edc7f13f503ce544f", - "is_verified": false, - "line_number": 5185, - "type": "Secret Keyword" - } - ], - "vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go": [ - { - "hashed_secret": "4be694160a6f69291b09cf6696c363dc663b30d9", - "is_verified": false, - "line_number": 2394, - "type": "Secret Keyword" - }, - { - "hashed_secret": "5bc5de340677f8b2970324d169d0325a566e37f1", - "is_verified": false, - "line_number": 2411, - "type": "Secret Keyword" - } - ], - "vendor/k8s.io/client-go/rest/config.go": [ - { - "hashed_secret": "a28a0fc0bc12361a290786828fb6d5951f76b0b3", - "is_verified": false, - "line_number": 161, - "type": "Secret Keyword" - }, - { - "hashed_secret": "f292701a838da72f0f50265db042e85a7c762f9b", - "is_verified": false, - "line_number": 519, - "type": "Secret Keyword" - } - ], - "vendor/k8s.io/client-go/rest/transport.go": [ - { - "hashed_secret": "8270abd5c176891ce9addc680f7a1b70e27c3809", - "is_verified": false, - "line_number": 78, - "type": "Secret Keyword" - } - ], - "vendor/k8s.io/klog/go.sum": [ - { - "hashed_secret": "08eaf9de4a27f0bc8994720e40867be6f2520389", - "is_verified": false, - "line_number": 1, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "00c8f1721208f284bccf58a724d1c8f5f2084399", - "is_verified": false, - "line_number": 2, - "type": "Base64 High Entropy String" - } - ], - "vendor/sigs.k8s.io/yaml/go.sum": [ - { - "hashed_secret": "e7d2b52449737744aa82f66895db6409dd7efc89", - "is_verified": false, - "line_number": 1, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "4dbbe18874344f8a812138c0c9b17841a5fab11d", - "is_verified": false, - "line_number": 2, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "aa945027601e7b92be504f177761e51cd2fc9293", - "is_verified": false, - "line_number": 3, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "9845b8addf932de31b286d4957cf79b9d302dd4f", - "is_verified": false, - "line_number": 4, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "1c49a79cdf8bf5e63c00a59b65b07566ada98aba", - "is_verified": false, - "line_number": 6, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "544f95865e9625c5f51acff5a827e1fdfcbabf1a", - "is_verified": false, - "line_number": 8, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "0bfcf978179e9b9371d132d9fd4065909f1fa3e6", - "is_verified": false, - "line_number": 9, - "type": "Base64 High Entropy String" - } ] }, "version": "0.13.1", diff --git a/handlers/utils.go b/handlers/utils.go index cd3ad1f..d3df5b2 100644 --- a/handlers/utils.go +++ b/handlers/utils.go @@ -56,6 +56,29 @@ func GetValueFromJSON(jsonBytes []byte, keys []string) (interface{}, error) { return dataMap, nil } +// Check that all "indexing" jobs have both Indexd and Metadata Service creds +// configured. If not, return an error. +func CheckIndexingJobsImageConfig(jobConfigs []JobConfig) error { + for _, jobConfig := range jobConfigs { + if jobConfig.Name == "indexing" { + imageConfig := jobConfig.ImageConfig.(map[string]interface{}) + if imageConfig["url"].(string) == "" || imageConfig["username"].(string) == "" || imageConfig["password"].(string) == "" { + return errors.New("indexing job imageConfig section missing indexd url and/or creds!") + } + mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds!" + if mdsConfig, ok := imageConfig["metadataService"]; ok { + mdsConfig := mdsConfig.(map[string]interface{}) + if mdsConfig["url"].(string) == "" || mdsConfig["username"].(string) == "" || mdsConfig["password"].(string) == "" { + return errors.New(mdsErrorMessage) + } + } else { + return errors.New(mdsErrorMessage) + } + } + } + return nil +} + func StringContainsPrefixInSlice(s string, prefixList []string) bool { for _, prefix := range prefixList { if strings.HasPrefix(s, prefix) { diff --git a/handlers/utils_test.go b/handlers/utils_test.go index af67288..5c49cd7 100644 --- a/handlers/utils_test.go +++ b/handlers/utils_test.go @@ -38,6 +38,7 @@ func makeTestJson() string { ] }` } + func TestUtils(t *testing.T) { jsonStr := makeTestJson() @@ -68,3 +69,242 @@ func TestUtils(t *testing.T) { assert.Equal(t, len(jobConfigs), 2) } + +// Test that CheckIndexingJobsImageConfig does not return an error when both +// Indexd and Metadata Service creds have been configured. +func TestCheckIndexingJobsImageConfigWithIndexdAndMDSCreds(t *testing.T) { + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "password": "test", + "metadataService": { + "url": "http://revproxy-service/mds", + "username": "dog", + "password": "paws" + } + } + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + err := CheckIndexingJobsImageConfig(jobConfigs) + assert.Equal(t, err, nil) +} + +// Test that CheckIndexingJobsImageConfig returns an error when MDS creds +// have not been configured. +func TestCheckIndexingJobsImageConfigWithoutMDSCreds(t *testing.T) { + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "password": "test" + } + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + err := CheckIndexingJobsImageConfig(jobConfigs) + assert.NotEqual(t, err, nil) +} + +// Test that CheckIndexingJobsImageConfig returns an error when MDS creds have +// not been configured for the second indexing job. +func TestCheckIndexingJobsImageConfigWithSecondIndexingJobMissingMDSCreds(t *testing.T) { + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "password": "test", + "metadataService": { + "url": "http://revproxy-service/mds", + "username": "dog", + "password": "paws" + } + } + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + }, + { + "name": "indexing", + "pattern": "s3://second-bucket/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "password": "test" + } + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + err := CheckIndexingJobsImageConfig(jobConfigs) + assert.NotEqual(t, err, nil) +} + +// Test that CheckIndexingJobsImageConfig panics when the MDS password has not +// been configured. +func TestCheckIndexingJobsImageConfigWithoutMDSPassword(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("Expecting CheckIndexingJobsImageConfig to panic since metadataService password is missing") + } + }() + + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "password": "test", + "metadataService": { + "url": "http://revproxy-service/mds", + "username": "dog" + } + } + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + CheckIndexingJobsImageConfig(jobConfigs) +} + +// Test that CheckIndexingJobsImageConfig panics when the Indexd password has +// not been configured. +func TestCheckIndexingJobsImageConfigWithoutIndexdPassword(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("Expecting CheckIndexingJobsImageConfig to panic since Indexd password is missing") + } + }() + + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": { + "url": "http://indexd-service/", + "username": "test", + "metadataService": { + "url": "http://revproxy-service/mds", + "username": "dog", + "password": "paws" + } + } + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + CheckIndexingJobsImageConfig(jobConfigs) +} + +// Test that CheckIndexingJobsImageConfig panics when the indexing job's +// imageConfig is blank +func TestCheckIndexingJobsImageConfigWithoutImageConfig(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("Expecting CheckIndexingJobsImageConfig to panic since indexing job imageConfig is missing") + } + }() + + jobsJson := + ` + [ + { + "name": "indexing", + "pattern": "s3://xssxs/*", + "image": "quay.io/cdis/indexs3client:master", + "imageConfig": {} + }, + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + CheckIndexingJobsImageConfig(jobConfigs) +} + +// Test that CheckIndexingJobsImageConfig does not return an error when there +// is no indexing job +func TestCheckIndexingJobsImageConfigWithoutIndexingJob(t *testing.T) { + jobsJson := + ` + [ + { + "name": "usersync", + "pattern": "s3://xssxs/user.yaml", + "image": "quay.io/cdis/fence:master", + "imageConfig": {} + } + ] + ` + jobConfigs := make([]JobConfig, 0) + json.Unmarshal([]byte(jobsJson), &jobConfigs) + err := CheckIndexingJobsImageConfig(jobConfigs) + assert.Equal(t, err, nil) +} diff --git a/main.go b/main.go index 48e6b40..cf77b65 100755 --- a/main.go +++ b/main.go @@ -11,7 +11,6 @@ package main import ( "encoding/json" - "errors" "flag" "fmt" "log" @@ -38,29 +37,6 @@ func init() { flag.Parse() } -// Check that all "indexing" jobs have both Indexd and Metadata Service creds -// configured. If not, return an error. -func checkIndexingJobsImageConfig(jobConfigs []handlers.JobConfig) error { - for _, jobConfig := range jobConfigs { - if jobConfig.Name == "indexing" { - imageConfig := jobConfig.ImageConfig.(map[string]interface{}) - if imageConfig["url"].(string) == "" || imageConfig["username"].(string) == "" || imageConfig["password"].(string) == "" { - return errors.New("indexing job imageConfig section missing indexd url and/or creds!") - } - mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds!" - if mdsConfig, ok := imageConfig["metadataService"]; ok { - mdsConfig := mdsConfig.(map[string]interface{}) - if mdsConfig["url"].(string) == "" || mdsConfig["username"].(string) == "" || mdsConfig["password"].(string) == "" { - return errors.New(mdsErrorMessage) - } - } else { - return errors.New(mdsErrorMessage) - } - } - } - return nil -} - func main() { jsonBytes, err := handlers.ReadFile(handlers.LookupCredFile()) if err != nil { @@ -85,7 +61,7 @@ func main() { jobConfigs := make([]handlers.JobConfig, 0) json.Unmarshal(b, &jobConfigs) - if err := checkIndexingJobsImageConfig(jobConfigs); err != nil { + if err := handlers.CheckIndexingJobsImageConfig(jobConfigs); err != nil { glog.Error(err) os.Exit(1) } From 6c001ab0d60c1b073e87a2a420223f6906d7cb59 Mon Sep 17 00:00:00 2001 From: John McCann Date: Sun, 18 Oct 2020 20:04:16 -0700 Subject: [PATCH 4/4] chore(CheckIndexingJobsImageConfig): use () in if --- handlers/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/utils.go b/handlers/utils.go index d3df5b2..222aaa9 100644 --- a/handlers/utils.go +++ b/handlers/utils.go @@ -62,13 +62,13 @@ func CheckIndexingJobsImageConfig(jobConfigs []JobConfig) error { for _, jobConfig := range jobConfigs { if jobConfig.Name == "indexing" { imageConfig := jobConfig.ImageConfig.(map[string]interface{}) - if imageConfig["url"].(string) == "" || imageConfig["username"].(string) == "" || imageConfig["password"].(string) == "" { + if (imageConfig["url"].(string) == "") || (imageConfig["username"].(string) == "") || (imageConfig["password"].(string) == "") { return errors.New("indexing job imageConfig section missing indexd url and/or creds!") } mdsErrorMessage := "indexing job imageConfig section missing metadataService url and/or creds!" if mdsConfig, ok := imageConfig["metadataService"]; ok { mdsConfig := mdsConfig.(map[string]interface{}) - if mdsConfig["url"].(string) == "" || mdsConfig["username"].(string) == "" || mdsConfig["password"].(string) == "" { + if (mdsConfig["url"].(string) == "") || (mdsConfig["username"].(string) == "") || (mdsConfig["password"].(string) == "") { return errors.New(mdsErrorMessage) } } else {