diff --git a/.gitignore b/.gitignore index b3f3915d..7f8c4b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ # End of https://www.toptal.com/developers/gitignore/api/go .dccache +hatchery_qabrh.json diff --git a/.secrets.baseline b/.secrets.baseline index 58cc1e0f..2ecab4ef 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": "go.sum", + "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2021-10-13T19:11:36Z", + "generated_at": "2023-04-07T16:22:23Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -61,6 +61,7 @@ "doc/explanation/dockstore.md": [ { "hashed_secret": "9b5925ea817163740dfb287a9894e8ab3aba2c18", + "is_secret": false, "is_verified": false, "line_number": 91, "type": "Secret Keyword" @@ -69,6 +70,7 @@ "doc/howto/configuration.md": [ { "hashed_secret": "e94cc2a86b04ad4ddc98fcbf91ed236437939d47", + "is_secret": false, "is_verified": false, "line_number": 30, "type": "Secret Keyword" @@ -77,6 +79,7 @@ "doc/howto/jupyterNotebook.md": [ { "hashed_secret": "e94cc2a86b04ad4ddc98fcbf91ed236437939d47", + "is_secret": false, "is_verified": false, "line_number": 35, "type": "Secret Keyword" @@ -85,14 +88,25 @@ "doc/howto/noVNCFirefox.md": [ { "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", + "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" } ], + "hatchery/ecs.go": [ + { + "hashed_secret": "96e77458817d8486fab4c56c8d4c51b83f85c79d", + "is_secret": false, + "is_verified": false, + "line_number": 391, + "type": "Secret Keyword" + } + ], "testData/dockstore/docker-compose.yml": [ { "hashed_secret": "770fc9a9befcecb410cd10a29eec487beb70009c", + "is_secret": false, "is_verified": false, "line_number": 81, "type": "Secret Keyword" @@ -101,6 +115,7 @@ "testData/testConfig.json": [ { "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", + "is_secret": false, "is_verified": false, "line_number": 55, "type": "Secret Keyword" diff --git a/hatchery/alb.go b/hatchery/alb.go index a0d88d72..b87c8998 100644 --- a/hatchery/alb.go +++ b/hatchery/alb.go @@ -217,6 +217,47 @@ func (creds *CREDS) CreateLoadBalancer(userName string) (*elbv2.CreateLoadBalanc return loadBalancer, targetGroup.TargetGroups[0].TargetGroupArn, listener, nil } +func (creds *CREDS) terminateLoadBalancerTargetGroup(userName string) error { + svc := elbv2.New(session.Must(session.NewSession(&aws.Config{ + Credentials: creds.creds, + Region: aws.String("us-east-1"), + }))) + tgName := truncateString(strings.ReplaceAll(os.Getenv("GEN3_ENDPOINT"), ".", "-")+userToResourceName(userName, "service")+"tg", 32) + Config.Logger.Printf("Deleting target group: %s", tgName) + tgArn, err := svc.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{ + Names: []*string{aws.String(tgName)}, + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case elbv2.ErrCodeTargetGroupNotFoundException: + // Target group not found, nothing to do + return nil + } + } else { + Config.Logger.Printf("Error describing target group: %s", err.Error()) + return err + } + } + input := &elbv2.DeleteTargetGroupInput{ + TargetGroupArn: tgArn.TargetGroups[0].TargetGroupArn, + } + + _, err = svc.DeleteTargetGroup(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case elbv2.ErrCodeResourceInUseException: + // Target group in use, nothing to do + return nil + } + } else { + Config.Logger.Printf("Error deleting target group: %s", err.Error()) + } + } + return nil +} + func (creds *CREDS) terminateLoadBalancer(userName string) error { svc := elbv2.New(session.Must(session.NewSession(&aws.Config{ Credentials: creds.creds, @@ -229,7 +270,15 @@ func (creds *CREDS) terminateLoadBalancer(userName string) error { } result, err := svc.DescribeLoadBalancers(getInput) if err != nil { - return err + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case elbv2.ErrCodeLoadBalancerNotFoundException: + // Load balancer doesn't exist, we are happy! :) + return nil + } + } else { + return err + } } if len(result.LoadBalancers) == 1 { delInput := &elbv2.DeleteLoadBalancerInput{ @@ -237,8 +286,16 @@ func (creds *CREDS) terminateLoadBalancer(userName string) error { } _, err := svc.DeleteLoadBalancer(delInput) if err != nil { - return err + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case elbv2.ErrCodeLoadBalancerNotFoundException: + fmt.Println(elbv2.ErrCodeLoadBalancerNotFoundException, aerr.Error()) + } + } else { + return err + } } } + return nil } diff --git a/hatchery/config.go b/hatchery/config.go index fa38712a..f6bef734 100644 --- a/hatchery/config.go +++ b/hatchery/config.go @@ -60,6 +60,8 @@ type PayModel struct { Name string `json:"workspace_type"` User string `json:"user_id"` AWSAccountId string `json:"account_id"` + Status string `json:"request_status"` + Local bool `json:"local"` Region string `json:"region"` Ecs bool `json:"ecs"` Subnet int `json:"subnet"` diff --git a/hatchery/ecs.go b/hatchery/ecs.go index 5993a100..95842600 100644 --- a/hatchery/ecs.go +++ b/hatchery/ecs.go @@ -58,6 +58,7 @@ func (sess *CREDS) launchEcsCluster(userName string) (*ecs.Cluster, error) { svc := sess.svc clusterName := strings.ReplaceAll(os.Getenv("GEN3_ENDPOINT"), ".", "-") + "-cluster" + // Setting up remote VPC _, err := setupVPC(userName) if err != nil { return nil, err @@ -219,7 +220,6 @@ func (sess *CREDS) statusEcsWorkspace(ctx context.Context, userName string, acce if len(args) > 0 { for i, arg := range args { if strings.Contains(*arg, "shutdown_no_activity_timeout=") { - Config.Logger.Printf("Found kernel idle shutdown time in args. Attempting to get last activity time\n") argSplit := strings.Split(*arg, "=") idleTimeLimit, err := strconv.Atoi(argSplit[len(argSplit)-1]) if err == nil { @@ -261,6 +261,7 @@ func (sess *CREDS) statusEcsWorkspace(ctx context.Context, userName string, acce // Terminate workspace running in ECS // TODO: Make this terminate ALB as well. func terminateEcsWorkspace(ctx context.Context, userName string, accessToken string, awsAcctID string) (string, error) { + Config.Logger.Printf("Terminating ECS workspace for user %s", userName) roleARN := "arn:aws:iam::" + awsAcctID + ":role/csoc_adminvm" sess := session.Must(session.NewSession(&aws.Config{ // TODO: Make this configurable @@ -320,7 +321,8 @@ func terminateEcsWorkspace(ctx context.Context, userName string, accessToken str Config.Logger.Printf("No container definition found for task definition %s, skipping API key deletion\n", taskDefName) } } - + // Terminate ECS service + Config.Logger.Printf("Terminating ECS service %s for user %s\n", svcName, userName) delServiceOutput, err := svc.svc.DeleteService(&ecs.DeleteServiceInput{ Cluster: cluster.ClusterName, Force: aws.Bool(true), @@ -331,48 +333,65 @@ func terminateEcsWorkspace(ctx context.Context, userName string, accessToken str } // Terminate load balancer + Config.Logger.Printf("Terminating load balancer for user %s\n", userName) err = svc.terminateLoadBalancer(userName) if err != nil { - return "", err + Config.Logger.Printf("Error occurred when terminating load balancer for user %s: %s\n", userName, err.Error()) } + // Terminate target group + err = svc.terminateLoadBalancerTargetGroup(userName) + if err != nil { + Config.Logger.Printf("Error occurred when terminating load balancer target group for user %s: %s\n", userName, err.Error()) + } + + // Terminate transit gateway err = teardownTransitGateway(userName) if err != nil { - return "", err + Config.Logger.Printf("Error occurred when terminating transit gateway resources for user %s: %s\n", userName, err.Error()) } return fmt.Sprintf("Service '%s' is in status: %s", userToResourceName(userName, "pod"), *delServiceOutput.Service.Status), nil } -func launchEcsWorkspace(ctx context.Context, userName string, hash string, accessToken string, payModel PayModel) error { +func launchEcsWorkspace(userName string, hash string, accessToken string, payModel PayModel) { + // Set up background context, as this runs in a goroutine + ctx := context.Background() + roleARN := "arn:aws:iam::" + payModel.AWSAccountId + ":role/csoc_adminvm" sess := session.Must(session.NewSession(&aws.Config{ // TODO: Make this configurable Region: aws.String("us-east-1"), })) svc := NewSVC(sess, roleARN) - Config.Logger.Printf("%s", userName) - hatchApp := Config.ContainersMap[hash] mem, err := mem(hatchApp.MemoryLimit) if err != nil { - return err + // Log error and return without launching workspace + Config.Logger.Printf("Failed to launch ECS workspace for user %v, Error: %v", userName, err) + return } cpu, err := cpu(hatchApp.CPULimit) if err != nil { - return err + // Log error and return without launching workspace + Config.Logger.Printf("Failed to launch ECS workspace for user %v, Error: %v", userName, err) } + // Make sure ECS cluster exists _, err = svc.launchEcsCluster(userName) if err != nil { - return err + Config.Logger.Printf("Failed to launch ECS cluster for user %v, Error: %v", userName, err) + return } + // Get Gen3 API key to be used in workspace + Config.Logger.Printf("Creating API key for user %s", userName) apiKey, err := getAPIKeyWithContext(ctx, accessToken) if err != nil { - Config.Logger.Printf("Failed to get API key for user %v, Error: %v", userName, err) - return err + Config.Logger.Printf("Failed to create API key for user %v, Error: %v. Moving on but workspace won't have API key", userName, err) + apiKey = &APIKeyStruct{} + } else { + Config.Logger.Printf("Created API key for user %v, key ID: %v", userName, apiKey.KeyID) } - Config.Logger.Printf("Created API key for user %v, key ID: %v", userName, apiKey.KeyID) envVars := []EnvVar{} for k, v := range hatchApp.Env { @@ -398,21 +417,31 @@ func launchEcsWorkspace(ctx context.Context, userName string, hash string, acces Key: "GEN3_ENDPOINT", Value: os.Getenv("GEN3_ENDPOINT"), }) + + Config.Logger.Printf("Settign up EFS for user %s", userName) volumes, err := svc.EFSFileSystem(userName) if err != nil { - return err + Config.Logger.Printf("Failed to set up EFS for user %v, Error: %v", userName, err) + return } + Config.Logger.Printf("Setting up task role for user %s", userName) taskRole, err := svc.taskRole(userName) if err != nil { - return err + // Log the error + Config.Logger.Printf("Failed to set up task role for user %v, Error: %v", userName, err) + return } + Config.Logger.Printf("Setting up execution role for user %s", userName) _, err = svc.CreateEcsTaskExecutionRole() if err != nil { - return err + // Log the error + Config.Logger.Printf("Failed to set up execution role for user %v, Error: %v", userName, err) + return } + Config.Logger.Printf("Setting up ECS task definition for user %s", userName) taskDef := CreateTaskDefinitionInput{ Image: hatchApp.Image, Cpu: cpu, @@ -483,28 +512,36 @@ func launchEcsWorkspace(ctx context.Context, userName string, hash string, acces } taskDefResult, err := svc.CreateTaskDefinition(&taskDef, userName, hash, payModel.AWSAccountId) if err != nil { + // Log the error + Config.Logger.Printf("Failed to set up task definition for user %v, Error: %v", userName, err) aerr := deleteAPIKeyWithContext(ctx, accessToken, apiKey.KeyID) if aerr != nil { Config.Logger.Printf("Error occurred when deleting API Key with ID %s for user %s: %s\n", apiKey.KeyID, userName, err.Error()) } - return err - } - - err = setupTransitGateway(userName) - if err != nil { - return err + return } + Config.Logger.Printf("Launching ECS workspace service for user %s", userName) launchTask, err := svc.launchService(ctx, taskDefResult, userName, hash, payModel) if err != nil { + // Log the error + Config.Logger.Printf("Failed to launch ECS workspace service for user %v, Error: %v", userName, err) aerr := deleteAPIKeyWithContext(ctx, accessToken, apiKey.KeyID) if aerr != nil { Config.Logger.Printf("Error occurred when deleting API Key with ID %s for user %s: %s\n", apiKey.KeyID, userName, err.Error()) } - return err + return } - fmt.Printf("Launched ECS workspace service at %s for user %s\n", launchTask, userName) - return nil + + Config.Logger.Printf("Setting up Transit Gateway for user %s", userName) + err = setupTransitGateway(userName) + if err != nil { + // Log the error + Config.Logger.Printf("Failed to set up Transit Gateway for user %v, Error: %v", userName, err) + return + } + + Config.Logger.Printf("Launched ECS workspace service at %s for user %s\n", launchTask, userName) } // Launch ECS service for task definition + LB for routing @@ -515,7 +552,6 @@ func (sess *CREDS) launchService(ctx context.Context, taskDefArn string, userNam if err != nil { return "", err } - Config.Logger.Printf("Cluster: %s", *cluster.ClusterName) networkConfig, err := sess.NetworkConfig(userName) if err != nil { @@ -551,31 +587,19 @@ func (sess *CREDS) launchService(ctx context.Context, taskDefArn string, userNam if err != nil { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { - case ecs.ErrCodeServerException: - Config.Logger.Println(ecs.ErrCodeServerException, aerr.Error()) - case ecs.ErrCodeClientException: - Config.Logger.Println(ecs.ErrCodeClientException, aerr.Error()) case ecs.ErrCodeInvalidParameterException: - Config.Logger.Println(ecs.ErrCodeInvalidParameterException, aerr.Error()) - case ecs.ErrCodeClusterNotFoundException: - Config.Logger.Println(ecs.ErrCodeClusterNotFoundException, aerr.Error()) - case ecs.ErrCodeUnsupportedFeatureException: - Config.Logger.Println(ecs.ErrCodeUnsupportedFeatureException, aerr.Error()) - case ecs.ErrCodePlatformUnknownException: - Config.Logger.Println(ecs.ErrCodePlatformUnknownException, aerr.Error()) - case ecs.ErrCodePlatformTaskDefinitionIncompatibilityException: - Config.Logger.Println(ecs.ErrCodePlatformTaskDefinitionIncompatibilityException, aerr.Error()) - case ecs.ErrCodeAccessDeniedException: - Config.Logger.Println(ecs.ErrCodeAccessDeniedException, aerr.Error()) - default: - Config.Logger.Println(aerr.Error()) + if aerr.Error() == "InvalidParameterException: Creation of service was not idempotent." { + Config.Logger.Print("Service already exists.. ") + return "", nil + } else { + Config.Logger.Println(ecs.ErrCodeInvalidParameterException, aerr.Error()) + } } } else { - // Print the error, cast err to awserr.Error to get the Code and - // Message from an error. + Config.Logger.Println(err.Error()) + return "", err } - return "", err } Config.Logger.Printf("Service launched: %s", *result.Service.ClusterArn) err = createLocalService(ctx, userName, hash, *loadBalancer.LoadBalancers[0].DNSName, payModel) diff --git a/hatchery/hatchery.go b/hatchery/hatchery.go index 9c61e104..3e2f3aa4 100644 --- a/hatchery/hatchery.go +++ b/hatchery/hatchery.go @@ -226,10 +226,25 @@ func launch(w http.ResponseWriter, r *http.Request) { if err != nil { Config.Logger.Printf(err.Error()) } - if payModel == nil { + if payModel == nil || payModel.Local { err = createLocalK8sPod(r.Context(), hash, userName, accessToken) } else if payModel.Ecs { - err = launchEcsWorkspace(r.Context(), userName, hash, accessToken, *payModel) + + if payModel.Status != "active" { + // send 500 response. + // TODO: 403 is the correct code, but it triggers a 302 to the default 403 page in revproxy instead of showing error message. + Config.Logger.Printf("Paymodel is not active. Launch forbidden for user %s", userName) + http.Error(w, "Paymodel is not active. Launch forbidden", http.StatusInternalServerError) + return + } + + Config.Logger.Printf("Launching ECS workspace for user %s", userName) + // Sending a 200 response straight away, but starting the launch in a goroutine + // TODO: Do more sanity checks before returning 200. + w.WriteHeader(http.StatusOK) + go launchEcsWorkspace(userName, hash, accessToken, *payModel) + fmt.Fprintf(w, "Launch accepted") + return } else { err = createExternalK8sPod(r.Context(), hash, userName, accessToken, *payModel) } @@ -248,17 +263,19 @@ func terminate(w http.ResponseWriter, r *http.Request) { } accessToken := getBearerToken(r) userName := getCurrentUserName(r) + Config.Logger.Printf("Terminating workspace for user %s", userName) payModel, err := getCurrentPayModel(userName) if err != nil { Config.Logger.Printf(err.Error()) } if payModel != nil && payModel.Ecs { - svc, err := terminateEcsWorkspace(r.Context(), userName, accessToken, payModel.AWSAccountId) + _, err := terminateEcsWorkspace(r.Context(), userName, accessToken, payModel.AWSAccountId) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } else { - fmt.Fprintf(w, "Terminated ECS workspace at %s", svc) + Config.Logger.Printf("Succesfully terminated all resources related to ECS workspace for user %s", userName) + fmt.Fprintf(w, "Terminated ECS workspace") } } else { err := deleteK8sPod(r.Context(), userName, accessToken, payModel) @@ -266,6 +283,7 @@ func terminate(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + Config.Logger.Printf("Terminated workspace for user %s", userName) fmt.Fprintf(w, "Terminated workspace") } } diff --git a/hatchery/paymodels.go b/hatchery/paymodels.go index 7b29c86f..68fc036b 100644 --- a/hatchery/paymodels.go +++ b/hatchery/paymodels.go @@ -22,8 +22,10 @@ func payModelsFromDatabase(userName string, current bool) (payModels *[]PayModel })) dynamodbSvc := dynamodb.New(sess) - filt := expression.Name("user_id").Equal(expression.Value(userName)) - filt = filt.And(expression.Name("request_status").Equal(expression.Value("active"))) + filtActive := expression.Name("request_status").Equal(expression.Value("active")) + filtAboveLimit := expression.Name("request_status").Equal(expression.Value("above limit")) + filt := expression.Name("user_id").Equal(expression.Value(userName)).And(filtActive.Or(filtAboveLimit)) + if current { filt = filt.And(expression.Name("current_pay_model").Equal(expression.Value(true))) } @@ -49,6 +51,7 @@ func payModelsFromDatabase(userName string, current bool) (payModels *[]PayModel var payModelMap []PayModel err = dynamodbattribute.UnmarshalListOfMaps(res.Items, &payModelMap) if err != nil { + Config.Logger.Printf("Got error unmarshalling paymodels: %s", err) return nil, err } @@ -69,44 +72,38 @@ func payModelFromConfig(userName string) (pm *PayModel, err error) { } func getCurrentPayModel(userName string) (result *PayModel, err error) { - if Config.Config.PayModelsDynamodbTable == "" { - // fallback for backward compatibility. - // Multiple paymodels not supported - pm, err := payModelFromConfig(userName) - if err != nil { - pm, err = getDefaultPayModel() - if err != nil { - return nil, NopaymodelsError - } - } - return pm, nil + + var pm *[]PayModel + + if Config.Config.PayModelsDynamodbTable != "" { + // Fetch pay models from DynamoDB with current_pay_model as `true` + pm, err = payModelsFromDatabase(userName, true) } payModel := PayModel{} - pm, err := payModelsFromDatabase(userName, true) - - if len(*pm) == 0 { - pm, err := payModelFromConfig(userName) + // If no dynamoDB or no current pay models in the DB, + // fallback to defaultPayModel from config + if pm == nil || len(*pm) == 0 { + pm, err := getDefaultPayModel() if err != nil { - pm, err = getDefaultPayModel() - if err != nil { - return nil, nil - } + return nil, nil } return pm, nil } - if len(*pm) == 1 { - payModel = (*pm)[0] - if err != nil { - Config.Logger.Printf("Got error unmarshalling: %s", err) - return nil, err - } - } + + // If more than one current pay model is found in the database if len(*pm) > 1 { - // TODO: Reset to zero current paymodels here. - // We don't want to be in a situation with multiple current paymodels - return nil, fmt.Errorf("multiple current paymodels set") + // TODO: Reset to zero current pay models here. + // We don't want to be in a situation with multiple current pay models + return nil, fmt.Errorf("multiple current pay models set") + } + + // If exactly one current pay model is found in the database + payModel = (*pm)[0] + if err != nil { + Config.Logger.Printf("Got error unmarshalling: %s", err) + return nil, err } return &payModel, nil } @@ -124,49 +121,33 @@ func getPayModelsForUser(userName string) (result *AllPayModels, err error) { return nil, fmt.Errorf("no username sent in header") } PayModels := AllPayModels{} + var payModelMap *[]PayModel - // Fallback to config-only if DynamoDB table is not configured - if Config.Config.PayModelsDynamodbTable == "" { - pm, err := payModelFromConfig(userName) + if Config.Config.PayModelsDynamodbTable != "" { + payModelMap, err = payModelsFromDatabase(userName, false) if err != nil { - pm, err = getDefaultPayModel() - if err != nil { - return nil, nil - } - } - if pm == nil { - return nil, NopaymodelsError + return nil, err } - PayModels.CurrentPayModel = pm - PayModels.PayModels = append(PayModels.PayModels, *pm) - return &PayModels, nil } - - payModelMap, err := payModelsFromDatabase(userName, false) + payModel, err := getCurrentPayModel(userName) if err != nil { return nil, err } - // temporary fallback to the config to get data for users that are not - // in DynamoDB - // TODO: remove this block once we only rely on DynamoDB - payModel, err := payModelFromConfig(userName) - if err == nil { - *payModelMap = append(*payModelMap, *payModel) + // If `getCurrentPayModel` returns nil, + // then there are no other paymodels to fallback to + if payModel == nil { + return nil, nil } - if len(*payModelMap) == 0 { - payModel, _ := getDefaultPayModel() + if payModelMap == nil { + payModelMap = &[]PayModel{*payModel} + } else if len(*payModelMap) == 0 { *payModelMap = append(*payModelMap, *payModel) } PayModels.PayModels = *payModelMap - payModel, err = getCurrentPayModel(userName) - if err != nil { - return nil, err - } - PayModels.CurrentPayModel = payModel return &PayModels, nil diff --git a/hatchery/pods.go b/hatchery/pods.go index bbba4828..8b8e1145 100644 --- a/hatchery/pods.go +++ b/hatchery/pods.go @@ -69,7 +69,7 @@ type WorkspaceStatus struct { } func getPodClient(ctx context.Context, userName string, payModelPtr *PayModel) (corev1.CoreV1Interface, bool, error) { - if payModelPtr != nil { + if payModelPtr != nil && !(*payModelPtr).Local { podClient, err := NewEKSClientset(ctx, userName, *payModelPtr) if err != nil { Config.Logger.Printf("Error fetching EKS kubeconfig: %v", err) @@ -509,8 +509,8 @@ func buildPod(hatchConfig *FullHatcheryConfig, hatchApp *Container, userName str Annotations: annotations, }, Spec: k8sv1.PodSpec{ - SecurityContext: &securityContext, - InitContainers: []k8sv1.Container{}, + SecurityContext: &securityContext, + InitContainers: []k8sv1.Container{}, EnableServiceLinks: &falseVal, Containers: []k8sv1.Container{ { @@ -624,7 +624,7 @@ func buildPod(hatchConfig *FullHatcheryConfig, hatchApp *Container, userName str func createLocalK8sPod(ctx context.Context, hash string, userName string, accessToken string) error { hatchApp := Config.ContainersMap[hash] - + Config.Logger.Printf("Creating a Local K8s Pod") var extraVars []k8sv1.EnvVar apiKey, err := getAPIKeyWithContext(ctx, accessToken) if err != nil { @@ -747,7 +747,7 @@ func createLocalK8sPod(ctx context.Context, hash string, userName string, access func createExternalK8sPod(ctx context.Context, hash string, userName string, accessToken string, payModel PayModel) error { hatchApp := Config.ContainersMap[hash] - + Config.Logger.Printf("Creating a External K8s Pod") podClient, err := NewEKSClientset(ctx, userName, payModel) if err != nil { Config.Logger.Printf("Failed to create pod client for user %v, Error: %v", userName, err) diff --git a/hatchery/transitgateway.go b/hatchery/transitgateway.go index 6baf6139..26219a6a 100644 --- a/hatchery/transitgateway.go +++ b/hatchery/transitgateway.go @@ -14,6 +14,7 @@ import ( ) func setupTransitGateway(userName string) error { + Config.Logger.Printf("Setting up transit gateway") _, err := createTransitGateway(userName) if err != nil { return fmt.Errorf("error creating transit gateway: %s", err.Error()) @@ -23,11 +24,12 @@ func setupTransitGateway(userName string) error { if err != nil { return fmt.Errorf("failed to setup remote account: %s", err.Error()) } - + Config.Logger.Printf("Remote account setup complete") return nil } func teardownTransitGateway(userName string) error { + Config.Logger.Printf("Terminating remote transit gateway attachment for user %s\n", userName) err := setupRemoteAccount(userName, true) if err != nil { return err @@ -83,7 +85,7 @@ func describeMainNetwork(vpcid string, svc *ec2.EC2) (*NetworkInfo, error) { }, { Name: aws.String("tag:Name"), - Values: []*string{aws.String("main")}, + Values: []*string{aws.String("eks_private")}, }, }, } @@ -112,7 +114,6 @@ func createTransitGateway(userName string) (*string, error) { ec2Local := ec2.New(sess) vpcid := os.Getenv("GEN3_VPCID") - Config.Logger.Printf("VPCID: %s", vpcid) tgwName := strings.ReplaceAll(os.Getenv("GEN3_ENDPOINT"), ".", "-") + "-tgw" // Check for existing transit gateway exTg, err := ec2Local.DescribeTransitGateways(&ec2.DescribeTransitGatewaysInput{ @@ -162,13 +163,20 @@ func createTransitGateway(userName string) (*string, error) { return nil, err } Config.Logger.Printf("Transit gateway created: %s", *tg.TransitGateway.TransitGatewayId) + + // Create Transit Gateway Attachment in local VPC + Config.Logger.Printf("Creating tgw attachment in local VPC: %s", vpcid) tgwAttachment, err := createTransitGatewayAttachments(ec2Local, vpcid, *tg.TransitGateway.TransitGatewayId, true, nil, userName) if err != nil { return nil, err } Config.Logger.Printf("Attachment created: %s", *tgwAttachment) + + // Create Transit Gateway Route Table _, err = TGWRoutes(userName, tg.TransitGateway.Options.AssociationDefaultRouteTableId, tgwAttachment, ec2Local, true, false, nil) if err != nil { + // Log error + Config.Logger.Printf("Failed to create TGW route table: %s", err.Error()) return nil, err } resourceshare, err := shareTransitGateway(sess, *tg.TransitGateway.TransitGatewayArn, pm.AWSAccountId) @@ -178,21 +186,33 @@ func createTransitGateway(userName string) (*string, error) { Config.Logger.Printf("Resources shared: %s", *resourceshare) return tg.TransitGateway.TransitGatewayId, nil } else { + Config.Logger.Print("Existing transit gateway found. Skipping creation...") tgwAttachment, err := createTransitGatewayAttachments(ec2Local, vpcid, *exTg.TransitGateways[len(exTg.TransitGateways)-1].TransitGatewayId, true, nil, userName) if err != nil { return nil, err } - Config.Logger.Printf("Attachment created: %s", *tgwAttachment) + Config.Logger.Printf("Local TGW Attachment created: %s", *tgwAttachment) resourceshare, err := shareTransitGateway(sess, *exTg.TransitGateways[len(exTg.TransitGateways)-1].TransitGatewayArn, pm.AWSAccountId) if err != nil { return nil, err } + + // Updating the route table to include the new attachment + Config.Logger.Printf("Updating route table to include new attachment: %s", *tgwAttachment) + _, err = TGWRoutes(userName, exTg.TransitGateways[len(exTg.TransitGateways)-1].Options.AssociationDefaultRouteTableId, tgwAttachment, ec2Local, true, false, nil) + if err != nil { + // Log error + Config.Logger.Printf("Failed to create TGW route table: %s", err.Error()) + return nil, err + } + Config.Logger.Printf("Resources shared: %s", *resourceshare) return exTg.TransitGateways[len(exTg.TransitGateways)-1].TransitGatewayId, nil } } func createTransitGatewayAttachments(svc *ec2.EC2, vpcid string, tgwid string, local bool, sess *CREDS, userName string) (*string, error) { + Config.Logger.Printf("Creating transit gateway attachment for VPC: %s", vpcid) // Check for existing transit gateway tgInput := &ec2.DescribeTransitGatewaysInput{ TransitGatewayIds: []*string{aws.String(tgwid)}, @@ -225,7 +245,7 @@ func createTransitGatewayAttachments(svc *ec2.EC2, vpcid string, tgwid string, l } for *exTg.TransitGateways[0].State != "available" { Config.Logger.Printf("TransitGateway is in state: %s ... Waiting for 5 seconds", *exTg.TransitGateways[0].State) - // sleep for 2 sec + // sleep for 10 sec time.Sleep(10 * time.Second) exTg, _ = svc.DescribeTransitGateways(tgInput) } @@ -292,8 +312,8 @@ func createTransitGatewayAttachments(svc *ec2.EC2, vpcid string, tgwid string, l } } -func deleteTransitGatewayAttachment(svc *ec2.EC2, tgwid string) (*string, error) { - +func deleteTransitGatewayAttachment(svc *ec2.EC2, tgwid string, userName string) (*string, error) { + tgwAttachmentName := userToResourceName(userName, "service") + "tgwa" exTgwAttachmentInput := &ec2.DescribeTransitGatewayAttachmentsInput{ Filters: []*ec2.Filter{ { @@ -304,14 +324,38 @@ func deleteTransitGatewayAttachment(svc *ec2.EC2, tgwid string) (*string, error) Name: aws.String("state"), Values: []*string{aws.String("available"), aws.String("pending")}, }, + { + Name: aws.String("tag:Name"), + Values: []*string{ + aws.String(tgwAttachmentName), + }, + }, + { + Name: aws.String("tag:Environment"), + Values: []*string{ + aws.String(os.Getenv("GEN3_ENDPOINT")), + }, + }, }, } exTgwAttachment, err := svc.DescribeTransitGatewayAttachments(exTgwAttachmentInput) if err != nil { - return nil, err + Config.Logger.Printf("Error: %s", err.Error()) + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case "InvalidTransitGatewayID.NotFound": + // No TGW attachment found, we are happy :) + return nil, nil + } + } else { + Config.Logger.Printf("Error: %s", err.Error()) + return nil, err + } } if len(exTgwAttachment.TransitGatewayAttachments) == 0 { - return nil, fmt.Errorf("No transit gateway attachments found") + // No transit gateway attachment found, we are happy :) + Config.Logger.Printf("No TGW attachment found, we are happy :)") + return nil, nil } delTGWAttachmentInput := &ec2.DeleteTransitGatewayVpcAttachmentInput{ @@ -418,7 +462,6 @@ func setupRemoteAccount(userName string, teardown bool) error { }))) vpcid := os.Getenv("GEN3_VPCID") - Config.Logger.Printf("VPCID: %s", vpcid) err = svc.acceptTGWShare() if err != nil { return err @@ -463,11 +506,10 @@ func setupRemoteAccount(userName string, teardown bool) error { } var tgw_attachment *string if teardown { - tgw_attachment, err = deleteTransitGatewayAttachment(ec2Remote, *exTg.TransitGateways[0].TransitGatewayId) + tgw_attachment, err = deleteTransitGatewayAttachment(ec2Remote, *exTg.TransitGateways[0].TransitGatewayId, userName) if err != nil { return err } - Config.Logger.Printf("tgw_attachment: %s", *tgw_attachment) } else { tgw_attachment, err = createTransitGatewayAttachments(ec2Remote, *vpc.Vpcs[0].VpcId, *exTg.TransitGateways[0].TransitGatewayId, false, &svc, userName) if err != nil { @@ -521,7 +563,6 @@ func (creds *CREDS) acceptTGWShare() error { func TGWRoutes(userName string, tgwRoutetableId *string, tgwAttachmentId *string, svc *ec2.EC2, local bool, teardown bool, sess *CREDS) (*string, error) { var networkInfo *NetworkInfo vpcid := os.Getenv("GEN3_VPCID") - Config.Logger.Printf("VPCID: %s", vpcid) err := *new(error) if local { networkInfo, err = describeMainNetwork(vpcid, svc) @@ -542,12 +583,20 @@ func TGWRoutes(userName string, tgwRoutetableId *string, tgwAttachmentId *string return nil, fmt.Errorf("error DescribeTransitGatewayAttachments: %s", err.Error()) } if teardown { + delRouteInput := &ec2.DeleteTransitGatewayRouteInput{ DestinationCidrBlock: networkInfo.vpc.Vpcs[0].CidrBlock, TransitGatewayRouteTableId: tgwRoutetableId, } _, err := svc.DeleteTransitGatewayRoute(delRouteInput) if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case "InvalidRoute.NotFound": + // Route already deleted, we are happy :) + return nil, nil + } + } return nil, err } return delRouteInput.TransitGatewayRouteTableId, nil @@ -572,6 +621,8 @@ func TGWRoutes(userName string, tgwRoutetableId *string, tgwAttachmentId *string } exRoutes, err := svc.SearchTransitGatewayRoutes(exRoutesInput) if err != nil { + // log error + Config.Logger.Printf("error SearchTransitGatewayRoutes: %s", err.Error()) return nil, err } @@ -651,12 +702,20 @@ func VPCRoutes(remote_network_info *NetworkInfo, main_network_info *NetworkInfo, Config.Logger.Printf("Route added to local VPC. %s", localRoute) return nil } else { + // Delete Routes for VPC + Config.Logger.Printf("Deleting Routes for remote VPC %s", *remote_network_info.vpc.Vpcs[0].VpcId) remoteDeleteRouteInput := &ec2.DeleteRouteInput{ DestinationCidrBlock: main_network_info.vpc.Vpcs[0].CidrBlock, RouteTableId: remote_network_info.routeTable.RouteTables[0].RouteTableId, } _, err := ec2_remote.DeleteRoute(remoteDeleteRouteInput) if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case "InvalidRoute.NotFound": + return nil + } + } return err } localDeleteRouteInput := &ec2.DeleteRouteInput{ diff --git a/hatchery/vpc.go b/hatchery/vpc.go index 5143e713..21aa049e 100644 --- a/hatchery/vpc.go +++ b/hatchery/vpc.go @@ -12,6 +12,7 @@ import ( ) func setupVPC(userName string) (*string, error) { + Config.Logger.Printf("Setting up VPC for user %s", userName) pm, err := getCurrentPayModel(userName) if err != nil { return nil, err @@ -32,7 +33,7 @@ func setupVPC(userName string) (*string, error) { // Subnets // TODO: make base CIDR configurable? - cidrstring := "192.165.0.0/12" + cidrstring := "192.160.0.0/12" _, IPNet, _ := net.ParseCIDR(cidrstring) subnet, err := cidr.Subnet(IPNet, 14, pm.Subnet) if err != nil { @@ -40,6 +41,8 @@ func setupVPC(userName string) (*string, error) { } subnetString := subnet.String() + Config.Logger.Printf("Using subnet: %s for user %s. Make sure this does not overlap with other users", subnetString, userName) + // VPC stuff vpcname := userToResourceName(userName, "service") + "-" + strings.ReplaceAll(os.Getenv("GEN3_ENDPOINT"), ".", "-") + "-vpc" descVPCInput := &ec2.DescribeVpcsInput{ @@ -171,6 +174,7 @@ func createSubnet(vpccidr string, vpcid string, svc *ec2.EC2) error { } func createInternetGW(name string, vpcid string, svc *ec2.EC2) (*string, error) { + Config.Logger.Printf("Setting up internet Gateway for VPC: %s", vpcid) describeInternetGWInput := &ec2.DescribeInternetGatewaysInput{ Filters: []*ec2.Filter{ { @@ -184,6 +188,7 @@ func createInternetGW(name string, vpcid string, svc *ec2.EC2) (*string, error) return nil, err } if len(exIgw.InternetGateways) == 0 { + Config.Logger.Printf("No existing gateways found. Creating internet gateway for VPC: %s", vpcid) createInternetGWInput := &ec2.CreateInternetGatewayInput{ TagSpecifications: []*ec2.TagSpecification{ { @@ -235,6 +240,7 @@ func createInternetGW(name string, vpcid string, svc *ec2.EC2) (*string, error) return igw.InternetGateway.InternetGatewayId, nil } else { if len(exIgw.InternetGateways[0].Attachments) == 0 { + Config.Logger.Printf("Existing gateway found but not attached to IGW. Attaching internet gateway for VPC: %s", vpcid) _, err = svc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ InternetGatewayId: exIgw.InternetGateways[0].InternetGatewayId, VpcId: &vpcid, @@ -256,9 +262,7 @@ func createInternetGW(name string, vpcid string, svc *ec2.EC2) (*string, error) return nil, err } - Config.Logger.Printf("Routes: %s", routeTable.RouteTables[0].Routes) - - route, err := svc.CreateRoute(&ec2.CreateRouteInput{ + _, err = svc.CreateRoute(&ec2.CreateRouteInput{ DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: exIgw.InternetGateways[0].InternetGatewayId, RouteTableId: routeTable.RouteTables[0].RouteTableId, @@ -266,7 +270,6 @@ func createInternetGW(name string, vpcid string, svc *ec2.EC2) (*string, error) if err != nil { return nil, err } - Config.Logger.Printf("Route: %s", route) return exIgw.InternetGateways[0].InternetGatewayId, nil }