Skip to content

Commit

Permalink
Refactor cloud inits and filter in operation
Browse files Browse the repository at this point in the history
  • Loading branch information
@mhmxs
mhmxs committed Mar 21, 2018
1 parent a1ae4ad commit f02856c
Show file tree
Hide file tree
Showing 11 changed files with 211 additions and 159 deletions.
16 changes: 13 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,20 @@ PKG_BASE=github.com/hortonworks/cloud-cost-reducer
BUILD_TIME=$(shell date +%FT%T)
LDFLAGS=-X $(PKG_BASE)/context.Version=${VERSION} -X $(PKG_BASE)/context.BuildTime=${BUILD_TIME}

GCP_OWNER_LABEL?=owner
LDFLAGS+= -X $(PKG_BASE)/gcp.OwnerLabel=$(GCP_OWNER_LABEL)
AWS_IGNORE_LABEL?=cloud-cost-reducer-ignore
LDFLAGS+= -X $(PKG_BASE)/context.AwsIgnoreLabel=$(AWS_IGNORE_LABEL)
AWS_OWNER_LABEL?=Owner
LDFLAGS+= -X $(PKG_BASE)/contextgcp.AwsOwnerLabel=$(AWS_OWNER_LABEL)

AZURE_IGNORE_LABEL?=cloud-cost-reducer-ignore
LDFLAGS+= -X $(PKG_BASE)/context.AzureIgnoreLabel=$(AZURE_IGNORE_LABEL)
AZURE_OWNER_LABEL?=Owner
LDFLAGS+= -X $(PKG_BASE)/contextgcp.AzureOwnerLabel=$(AZURE_OWNER_LABEL)

GCP_IGNORE_LABEL?=cloud-cost-reducer-ignore
LDFLAGS+= -X $(PKG_BASE)/gcp.IgnoreLabel=$(GCP_IGNORE_LABEL)
LDFLAGS+= -X $(PKG_BASE)/context.GcpIgnoreLabel=$(GCP_IGNORE_LABEL)
GCP_OWNER_LABEL?=owner
LDFLAGS+= -X $(PKG_BASE)/contextgcp.GcpOwnerLabel=$(GCP_OWNER_LABEL)

GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./.git/*")

Expand Down
2 changes: 1 addition & 1 deletion action/termination.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func (a TerminationAction) Execute(allInstances []*types.Instance) {
log.Errorf("[TERMINATION] Failed to terminate instances on %s, err: %s", cType.String(), err.Error())
}
}
}(t, p)
}(t, p())
}
wg.Wait()
}
133 changes: 70 additions & 63 deletions aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"crypto/tls"
"errors"
"fmt"
"net/http"
"sync"

Expand All @@ -20,28 +21,74 @@ var (
)

func init() {
log.Infof("[AWS] Trying to register as provider")
var err error
regions, err = getRegions()
if err != nil {
log.Errorf("[AWS] Failed to authenticate, err: %s", err.Error())
return
context.CloudProviders[types.AWS] = func() types.CloudProvider {
prepare()
return new(AwsProvider)
}
for _, region := range regions {
if client, err := newEc2Client(&region); err != nil {
log.Errorf("[AWS] Failed to create client in region %s, err: %s", region, err.Error())
return
} else {
regionClients[region] = client
}

func prepare() {
if len(regionClients) == 0 {
log.Infof("[AWS] Trying to prepare")
var err error
regions, err = getRegions()
if err != nil {
panic("[AWS] Failed to authenticate, err: " + err.Error())
}
for _, region := range regions {
if client, err := newEc2Client(&region); err != nil {
panic(fmt.Sprintf("[AWS] Failed to create client in region %s, err: %s", region, err.Error()))
} else {
regionClients[region] = client
}
}
log.Infof("[AWS] Successfully prepared")
}
context.CloudProviders[types.AWS] = new(AwsProvider)
log.Infof("[AWS] Successfully registered as provider")
}

type AwsProvider struct {
}

func (p *AwsProvider) GetRunningInstances() ([]*types.Instance, error) {
instChan := make(chan *types.Instance, 5)
wg := sync.WaitGroup{}
wg.Add(len(regions))
for _, region := range regions {
go func(region string) {
defer wg.Done()

filterName := "instance-state-name"
filterValue := ec2.InstanceStateNameRunning
runningFilter := []*ec2.Filter{{Name: &filterName, Values: []*string{&filterValue}}}
instanceResult, e := regionClients[region].DescribeInstances(&ec2.DescribeInstancesInput{
Filters: runningFilter,
})
if e != nil {
log.Errorf("[AWS] Failed to fetch the running instances in region: %s, err: %s", region, e)
return
}
for _, res := range instanceResult.Reservations {
for _, inst := range res.Instances {
instChan <- newInstance(inst)
}
}
}(region)
}
go func() {
wg.Wait()
close(instChan)
}()
instances := []*types.Instance{}
for inst := range instChan {
instances = append(instances, inst)
}
return instances, nil
}

func (a AwsProvider) TerminateInstances([]*types.Instance) error {
return errors.New("[AWS] Termination not supported")
}

func getRegions() ([]string, error) {
client, err := newEc2Client(nil)
if err != nil {
Expand Down Expand Up @@ -86,49 +133,6 @@ func newEc2Client(region *string) (*ec2.EC2, error) {
return ec2.New(awsSession), nil
}

func (p *AwsProvider) GetRunningInstances() ([]*types.Instance, error) {
instChan := make(chan *types.Instance, 5)
wg := sync.WaitGroup{}
wg.Add(len(regions))
for _, region := range regions {
go func(region string) {
defer wg.Done()

filterName := "instance-state-name"
filterValue := ec2.InstanceStateNameRunning
runningFilter := []*ec2.Filter{{Name: &filterName, Values: []*string{&filterValue}}}
instanceResult, e := regionClients[region].DescribeInstances(&ec2.DescribeInstancesInput{
Filters: runningFilter,
})
if e != nil {
log.Errorf("[AWS] Failed to fetch the running instances in region: %s, err: %s", region, e)
return
}
for _, res := range instanceResult.Reservations {
for _, inst := range res.Instances { // TODO filter by ignore tag
tags := getTags(inst.Tags)
instChan <- &types.Instance{
Id: *inst.InstanceId,
Name: tags["Name"],
Created: *inst.LaunchTime,
CloudType: types.AWS,
Tags: tags,
}
}
}
}(region)
}
go func() {
wg.Wait()
close(instChan)
}()
instances := []*types.Instance{}
for inst := range instChan {
instances = append(instances, inst)
}
return instances, nil
}

func getTags(ec2Tags []*ec2.Tag) types.Tags {
tags := make(types.Tags, 0)
for _, t := range ec2Tags {
Expand All @@ -137,10 +141,13 @@ func getTags(ec2Tags []*ec2.Tag) types.Tags {
return tags
}

func (a AwsProvider) GetOwnerLessInstances() ([]*types.Instance, error) {
return nil, errors.New("[AWS] Ownerless operation not supported")
}

func (a AwsProvider) TerminateInstances([]*types.Instance) error {
return errors.New("[AWS] Termination not supported")
func newInstance(inst *ec2.Instance) *types.Instance {
tags := getTags(inst.Tags)
return &types.Instance{
Id: *inst.InstanceId,
Name: tags["Name"],
Created: *inst.LaunchTime,
CloudType: types.AWS,
Tags: tags,
}
}
63 changes: 34 additions & 29 deletions azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,42 +8,43 @@ import (
"github.com/hortonworks/cloud-cost-reducer/utils"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
"github.com/Azure/go-autorest/autorest/azure/auth"
log "github.com/Sirupsen/logrus"
"github.com/hortonworks/cloud-cost-reducer/context"
"github.com/hortonworks/cloud-cost-reducer/types"
)

var (
IgnoreLabel string = "cloud-cost-reducer-ignore"
subscriptionId string
vmClient compute.VirtualMachinesClient
rgClient resources.GroupsClient
resClient resources.Client
// rgClient resources.GroupsClient
// resClient resources.Client
typesToCollect = map[string]bool{"Microsoft.Compute/virtualMachines": true}
)

var typesToCollect = map[string]bool{"Microsoft.Compute/virtualMachines": true}

func init() {
subscriptionId = os.Getenv("AZURE_SUBSCRIPTION_ID")
if len(subscriptionId) > 0 {
log.Infof("[AZURE] Trying to register as provider")
context.CloudProviders[types.AZURE] = func() types.CloudProvider {
prepare()
return new(AzureProvider)
}
}

func prepare() {
if len(vmClient.SubscriptionID) == 0 {
subscriptionId = os.Getenv("AZURE_SUBSCRIPTION_ID")
if len(subscriptionId) == 0 {
panic("[AZURE] AZURE_SUBSCRIPTION_ID environment variable is missing")
}
log.Infof("[AZURE] Trying to prepare")
authorization, err := auth.NewAuthorizerFromEnvironment()
if err != nil {
log.Errorf("[AZURE] Failed to authenticate, err: %s", err.Error())
return
panic("[AZURE] Failed to authenticate, err: " + err.Error())
}
vmClient = compute.NewVirtualMachinesClient(subscriptionId)
vmClient.Authorizer = authorization
rgClient = resources.NewGroupsClient(subscriptionId)
rgClient.Authorizer = authorization
resClient = resources.NewClient(subscriptionId)
resClient.Authorizer = authorization

context.CloudProviders[types.AZURE] = new(AzureProvider)
log.Info("[AZURE] Successfully registered as provider")
} else {
log.Warn("[AZURE] AZURE_SUBSCRIPTION_ID environment variable is missing")
log.Info("[AZURE] Successfully prepared")
}
}

Expand All @@ -57,24 +58,19 @@ func (p *AzureProvider) GetRunningInstances() ([]*types.Instance, error) {
log.Errorf("[AZURE] Failed to fetch the running instances, err: %s", err.Error())
return nil, err
}
for _, inst := range result.Values() { // TODO filter by ignore tag
instances = append(instances, &types.Instance{
Name: *inst.Name,
Id: *inst.ID,
CloudType: types.AZURE,
Tags: utils.ConvertTags(inst.Tags),
})
for _, inst := range result.Values() {
instances = append(instances, newInstance(inst))
}
return instances, nil
}

func (a AzureProvider) GetOwnerLessInstances() ([]*types.Instance, error) {
return nil, errors.New("[AZURE] Ownerless operation not supported")
}

func (a AzureProvider) TerminateInstances([]*types.Instance) error {
return errors.New("[AZURE] Termination not supported")
// AZURE
// rgClient = resources.NewGroupsClient(subscriptionId)
// rgClient.Authorizer = authorization
// resClient = resources.NewClient(subscriptionId)
// resClient.Authorizer = authorization
// instances := make([]*types.Instance, 0)
// groups, err := rgClient.List(ctx.Background(), "", nil)
// if err != nil {
Expand Down Expand Up @@ -103,3 +99,12 @@ func (a AzureProvider) TerminateInstances([]*types.Instance) error {

// return instances, nil
}

func newInstance(inst compute.VirtualMachine) *types.Instance {
return &types.Instance{
Name: *inst.Name,
Id: *inst.ID,
CloudType: types.AZURE,
Tags: utils.ConvertTags(inst.Tags),
}
}
24 changes: 23 additions & 1 deletion context/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,36 @@ import (
var (
Version string
BuildTime string

AwsIgnoreLabel string = "cloud-cost-reducer-ignore"
AzureIgnoreLabel string = "cloud-cost-reducer-ignore"
GcpIgnoreLabel string = "cloud-cost-reducer-ignore"

AwsOwnerLabel string = "Owner"
AzureOwnerLabel string = "Owner"
GcpOwnerLabel string = "owner"
)

var DryRun = false

var Operations = make(map[types.OpType]types.Operation)

var CloudProviders = make(map[types.CloudType]types.CloudProvider)
var CloudProviders = make(map[types.CloudType]func() types.CloudProvider)

var IgnoreLabels = make(map[types.CloudType]string)

var OwnerLabels = make(map[types.CloudType]string)

var Dispatchers = make(map[string]types.Dispatcher)

var Actions = make(map[types.ActionType]types.Action)

func init() {
IgnoreLabels[types.AWS] = AwsIgnoreLabel
IgnoreLabels[types.AZURE] = AzureIgnoreLabel
IgnoreLabels[types.GCP] = GcpIgnoreLabel

OwnerLabels[types.AWS] = AwsOwnerLabel
OwnerLabels[types.AZURE] = AzureOwnerLabel
OwnerLabels[types.GCP] = GcpOwnerLabel
}
Loading

0 comments on commit f02856c

Please sign in to comment.