Skip to content

Commit

Permalink
Merge pull request #100690 from feiskyer/fix-node-pip
Browse files Browse the repository at this point in the history 
azure: fix node public IP not able to fetch issues from IMDS
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored May 6, 2021
2 parents 44e7d15 + 85daff0 commit 7a0092f
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 15 deletions.
2 changes: 1 addition & 1 deletion staging/src/k8s.io/legacy-cloud-providers/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
az.Config = *config
az.Environment = *env
az.ResourceRequestBackoff = resourceRequestBackoff
az.metadata, err = NewInstanceMetadataService(metadataURL)
az.metadata, err = NewInstanceMetadataService(imdsServer)
if err != nil {
return err
}
Expand Down
125 changes: 114 additions & 11 deletions staging/src/k8s.io/legacy-cloud-providers/azure/azure_instance_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,18 @@ import (
"net/http"
"time"

"k8s.io/klog/v2"
azcache "k8s.io/legacy-cloud-providers/azure/cache"
)

const (
metadataCacheTTL = time.Minute
metadataCacheKey = "InstanceMetadata"
metadataURL = "http://169.254.169.254/metadata/instance"
metadataCacheTTL = time.Minute
metadataCacheKey = "InstanceMetadata"
imdsInstanceAPIVersion = "2019-03-11"
imdsLoadBalancerAPIVersion = "2020-10-01"
imdsServer = "http://169.254.169.254"
imdsInstanceURI = "/metadata/instance"
imdsLoadBalancerURI = "/metadata/loadbalancer"
)

// NetworkMetadata contains metadata about an instance's network
Expand Down Expand Up @@ -86,19 +91,35 @@ type InstanceMetadata struct {
Network *NetworkMetadata `json:"network,omitempty"`
}

// PublicIPMetadata represents the public IP metadata.
type PublicIPMetadata struct {
FrontendIPAddress string `json:"frontendIpAddress,omitempty"`
PrivateIPAddress string `json:"privateIpAddress,omitempty"`
}

// LoadbalancerProfile represents load balancer profile in IMDS.
type LoadbalancerProfile struct {
PublicIPAddresses []PublicIPMetadata `json:"publicIpAddresses,omitempty"`
}

// LoadBalancerMetadata represents load balancer metadata.
type LoadBalancerMetadata struct {
LoadBalancer *LoadbalancerProfile `json:"loadbalancer,omitempty"`
}

// InstanceMetadataService knows how to query the Azure instance metadata server.
type InstanceMetadataService struct {
metadataURL string
imsCache *azcache.TimedCache
imdsServer string
imsCache *azcache.TimedCache
}

// NewInstanceMetadataService creates an instance of the InstanceMetadataService accessor object.
func NewInstanceMetadataService(metadataURL string) (*InstanceMetadataService, error) {
func NewInstanceMetadataService(imdsServer string) (*InstanceMetadataService, error) {
ims := &InstanceMetadataService{
metadataURL: metadataURL,
imdsServer: imdsServer,
}

imsCache, err := azcache.NewTimedcache(metadataCacheTTL, ims.getInstanceMetadata)
imsCache, err := azcache.NewTimedcache(metadataCacheTTL, ims.getMetadata)
if err != nil {
return nil, err
}
Expand All @@ -107,8 +128,52 @@ func NewInstanceMetadataService(metadataURL string) (*InstanceMetadataService, e
return ims, nil
}

func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}, error) {
req, err := http.NewRequest("GET", ims.metadataURL, nil)
func (ims *InstanceMetadataService) getMetadata(key string) (interface{}, error) {
instanceMetadata, err := ims.getInstanceMetadata(key)
if err != nil {
return nil, err
}

if instanceMetadata.Network != nil && len(instanceMetadata.Network.Interface) > 0 {
netInterface := instanceMetadata.Network.Interface[0]
if (len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PublicIP) > 0) ||
(len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PublicIP) > 0) {
// Return if public IP address has already part of instance metadata.
return instanceMetadata, nil
}

loadBalancerMetadata, err := ims.getLoadBalancerMetadata()
if err != nil || loadBalancerMetadata == nil || loadBalancerMetadata.LoadBalancer == nil {
// Log a warning since loadbalancer metadata may not be available when the VM
// is not in standard LoadBalancer backend address pool.
klog.V(4).Infof("Warning: failed to get loadbalancer metadata: %v", err)
return instanceMetadata, nil
}

publicIPs := loadBalancerMetadata.LoadBalancer.PublicIPAddresses
if len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PrivateIP) > 0 {
for _, pip := range publicIPs {
if pip.PrivateIPAddress == netInterface.IPV4.IPAddress[0].PrivateIP {
netInterface.IPV4.IPAddress[0].PublicIP = pip.FrontendIPAddress
break
}
}
}
if len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PrivateIP) > 0 {
for _, pip := range publicIPs {
if pip.PrivateIPAddress == netInterface.IPV6.IPAddress[0].PrivateIP {
netInterface.IPV6.IPAddress[0].PublicIP = pip.FrontendIPAddress
break
}
}
}
}

return instanceMetadata, nil
}

func (ims *InstanceMetadataService) getInstanceMetadata(key string) (*InstanceMetadata, error) {
req, err := http.NewRequest("GET", ims.imdsServer+imdsInstanceURI, nil)
if err != nil {
return nil, err
}
Expand All @@ -117,7 +182,7 @@ func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}

q := req.URL.Query()
q.Add("format", "json")
q.Add("api-version", "2019-03-11")
q.Add("api-version", imdsInstanceAPIVersion)
req.URL.RawQuery = q.Encode()

client := &http.Client{}
Expand Down Expand Up @@ -145,6 +210,44 @@ func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}
return &obj, nil
}

func (ims *InstanceMetadataService) getLoadBalancerMetadata() (*LoadBalancerMetadata, error) {
req, err := http.NewRequest("GET", ims.imdsServer+imdsLoadBalancerURI, nil)
if err != nil {
return nil, err
}
req.Header.Add("Metadata", "True")
req.Header.Add("User-Agent", "golang/kubernetes-cloud-provider")

q := req.URL.Query()
q.Add("format", "json")
q.Add("api-version", imdsLoadBalancerAPIVersion)
req.URL.RawQuery = q.Encode()

client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failure of getting loadbalancer metadata with response %q", resp.Status)
}

data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}

obj := LoadBalancerMetadata{}
err = json.Unmarshal(data, &obj)
if err != nil {
return nil, err
}

return &obj, nil
}

// GetMetadata gets instance metadata from cache.
// crt determines if we can get data from stalled cache/need fresh if cache expired.
func (ims *InstanceMetadataService) GetMetadata(crt azcache.AzureCacheReadType) (*InstanceMetadata, error) {
Expand Down
52 changes: 49 additions & 3 deletions staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"fmt"
"net"
"net/http"
"strings"
"testing"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute"
Expand Down Expand Up @@ -399,7 +400,7 @@ func TestNodeAddresses(t *testing.T) {
},
}
metadataTemplate := `{"compute":{"name":"%s"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}`

loadbalancerTemplate := `{"loadbalancer": {"publicIpAddresses": [{"frontendIpAddress": "%s","privateIpAddress": "%s"},{"frontendIpAddress": "%s","privateIpAddress": "%s"}]}}`
testcases := []struct {
name string
nodeName string
Expand All @@ -410,6 +411,7 @@ func TestNodeAddresses(t *testing.T) {
ipV6 string
ipV4Public string
ipV6Public string
loadBalancerSku string
expectedAddress []v1.NodeAddress
useInstanceMetadata bool
useCustomImsCache bool
Expand Down Expand Up @@ -484,14 +486,49 @@ func TestNodeAddresses(t *testing.T) {
expectedAddress: expectedNodeAddress,
},
{
name: "NodeAddresses should get IP addresses from local if node's name is equal to metadataName",
name: "NodeAddresses should get IP addresses from local IMDS if node's name is equal to metadataName",
nodeName: "vm1",
metadataName: "vm1",
vmType: vmTypeStandard,
ipV4: "10.240.0.1",
ipV4Public: "192.168.1.12",
ipV6: "1111:11111:00:00:1111:1111:000:111",
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
loadBalancerSku: "basic",
useInstanceMetadata: true,
expectedAddress: []v1.NodeAddress{
{
Type: v1.NodeHostName,
Address: "vm1",
},
{
Type: v1.NodeInternalIP,
Address: "10.240.0.1",
},
{
Type: v1.NodeExternalIP,
Address: "192.168.1.12",
},
{
Type: v1.NodeInternalIP,
Address: "1111:11111:00:00:1111:1111:000:111",
},
{
Type: v1.NodeExternalIP,
Address: "2222:22221:00:00:2222:2222:000:111",
},
},
},
{
name: "NodeAddresses should get IP addresses from local IMDS for standard LoadBalancer if node's name is equal to metadataName",
nodeName: "vm1",
metadataName: "vm1",
vmType: vmTypeStandard,
ipV4: "10.240.0.1",
ipV4Public: "192.168.1.12",
ipV6: "1111:11111:00:00:1111:1111:000:111",
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
loadBalancerSku: "standard",
useInstanceMetadata: true,
expectedAddress: []v1.NodeAddress{
{
Expand Down Expand Up @@ -533,10 +570,19 @@ func TestNodeAddresses(t *testing.T) {

mux := http.NewServeMux()
mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.RequestURI, imdsLoadBalancerURI) {
fmt.Fprintf(w, loadbalancerTemplate, test.ipV4Public, test.ipV4, test.ipV6Public, test.ipV6)
return
}

if test.metadataTemplate != "" {
fmt.Fprintf(w, test.metadataTemplate)
} else {
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public)
if test.loadBalancerSku == "standard" {
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, "", test.ipV6, "")
} else {
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public)
}
}
}))
go func() {
Expand Down

0 comments on commit 7a0092f

Please sign in to comment.