Skip to content

Commit

Permalink
UPSTREAM: 89885: allow to read openstack cloud provider config from a…
Browse files Browse the repository at this point in the history 
… secret

This patch brings back the downstream changes that were introduced
to allow reading openstack cloud provider config from a secret.
They are available in release-4.4, but were reverted in master with
openshift/origin#24719

This change includes:

- Ability to read metadata values for kubelet. Since the service
does not have access to the secret to read the configuration, but
it needs data to download (e.g. hostname or flavor), we are trying
to get it from the metadata server.

- Deprecation of kubeConfig parameter. Now we read the file that
was provided with --kubeconfig option.

Origin-commit: f95edc26155a29769b3c5b80c03755a01a87b5fc

UPSTREAM: 89885: legacy-cloud-provider/openstack: include / prefix in instance ID output

When we want to read an instance ID from the metadata service, cloud provider
doesn't include "/" prefix, which is required for successful parsing of
provider the ID later.
This commit adds the missing "/" prefix to the output.

UPSTREAM: 89885: SQUASH: Fix Cinder provisioning crashing on nil cloud provider

OpenStack cloud provider must not use nil when provisioning a Cinder
volume.

UPSTREAM: 89885: SQUASH: Report OpenStack cloud initialization errors
  • Loading branch information
@Fedosin @soltysh
Fedosin authored and soltysh committed Sep 8, 2021
1 parent 0873fc9 commit 6b22017
Show file tree
Hide file tree
Showing 4 changed files with 220 additions and 61 deletions.
5 changes: 2 additions & 3 deletions staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,12 @@ go_library(
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library",
Expand Down
67 changes: 65 additions & 2 deletions staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,18 @@ const (

// configDriveID is used as an identifier on the metadata search order configuration.
configDriveID = "configDrive"

// We have to use AWS compatible metadata for the next urls, because OpenStack doesn't
// provide this information.

// instanceTypeURL contains url to get the instance type from metadata server.
instanceTypeURL = "http://169.254.169.254/2009-04-04/meta-data/instance-type"

// localAddressURL contains url to get the instance local ip address from metadata server.
localAddressURL = "http://169.254.169.254/2009-04-04/meta-data/local-ipv4"

// publicAddressURL contains url to get the instance public ip address from metadata server.
publicAddressURL = "http://169.254.169.254/2009-04-04/meta-data/public-ipv4"
)

// ErrBadMetadata is used to indicate a problem parsing data from metadata server
Expand Down Expand Up @@ -159,13 +171,64 @@ func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) {
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("unexpected status code when reading metadata from %s: %s", metadataURL, resp.Status)
return nil, err
return nil, fmt.Errorf("unexpected status code when reading metadata from %s: %s", metadataURL, resp.Status)
}

return parseMetadata(resp.Body)
}

func getIntanceType() (string, error) {
klog.V(4).Infof("Attempting to fetch instance type from %s", instanceTypeURL)
resp, err := http.Get(instanceTypeURL)
if err != nil {
return "", fmt.Errorf("error fetching %s: %v", instanceTypeURL, err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected status code when reading instance type from %s: %s", instanceTypeURL, resp.Status)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("cannot read the response body %s: %v", instanceTypeURL, err)
}

return string(body), nil
}

func getNodeAddress(url string) (string, error) {
klog.V(4).Infof("Attempting to fetch instance address from %s", url)
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("error fetching %s: %v", url, err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected status code when reading instance address from %s: %s", url, resp.Status)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("cannot read the response body %s: %v", url, err)
}

return string(body), nil
}

func getNodeAddresses() (string, string, error) {
localAddess, err := getNodeAddress(localAddressURL)
if err != nil {
return "", "", err
}

publicAddress, err := getNodeAddress(publicAddressURL)
if err != nil {
return "", "", err
}

return localAddess, publicAddress, nil
}

// Metadata is fixed for the current host, so cache the value process-wide
var metadataCache *Metadata

Expand Down
140 changes: 84 additions & 56 deletions staging/src/k8s.io/legacy-cloud-providers/openstack/openstack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,10 @@ import (
"gopkg.in/gcfg.v1"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/informers"
corelistersv1 "k8s.io/client-go/listers/core/v1"
certutil "k8s.io/client-go/util/cert"
cloudprovider "k8s.io/cloud-provider"
nodehelpers "k8s.io/cloud-provider/node/helpers"
Expand Down Expand Up @@ -145,8 +144,15 @@ type OpenStack struct {
metadataOpts MetadataOpts
// InstanceID of the server where this OpenStack object is instantiated.
localInstanceID string
// Options for reading config data from a secret
secretName string
secretNamespace string
// SecretLister generated by SetInformers
secretLister corelistersv1.SecretLister
}

var _ cloudprovider.InformerUser = &OpenStack{}

// Config is used to read and store information from the cloud configuration file
// NOTE: Cloud config files should follow the same Kubernetes deprecation policy as
// flags or CLIs. Config fields should not change behavior in incompatible ways and
Expand All @@ -168,7 +174,8 @@ type Config struct {
CAFile string `gcfg:"ca-file"`
SecretName string `gcfg:"secret-name"`
SecretNamespace string `gcfg:"secret-namespace"`
KubeconfigPath string `gcfg:"kubeconfig-path"`
// KubeconfigPath option is deprecated and shouldn't be used
KubeconfigPath string `gcfg:"kubeconfig-path"`
}
LoadBalancer LoadBalancerOpts
BlockStorage BlockStorageOpts
Expand Down Expand Up @@ -261,59 +268,59 @@ func configFromEnv() (cfg Config, ok bool) {
return
}

func createKubernetesClient(kubeconfigPath string) (*kubernetes.Clientset, error) {
klog.Info("Creating kubernetes API client.")

cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
if err != nil {
return nil, err
}
cfg.DisableCompression = true

client, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, err
}

v, err := client.Discovery().ServerVersion()
if err != nil {
return nil, err
}

klog.Infof("Kubernetes API client created, server version %s", fmt.Sprintf("v%v.%v", v.Major, v.Minor))
return client, nil
// SetInformers initializes secret lister
func (os *OpenStack) SetInformers(informerFactory informers.SharedInformerFactory) {
os.secretLister = informerFactory.Core().V1().Secrets().Lister()
}

// setConfigFromSecret allows setting up the config from k8s secret
func setConfigFromSecret(cfg *Config) error {
secretName := cfg.Global.SecretName
secretNamespace := cfg.Global.SecretNamespace
kubeconfigPath := cfg.Global.KubeconfigPath

k8sClient, err := createKubernetesClient(kubeconfigPath)
if err != nil {
return fmt.Errorf("failed to get kubernetes client: %v", err)
func (os *OpenStack) setConfigFromSecret() error {
if os.secretLister == nil {
return fmt.Errorf("secret lister is not initialized")
}

secret, err := k8sClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
secret, err := os.secretLister.Secrets(os.secretNamespace).Get(os.secretName)
if err != nil {
klog.Warningf("Cannot get secret %s in namespace %s. error: %q", secretName, secretNamespace, err)
klog.Errorf("cannot get secret %s in namespace %s. error: %q", os.secretName, os.secretNamespace, err)
return err
}

if content, ok := secret.Data["clouds.conf"]; ok {
cfg := &Config{}

err = gcfg.ReadStringInto(cfg, string(content))
if err != nil {
klog.Error("Cannot parse data from the secret.")
return fmt.Errorf("cannot parse data from the secret")
return fmt.Errorf("cannot parse data from the secret: %s", err)
}
provider, err := newProvider(*cfg)
if err != nil {
return fmt.Errorf("cannot initialize cloud provider using data from the secret: %s", err)
}
os.provider = provider
os.region = cfg.Global.Region
klog.Info("OpenStack cloud provider was initialized using data from the secret.")
return nil
}

klog.Error("Cannot find \"clouds.conf\" key in the secret.")
return fmt.Errorf("cannot find \"clouds.conf\" key in the secret")
}

func (os *OpenStack) ensureCloudProviderWasInitialized() error {
if os.provider != nil {
return nil
}

if os.secretName != "" && os.secretNamespace != "" {
err := os.setConfigFromSecret()
if err != nil {
return fmt.Errorf("cloud provider is not initialized: %s", err)
}
return nil
}

return fmt.Errorf("cloud provider is not initialized")
}

func readConfig(config io.Reader) (Config, error) {
if config == nil {
return Config{}, fmt.Errorf("no OpenStack cloud provider config file given")
Expand All @@ -332,14 +339,6 @@ func readConfig(config io.Reader) (Config, error) {
return cfg, err
}

if cfg.Global.SecretName != "" && cfg.Global.SecretNamespace != "" {
klog.Infof("Set credentials from secret %s in namespace %s", cfg.Global.SecretName, cfg.Global.SecretNamespace)
err = setConfigFromSecret(&cfg)
if err != nil {
return cfg, err
}
}

return cfg, nil
}

Expand Down Expand Up @@ -398,7 +397,7 @@ func checkOpenStackOpts(openstackOpts *OpenStack) error {
return checkMetadataSearchOrder(openstackOpts.metadataOpts.SearchOrder)
}

func newOpenStack(cfg Config) (*OpenStack, error) {
func newProvider(cfg Config) (*gophercloud.ProviderClient, error) {
provider, err := openstack.NewClient(cfg.Global.AuthURL)
if err != nil {
return nil, err
Expand Down Expand Up @@ -428,23 +427,37 @@ func newOpenStack(cfg Config) (*OpenStack, error) {
return nil, err
}

provider.HTTPClient.Timeout = cfg.Metadata.RequestTimeout.Duration

return provider, nil
}

func newOpenStack(cfg Config) (*OpenStack, error) {
emptyDuration := MyDuration{}
if cfg.Metadata.RequestTimeout == emptyDuration {
cfg.Metadata.RequestTimeout.Duration = time.Duration(defaultTimeOut)
}
provider.HTTPClient.Timeout = cfg.Metadata.RequestTimeout.Duration

os := OpenStack{
provider: provider,
region: cfg.Global.Region,
lbOpts: cfg.LoadBalancer,
bsOpts: cfg.BlockStorage,
routeOpts: cfg.Route,
metadataOpts: cfg.Metadata,
secretName: cfg.Global.SecretName,
secretNamespace: cfg.Global.SecretNamespace,
region: cfg.Global.Region,
lbOpts: cfg.LoadBalancer,
bsOpts: cfg.BlockStorage,
routeOpts: cfg.Route,
metadataOpts: cfg.Metadata,
}

// Skip provider initialization if we're going to read auth data from a secret.
if cfg.Global.SecretName == "" || cfg.Global.SecretNamespace == "" {
provider, err := newProvider(cfg)
if err != nil {
return nil, err
}
os.provider = provider
}

err = checkOpenStackOpts(&os)
if err != nil {
if err := checkOpenStackOpts(&os); err != nil {
return nil, err
}

Expand Down Expand Up @@ -479,6 +492,7 @@ func NewFakeOpenStackCloud(cfg Config) (*OpenStack, error) {

// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
func (os *OpenStack) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) {
os.ensureCloudProviderWasInitialized()
}

// mapNodeNameToServerName maps a k8s NodeName to an OpenStack Server Name
Expand Down Expand Up @@ -697,6 +711,11 @@ func (os *OpenStack) HasClusterID() bool {
func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
klog.V(4).Info("openstack.LoadBalancer() called")

err := os.ensureCloudProviderWasInitialized()
if err != nil {
return nil, false
}

if reflect.DeepEqual(os.lbOpts, LoadBalancerOpts{}) {
klog.V(4).Info("LoadBalancer section is empty/not defined in cloud-config")
return nil, false
Expand Down Expand Up @@ -821,6 +840,11 @@ func (os *OpenStack) GetZoneByNodeName(ctx context.Context, nodeName types.NodeN
func (os *OpenStack) Routes() (cloudprovider.Routes, bool) {
klog.V(4).Info("openstack.Routes() called")

err := os.ensureCloudProviderWasInitialized()
if err != nil {
return nil, false
}

network, err := os.NewNetworkV2()
if err != nil {
return nil, false
Expand Down Expand Up @@ -853,6 +877,10 @@ func (os *OpenStack) Routes() (cloudprovider.Routes, bool) {
}

func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) {
if err := os.ensureCloudProviderWasInitialized(); err != nil {
return nil, err
}

bsVersion := ""
if forceVersion == "" {
bsVersion = os.bsOpts.BSVersion
Expand Down
Loading

0 comments on commit 6b22017

Please sign in to comment.