From dc71e84f0704690b528e7f1c2b56cb4898374fbf Mon Sep 17 00:00:00 2001 From: James Milligan <75740990+james-milligan@users.noreply.github.com> Date: Thu, 2 Mar 2023 16:44:42 +0000 Subject: [PATCH 1/7] feat: add debug logging for merge behaviour (#456) ## This PR - adds logging describing merge events, specifically when overwrites / deletes do not take place ### Related Issues https://github.com/open-feature/flagd/issues/453 ### Notes ### Follow-up Tasks ### How to test --------- Signed-off-by: James Milligan --- pkg/store/flags.go | 56 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/store/flags.go b/pkg/store/flags.go index 91b5b0898..dc86dffc4 100644 --- a/pkg/store/flags.go +++ b/pkg/store/flags.go @@ -87,6 +87,14 @@ func (f *Flags) Add(logger *logger.Logger, source string, flags map[string]model for k, newFlag := range flags { storedFlag, ok := f.Get(k) if ok && !f.hasPriority(storedFlag.Source, source) { + logger.Debug( + fmt.Sprintf( + "not overwriting: flag %s from source %s does not have priority over %s", + k, + source, + storedFlag.Source, + ), + ) continue } @@ -118,6 +126,14 @@ func (f *Flags) Update(logger *logger.Logger, source string, flags map[string]mo continue } if !f.hasPriority(storedFlag.Source, source) { + logger.Debug( + fmt.Sprintf( + "not updating: flag %s from source %s does not have priority over %s", + k, + source, + storedFlag.Source, + ), + ) continue } @@ -135,6 +151,12 @@ func (f *Flags) Update(logger *logger.Logger, source string, flags map[string]mo // DeleteFlags matching flags from source. func (f *Flags) DeleteFlags(logger *logger.Logger, source string, flags map[string]model.Flag) map[string]interface{} { + logger.Debug( + fmt.Sprintf( + "store resync triggered: delete event from source %s", + source, + ), + ) notifications := map[string]interface{}{} if len(flags) == 0 { allFlags := f.GetAll() @@ -154,6 +176,14 @@ func (f *Flags) DeleteFlags(logger *logger.Logger, source string, flags map[stri flag, ok := f.Get(k) if ok { if !f.hasPriority(flag.Source, source) { + logger.Debug( + fmt.Sprintf( + "not deleting: flag %s from source %s cannot be deleted by %s", + k, + flag.Source, + source, + ), + ) continue } notifications[k] = map[string]interface{}{ @@ -181,7 +211,6 @@ func (f *Flags) Merge( ) (map[string]interface{}, bool) { notifications := map[string]interface{}{} resyncRequired := false - f.mx.Lock() for k, v := range f.Flags { if v.Source == source { @@ -193,18 +222,33 @@ func (f *Flags) Merge( "source": source, } resyncRequired = true + logger.Debug( + fmt.Sprintf( + "store resync triggered: flag %s has been deleted from source %s", + k, source, + ), + ) continue } } } f.mx.Unlock() - for k, newFlag := range flags { newFlag.Source = source - storedFlag, ok := f.Get(k) - if ok && (!f.hasPriority(storedFlag.Source, source) || reflect.DeepEqual(storedFlag, newFlag)) { - continue + if ok { + if !f.hasPriority(storedFlag.Source, source) { + logger.Debug( + fmt.Sprintf( + "not merging: flag %s from source %s does not have priority over %s", + k, source, storedFlag.Source, + ), + ) + continue + } + if reflect.DeepEqual(storedFlag, newFlag) { + continue + } } if !ok { notifications[k] = map[string]interface{}{ @@ -217,10 +261,8 @@ func (f *Flags) Merge( "source": source, } } - // Store the new version of the flag f.Set(k, newFlag) } - return notifications, resyncRequired } From 4c03bfc812e7ceabcac0979290bd74d9efc9da15 Mon Sep 17 00:00:00 2001 From: Kavindu Dodanduwa Date: Thu, 2 Mar 2023 08:57:24 -0800 Subject: [PATCH 2/7] feat: refactor and improve K8s sync provider (#443) ## This PR fixes #434 This is a refactoring and internal improvement for K8s ISync provider. Improvements include, - Reduce K8s API load by utilizing Informer cache - Yet, provide a fallback if cache miss occurs (Note - we rely on K8s Informer for cache refill and consistency) - Informer now only watches a specific namespace (compared to **\***) - this is a potential performance improvement and a security improvement - Reduced informer handlers with extracted common logics - Unit tests where possible --------- Signed-off-by: Kavindu Dodanduwa --- go.mod | 2 +- go.sum | 13 - pkg/sync/kubernetes/kubernetes_sync.go | 272 +++++---- pkg/sync/kubernetes/kubernetes_sync_test.go | 611 ++++++++++++++++++++ 4 files changed, 765 insertions(+), 133 deletions(-) create mode 100644 pkg/sync/kubernetes/kubernetes_sync_test.go diff --git a/go.mod b/go.mod index 36c65d80c..b8d301b1d 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,6 @@ require ( google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.26.2 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 sigs.k8s.io/controller-runtime v0.14.5 @@ -115,6 +114,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/api v0.26.2 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/component-base v0.26.1 // indirect k8s.io/klog/v2 v2.90.0 // indirect diff --git a/go.sum b/go.sum index dbcd2418f..3f25c0628 100644 --- a/go.sum +++ b/go.sum @@ -285,18 +285,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/open-feature/go-sdk v1.1.0 h1:JOOa0AleJFUvnWoF9KWdLqYosi5fDIRBDzPYZPr5qgM= -github.com/open-feature/go-sdk v1.1.0/go.mod h1:R8QJmLdSHFaRdrWtwmp5bVK35Q+O/cEGtYaiy6NM6kc= github.com/open-feature/go-sdk v1.2.0 h1:2xsUgNUUDITpryB9nFS43CI9gAF415I1He22Q1d4+Po= github.com/open-feature/go-sdk v1.2.0/go.mod h1:UQJJXUptk92An4F6so2Vd0iRo6EEZ+QGa7HVyQ/GPi0= -github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.7 h1:0s8reX/EfCNV37PsGSr55wUpppPtyp0jZKeuVAaWZ+4= -github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.7/go.mod h1:dHB0hsYykZ1Un+CdnWErqLqUQswUADIvDg2VwDLx7gs= github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.9 h1:hHa7sjOzohj9ZhYR6ym+Xjk517ogb4q2QIE6ztdLZMg= github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.9/go.mod h1:IibpAPNmtUIJsJA6T4X1IcD4+BG1hCLw86luG8YQcqA= github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.1 h1:Tg712Egcqb5dsYxOGEaQbfD3g1mqPFdV4tSmKKKxDPk= github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.1/go.mod h1:zw/xpuDy9ziBEKVA1t4VoQtzFc80btAAQCiZkX6y9oQ= -github.com/open-feature/open-feature-operator v0.2.28 h1:qzzVq8v9G7aXO7luocO/wQCGnTJjtcQh75mDOqjnFxo= -github.com/open-feature/open-feature-operator v0.2.28/go.mod h1:bQncVK7hvhj5QStPwexxQ1aArPwox2Y1vWrVei/qIFg= github.com/open-feature/open-feature-operator v0.2.29 h1:Ky/SMzwEiBV5x9qOfHTj1jl/CakPZNClRtoeSPqVbNo= github.com/open-feature/open-feature-operator v0.2.29/go.mod h1:bQncVK7hvhj5QStPwexxQ1aArPwox2Y1vWrVei/qIFg= github.com/open-feature/schemas v0.2.8 h1:oA75hJXpOd9SFgmNI2IAxWZkwzQPUDm7Jyyh3q489wM= @@ -359,7 +353,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -756,18 +749,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= -k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ= k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ= k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= -k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI= k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index c06195bd2..6d2378418 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -2,16 +2,15 @@ package kubernetes import ( "context" - "errors" "fmt" "os" "strings" + msync "sync" "time" "github.com/open-feature/flagd/pkg/logger" "github.com/open-feature/flagd/pkg/sync" "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" @@ -23,15 +22,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var resyncPeriod = 1 * time.Minute +var ( + resyncPeriod = 1 * time.Minute + apiVersion = fmt.Sprintf("%s/%s", v1alpha1.GroupVersion.Group, v1alpha1.GroupVersion.Version) +) type Sync struct { Logger *logger.Logger ProviderArgs sync.ProviderArgs - client client.Client URI string - Source string - ready bool + + Source string + ready bool + namespace string + crdName string + readClient client.Reader + informer cache.SharedInformer } func (k *Sync) ReSync(ctx context.Context, dataSync chan<- sync.DataSync) error { @@ -44,16 +50,49 @@ func (k *Sync) ReSync(ctx context.Context, dataSync chan<- sync.DataSync) error } func (k *Sync) Init(ctx context.Context) error { - // noop + var err error + + k.namespace, k.crdName, err = parseURI(k.URI) + if err != nil { + return err + } + + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + clusterConfig, err := k8sClusterConfig() + if err != nil { + return err + } + + k.readClient, err = client.New(clusterConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return err + } + + dynamicClient, err := dynamic.NewForConfig(clusterConfig) + if err != nil { + return err + } + + resource := v1alpha1.GroupVersion.WithResource("featureflagconfigurations") + + // The created informer will not do resyncs if the given defaultEventHandlerResyncPeriod is zero. + // For more details on resync implications refer to tools/cache/shared_informer.go + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncPeriod, k.namespace, nil) + + k.informer = factory.ForResource(resource).Informer() + return nil } func (k *Sync) IsReady() bool { - // we cannot reliably check external HTTP(s) sources return k.ready } func (k *Sync) Sync(ctx context.Context, dataSync chan<- sync.DataSync) error { + k.Logger.Info(fmt.Sprintf("starting kubernetes sync notifier for resource: %s", k.URI)) + // Initial fetch fetch, err := k.fetch(ctx) if err != nil { @@ -65,12 +104,31 @@ func (k *Sync) Sync(ctx context.Context, dataSync chan<- sync.DataSync) error { notifies := make(chan INotify) - go k.notify(ctx, notifies) + var wg msync.WaitGroup + + // Start K8s resource notifier + wg.Add(1) + go func() { + defer wg.Done() + k.notify(ctx, notifies) + }() + + // Start notifier watcher + wg.Add(1) + go func() { + defer wg.Done() + k.watcher(ctx, notifies, dataSync) + }() + + wg.Wait() + return nil +} +func (k *Sync) watcher(ctx context.Context, notifies chan INotify, dataSync chan<- sync.DataSync) { for { select { case <-ctx.Done(): - return nil + return case w := <-notifies: switch w.GetEvent().EventType { case DefaultEventTypeCreate: @@ -86,7 +144,7 @@ func (k *Sync) Sync(ctx context.Context, dataSync chan<- sync.DataSync) error { k.Logger.Debug("Configuration modified") msg, err := k.fetch(ctx) if err != nil { - k.Logger.Error(fmt.Sprintf("error fetching after write notification: %s", err.Error())) + k.Logger.Error(fmt.Sprintf("error fetching after update notification: %s", err.Error())) continue } @@ -101,101 +159,47 @@ func (k *Sync) Sync(ctx context.Context, dataSync chan<- sync.DataSync) error { } } +// fetch attempts to retrieve the latest feature flag configurations func (k *Sync) fetch(ctx context.Context) (string, error) { - if k.URI == "" { - k.Logger.Error("no target feature flag configuration set") - return "{}", nil - } - - ns, name, err := parseURI(k.URI) + // first check the store - avoid overloading API + item, exist, err := k.informer.GetStore().GetByKey(k.URI) if err != nil { - k.Logger.Error(err.Error()) - return "{}", nil + return "", err } - if k.client == nil { - k.Logger.Warn("client not initialised") - return "{}", nil + if exist { + configuration, err := toFFCfg(item) + if err != nil { + return "", err + } + + k.Logger.Debug(fmt.Sprintf("resource %s served from the informer cache", k.URI)) + return configuration.Spec.FeatureFlagSpec, nil } + // fallback to API access - this is an informer cache miss. Could happen at the startup where cache is not filled var ff v1alpha1.FeatureFlagConfiguration - err = k.client.Get(ctx, client.ObjectKey{ - Name: name, - Namespace: ns, + err = k.readClient.Get(ctx, client.ObjectKey{ + Name: k.crdName, + Namespace: k.namespace, }, &ff) - - return ff.Spec.FeatureFlagSpec, err -} - -func parseURI(uri string) (string, string, error) { - s := strings.Split(uri, "/") - if len(s) != 2 { - return "", "", fmt.Errorf("invalid uri received: %s", uri) - } - return s[0], s[1], nil -} - -func (k *Sync) buildConfiguration() (*rest.Config, error) { - kubeconfig := os.Getenv("KUBECONFIG") - var clusterConfig *rest.Config - var err error - if kubeconfig != "" { - clusterConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - } else { - clusterConfig, err = rest.InClusterConfig() - } if err != nil { - return nil, err + return "", err } - return clusterConfig, nil + k.Logger.Debug(fmt.Sprintf("resource %s served from API server", k.URI)) + return ff.Spec.FeatureFlagSpec, nil } -//nolint:funlen func (k *Sync) notify(ctx context.Context, c chan<- INotify) { - if k.URI == "" { - k.Logger.Error("No target feature flag configuration set") - return - } - ns, name, err := parseURI(k.URI) - if err != nil { - k.Logger.Error(err.Error()) - return - } - k.Logger.Info( - fmt.Sprintf("starting kubernetes sync notifier for resource: %s", - k.URI, - ), - ) - clusterConfig, err := k.buildConfiguration() - if err != nil { - k.Logger.Error(fmt.Sprintf("error building configuration: %s", err)) - } - if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { - k.Logger.Fatal(err.Error()) - } - k.client, err = client.New(clusterConfig, client.Options{Scheme: scheme.Scheme}) - if err != nil { - k.Logger.Fatal(err.Error()) - } - clusterClient, err := dynamic.NewForConfig(clusterConfig) - if err != nil { - k.Logger.Fatal(err.Error()) - } - resource := v1alpha1.GroupVersion.WithResource("featureflagconfigurations") - // The created informer will not do resyncs if the given defaultEventHandlerResyncPeriod is zero. - // For more details on resync implications refer to tools/cache/shared_informer.go - factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(clusterClient, - resyncPeriod, corev1.NamespaceAll, nil) - informer := factory.ForResource(resource).Informer() objectKey := client.ObjectKey{ - Name: name, - Namespace: ns, + Name: k.crdName, + Namespace: k.namespace, } - if _, err = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + if _, err := k.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { k.Logger.Info(fmt.Sprintf("kube sync notifier event: add: %s %s", objectKey.Namespace, objectKey.Name)) - if err := createFuncHandler(obj, objectKey, c); err != nil { + if err := commonHandler(obj, objectKey, DefaultEventTypeCreate, c); err != nil { k.Logger.Warn(err.Error()) } }, @@ -207,7 +211,7 @@ func (k *Sync) notify(ctx context.Context, c chan<- INotify) { }, DeleteFunc: func(obj interface{}) { k.Logger.Info(fmt.Sprintf("kube sync notifier event: delete: %s %s", objectKey.Namespace, objectKey.Name)) - if err := deleteFuncHandler(obj, objectKey, c); err != nil { + if err := commonHandler(obj, objectKey, DefaultEventTypeDelete, c); err != nil { k.Logger.Warn(err.Error()) } }, @@ -220,48 +224,52 @@ func (k *Sync) notify(ctx context.Context, c chan<- INotify) { EventType: DefaultEventTypeReady, }, } - informer.Run(ctx.Done()) + + k.informer.Run(ctx.Done()) } -func createFuncHandler(obj interface{}, object client.ObjectKey, c chan<- INotify) error { - var ffObj v1alpha1.FeatureFlagConfiguration - u := obj.(*unstructured.Unstructured) - err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &ffObj) +// commonHandler emits the desired event if and only if handler receive an object matching apiVersion and resource name +func commonHandler(obj interface{}, object client.ObjectKey, emitEvent DefaultEventType, c chan<- INotify) error { + ffObj, err := toFFCfg(obj) if err != nil { return err } - if ffObj.APIVersion != fmt.Sprintf("%s/%s", v1alpha1.GroupVersion.Group, v1alpha1.GroupVersion.Version) { - return errors.New("invalid api version") + + if ffObj.APIVersion != apiVersion { + return fmt.Errorf("invalid api version %s, expected %s", ffObj.APIVersion, apiVersion) } + if ffObj.Name == object.Name { c <- &Notifier{ Event: Event[DefaultEventType]{ - EventType: DefaultEventTypeCreate, + EventType: emitEvent, }, } } + return nil } +// updateFuncHandler handles updates. Event is emitted if and only if resource name, apiVersion of old & new are equal func updateFuncHandler(oldObj interface{}, newObj interface{}, object client.ObjectKey, c chan<- INotify) error { - var ffOldObj v1alpha1.FeatureFlagConfiguration - u := oldObj.(*unstructured.Unstructured) - err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &ffOldObj) + ffOldObj, err := toFFCfg(oldObj) if err != nil { return err } - if ffOldObj.APIVersion != fmt.Sprintf("%s/%s", v1alpha1.GroupVersion.Group, v1alpha1.GroupVersion.Version) { - return errors.New("invalid api version") + + if ffOldObj.APIVersion != apiVersion { + return fmt.Errorf("invalid api version %s, expected %s", ffOldObj.APIVersion, apiVersion) } - var ffNewObj v1alpha1.FeatureFlagConfiguration - u = newObj.(*unstructured.Unstructured) - err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &ffNewObj) + + ffNewObj, err := toFFCfg(newObj) if err != nil { return err } - if ffNewObj.APIVersion != fmt.Sprintf("%s/%s", v1alpha1.GroupVersion.Group, v1alpha1.GroupVersion.Version) { - return errors.New("invalid api version") + + if ffNewObj.APIVersion != apiVersion { + return fmt.Errorf("invalid api version %s, expected %s", ffNewObj.APIVersion, apiVersion) } + if object.Name == ffNewObj.Name && ffOldObj.ResourceVersion != ffNewObj.ResourceVersion { // Only update if there is an actual featureFlagSpec change c <- &Notifier{ @@ -273,22 +281,48 @@ func updateFuncHandler(oldObj interface{}, newObj interface{}, object client.Obj return nil } -func deleteFuncHandler(obj interface{}, object client.ObjectKey, c chan<- INotify) error { +// toFFCfg attempts to covert unstructured payload to configurations +func toFFCfg(object interface{}) (*v1alpha1.FeatureFlagConfiguration, error) { + u, ok := object.(*unstructured.Unstructured) + if !ok { + return nil, fmt.Errorf("provided value is not of type *unstructured.Unstructured") + } + var ffObj v1alpha1.FeatureFlagConfiguration - u := obj.(*unstructured.Unstructured) err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &ffObj) if err != nil { - return err + return nil, err } - if ffObj.APIVersion != fmt.Sprintf("%s/%s", v1alpha1.GroupVersion.Group, v1alpha1.GroupVersion.Version) { - return errors.New("invalid api version") + + return &ffObj, nil +} + +// parseURI parse provided uri in the format of / to namespace, crdName. Results in an error +// for invalid format or failed parsing +func parseURI(uri string) (string, string, error) { + s := strings.Split(uri, "/") + if len(s) != 2 || len(s[0]) == 0 || len(s[1]) == 0 { + return "", "", fmt.Errorf("invalid resource uri format, expected / but got: %s", uri) } - if ffObj.Name == object.Name { - c <- &Notifier{ - Event: Event[DefaultEventType]{ - EventType: DefaultEventTypeDelete, - }, - } + return s[0], s[1], nil +} + +// k8sClusterConfig build K8s connection config based available configurations +func k8sClusterConfig() (*rest.Config, error) { + cfg := os.Getenv("KUBECONFIG") + + var clusterConfig *rest.Config + var err error + + if cfg != "" { + clusterConfig, err = clientcmd.BuildConfigFromFlags("", cfg) + } else { + clusterConfig, err = rest.InClusterConfig() } - return nil + + if err != nil { + return nil, err + } + + return clusterConfig, nil } diff --git a/pkg/sync/kubernetes/kubernetes_sync_test.go b/pkg/sync/kubernetes/kubernetes_sync_test.go new file mode 100644 index 000000000..0f92013ac --- /dev/null +++ b/pkg/sync/kubernetes/kubernetes_sync_test.go @@ -0,0 +1,611 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + "time" + + "github.com/open-feature/flagd/pkg/sync" + + "github.com/open-feature/flagd/pkg/logger" + "k8s.io/client-go/tools/cache" + + "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + + "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var Metadata = v1.TypeMeta{ + Kind: "FeatureFlagConfiguration", + APIVersion: apiVersion, +} + +func Test_parseURI(t *testing.T) { + tests := []struct { + name string + uri string + ns string + resource string + err bool + }{ + { + name: "simple success", + uri: "namespace/resource", + ns: "namespace", + resource: "resource", + err: false, + }, + { + name: "simple error - no ns", + uri: "/resource", + err: true, + }, + { + name: "simple error - no resource", + uri: "resource/", + err: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns, rs, err := parseURI(tt.uri) + if (err != nil) != tt.err { + t.Errorf("parseURI() error = %v, wantErr %v", err, tt.err) + return + } + if ns != tt.ns { + t.Errorf("parseURI() got = %v, want %v", ns, tt.ns) + } + if rs != tt.resource { + t.Errorf("parseURI() got1 = %v, want %v", rs, tt.resource) + } + }) + } +} + +func Test_toFFCfg(t *testing.T) { + validFFCfg := v1alpha1.FeatureFlagConfiguration{ + TypeMeta: Metadata, + } + + tests := []struct { + name string + input interface{} + want *v1alpha1.FeatureFlagConfiguration + wantErr bool + }{ + { + name: "Simple success", + input: toUnstructured(t, validFFCfg), + want: &validFFCfg, + wantErr: false, + }, + { + name: "Simple error", + input: struct { + flag string + }{ + flag: "test", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := toFFCfg(tt.input) + + if (err != nil) != tt.wantErr { + t.Errorf("toFFCfg() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("toFFCfg() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_commonHandler(t *testing.T) { + cfgNs := "resourceNS" + cfgName := "resourceName" + + validFFCfg := v1alpha1.FeatureFlagConfiguration{ + TypeMeta: Metadata, + ObjectMeta: v1.ObjectMeta{ + Namespace: cfgNs, + Name: cfgName, + }, + } + + type args struct { + obj interface{} + object client.ObjectKey + } + tests := []struct { + name string + args args + wantErr bool + wantEvent bool + eventType DefaultEventType + }{ + { + name: "simple success", + args: args{ + obj: toUnstructured(t, validFFCfg), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: cfgName, + }, + }, + wantEvent: true, + wantErr: false, + }, + { + name: "simple scenario - only notify if resource name matches", + args: args{ + obj: toUnstructured(t, validFFCfg), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: "SomeOtherResource", + }, + }, + wantEvent: false, + wantErr: false, + }, + { + name: "simple error - API mismatch", + args: args{ + obj: toUnstructured(t, v1alpha1.FeatureFlagConfiguration{ + TypeMeta: v1.TypeMeta{ + Kind: "FeatureFlagConfiguration", + APIVersion: "someAPIVersion", + }, + }), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: cfgName, + }, + }, + wantErr: true, + wantEvent: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + syncChan := make(chan INotify, 1) + + err := commonHandler(tt.args.obj, tt.args.object, tt.eventType, syncChan) + if err != nil && !tt.wantErr { + t.Errorf("commonHandler() error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.wantErr { + if err == nil { + t.Errorf("commonHandler() expected error but received none.") + } + + // Expected error occurred, hence continue + return + } + + if tt.wantEvent != true { + // Not interested in the event, hence ignore notification check. But check for chan writes + if len(syncChan) != 0 { + t.Errorf("commonHandler() expected no events, but events are available: %d", len(syncChan)) + } + + return + } + + // watch events with a timeout + var notify INotify + select { + case notify = <-syncChan: + case <-time.After(2 * time.Second): + t.Errorf("timedout waiting for events from commonHandler()") + } + + if notify.GetEvent().EventType != tt.eventType { + t.Errorf("commonHandler() event = %v, wanted %v", notify.GetEvent().EventType, DefaultEventTypeDelete) + } + }) + } +} + +func Test_updateFuncHandler(t *testing.T) { + cfgNs := "resourceNS" + cfgName := "resourceName" + + validFFCfgOld := v1alpha1.FeatureFlagConfiguration{ + TypeMeta: Metadata, + ObjectMeta: v1.ObjectMeta{ + Namespace: cfgNs, + Name: cfgName, + ResourceVersion: "v1", + }, + } + + validFFCfgNew := validFFCfgOld + validFFCfgNew.ResourceVersion = "v2" + + type args struct { + oldObj interface{} + newObj interface{} + object client.ObjectKey + } + tests := []struct { + name string + args args + wantErr bool + wantEvent bool + }{ + { + name: "Simple success", + args: args{ + oldObj: toUnstructured(t, validFFCfgOld), + newObj: toUnstructured(t, validFFCfgNew), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: cfgName, + }, + }, + wantErr: false, + wantEvent: true, + }, + { + name: "Simple scenario - notify only if resource name match", + args: args{ + oldObj: toUnstructured(t, validFFCfgOld), + newObj: toUnstructured(t, validFFCfgNew), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: "SomeOtherResource", + }, + }, + wantErr: false, + wantEvent: false, + }, + { + name: "Simple scenario - notify only if resource version is new", + args: args{ + oldObj: toUnstructured(t, validFFCfgOld), + newObj: toUnstructured(t, validFFCfgOld), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: "SomeOtherResource", + }, + }, + wantErr: false, + wantEvent: false, + }, + { + name: "Simple error - API version mismatch new object", + args: args{ + oldObj: toUnstructured(t, validFFCfgOld), + newObj: toUnstructured(t, v1alpha1.FeatureFlagConfiguration{ + TypeMeta: v1.TypeMeta{ + Kind: "FeatureFlagConfiguration", + APIVersion: "someAPIVersion", + }, + }), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: cfgName, + }, + }, + wantErr: true, + wantEvent: false, + }, + { + name: "Simple error - API version mismatch old object", + args: args{ + oldObj: toUnstructured(t, v1alpha1.FeatureFlagConfiguration{ + TypeMeta: v1.TypeMeta{ + Kind: "FeatureFlagConfiguration", + APIVersion: "someAPIVersion", + }, + }), + newObj: toUnstructured(t, validFFCfgNew), + object: client.ObjectKey{ + Namespace: cfgNs, + Name: cfgName, + }, + }, + wantErr: true, + wantEvent: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + syncChan := make(chan INotify, 1) + + err := updateFuncHandler(tt.args.oldObj, tt.args.newObj, tt.args.object, syncChan) + if err != nil && !tt.wantErr { + t.Errorf("updateFuncHandler() error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.wantErr { + if err == nil { + t.Errorf("updateFuncHandler() expected error but received none.") + } + + // Expected error occurred, hence continue + return + } + + if tt.wantEvent != true { + // Not interested in the event, hence ignore notification check. But check for chan writes + if len(syncChan) != 0 { + t.Errorf("updateFuncHandler() expected no events, but events are available: %d", len(syncChan)) + } + + return + } + + // watch events with a timeout + var notify INotify + select { + case notify = <-syncChan: + case <-time.After(2 * time.Second): + t.Errorf("timedout waiting for events from updateFuncHandler()") + } + + if notify.GetEvent().EventType != DefaultEventTypeModify { + t.Errorf("updateFuncHandler() event = %v, wanted %v", notify.GetEvent().EventType, DefaultEventTypeModify) + } + }) + } +} + +func TestSync_fetch(t *testing.T) { + flagSpec := "fakeFlagSpec" + + validCfg := v1alpha1.FeatureFlagConfiguration{ + TypeMeta: Metadata, + ObjectMeta: v1.ObjectMeta{ + Namespace: "resourceNS", + Name: "resourceName", + ResourceVersion: "v1", + }, + Spec: v1alpha1.FeatureFlagConfigurationSpec{ + FeatureFlagSpec: flagSpec, + }, + } + + type args struct { + InformerGetFunc func(key string) (item interface{}, exists bool, err error) + ClientResponse v1alpha1.FeatureFlagConfiguration + ClientError error + } + + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Scenario - get from informer cache", + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return toUnstructured(t, validCfg), true, nil + }, + }, + wantErr: false, + want: flagSpec, + }, + { + name: "Scenario - get from API if informer cache miss", + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return nil, false, nil + }, + ClientResponse: validCfg, + }, + wantErr: false, + want: flagSpec, + }, + { + name: "Scenario - error for informer cache read error", + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return nil, false, errors.New("mock error") + }, + }, + wantErr: true, + }, + { + name: "Scenario - error for API get error", + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return nil, false, nil + }, + ClientError: errors.New("mock error"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup with args + k := &Sync{ + informer: &MockInformer{ + fakeStore: cache.FakeCustomStore{ + GetByKeyFunc: tt.args.InformerGetFunc, + }, + }, + readClient: &MockClient{ + getResponse: tt.args.ClientResponse, + clientErr: tt.args.ClientError, + }, + Logger: logger.NewLogger(nil, false), + } + + // Test fetch + got, err := k.fetch(context.Background()) + + if (err != nil) != tt.wantErr { + t.Errorf("fetch() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if got != tt.want { + t.Errorf("fetch() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSync_watcher(t *testing.T) { + flagSpec := "fakeFlagSpec" + + validCfg := v1alpha1.FeatureFlagConfiguration{ + TypeMeta: Metadata, + ObjectMeta: v1.ObjectMeta{ + Namespace: "resourceNS", + Name: "resourceName", + ResourceVersion: "v1", + }, + Spec: v1alpha1.FeatureFlagConfigurationSpec{ + FeatureFlagSpec: flagSpec, + }, + } + + type args struct { + InformerGetFunc func(key string) (item interface{}, exists bool, err error) + notification INotify + } + + tests := []struct { + name string + args args + want string + }{ + { + name: "scenario - create event", + want: flagSpec, + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return toUnstructured(t, validCfg), true, nil + }, + notification: &Notifier{ + Event: Event[DefaultEventType]{ + EventType: DefaultEventTypeCreate, + }, + }, + }, + }, + { + name: "scenario - modify event", + want: flagSpec, + args: args{ + InformerGetFunc: func(key string) (item interface{}, exists bool, err error) { + return toUnstructured(t, validCfg), true, nil + }, + notification: &Notifier{ + Event: Event[DefaultEventType]{ + EventType: DefaultEventTypeModify, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup sync + k := &Sync{ + informer: &MockInformer{ + fakeStore: cache.FakeCustomStore{ + GetByKeyFunc: tt.args.InformerGetFunc, + }, + }, + Logger: logger.NewLogger(nil, false), + } + + // create communication channels with buffer to so that calls are non-blocking + notifies := make(chan INotify, 1) + dataSyncs := make(chan sync.DataSync, 1) + + // emit event + notifies <- tt.args.notification + + tCtx, cFunc := context.WithTimeout(context.Background(), 2*time.Second) + defer cFunc() + + // start watcher + go k.watcher(tCtx, notifies, dataSyncs) + + // wait for data sync + select { + case <-tCtx.Done(): + t.Errorf("timeout waiting for the results") + case dataSyncs := <-dataSyncs: + if dataSyncs.FlagData != tt.want { + t.Errorf("fetch() got = %v, want %v", dataSyncs.FlagData, tt.want) + } + } + }) + } +} + +// toUnstructured helper to convert an interface to unstructured.Unstructured +func toUnstructured(t *testing.T, obj interface{}) interface{} { + bytes, err := json.Marshal(obj) + if err != nil { + t.Errorf("test setup faulure: %s", err.Error()) + } + + var res map[string]interface{} + + err = json.Unmarshal(bytes, &res) + if err != nil { + t.Errorf("test setup faulure: %s", err.Error()) + } + + return &unstructured.Unstructured{Object: res} +} + +// Mock implementations + +// MockClient contains an embedded client.Reader for desired method overriding +type MockClient struct { + client.Reader + clientErr error + + getResponse v1alpha1.FeatureFlagConfiguration +} + +func (m MockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + // return error if error is set + if m.clientErr != nil { + return m.clientErr + } + + // else try returning response + cfg, ok := obj.(*v1alpha1.FeatureFlagConfiguration) + if !ok { + return errors.New("must contain a pointer typed v1alpha1.FeatureFlagConfiguration") + } + + *cfg = m.getResponse + return nil +} + +// MockInformer contains an embedded controllertest.FakeInformer for desired method overriding +type MockInformer struct { + controllertest.FakeInformer + + fakeStore cache.FakeCustomStore +} + +func (m MockInformer) GetStore() cache.Store { + return &m.fakeStore +} From 6cb66b14d01b6ee1c270bbdd3e30d4016757eae5 Mon Sep 17 00:00:00 2001 From: James Milligan <75740990+james-milligan@users.noreply.github.com> Date: Thu, 2 Mar 2023 19:47:01 +0000 Subject: [PATCH 3/7] docs: configuration merge docs (#455) ## This PR - adds documentation for merge events + resyncs Signed-off-by: James Milligan Co-authored-by: Skye Gill --- docs/README.md | 1 + .../flag_configuration_merging.md | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 docs/configuration/flag_configuration_merging.md diff --git a/docs/README.md b/docs/README.md index 4ba15e7cd..14baba580 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ Flagd is configured via CLI arguments on startup, these configuration options ca - [Flag configuration](./configuration/flag_configuration.md) - [Fractional evaluation](./configuration/fractional_evaluation.md) - [Reusable targeting rules](./configuration/reusable_targeting_rules.md) +- [Flag configuration merging](./configuration/flag_configuration_merging.md) ## Help diff --git a/docs/configuration/flag_configuration_merging.md b/docs/configuration/flag_configuration_merging.md new file mode 100644 index 000000000..fc0527605 --- /dev/null +++ b/docs/configuration/flag_configuration_merging.md @@ -0,0 +1,54 @@ +# Flag Configuration Merging + +Flagd can be configured to read from multiple sources at once, when this is the case flagd will merge all flag configurations into a single +merged state. For example: + +```mermaid +flowchart LR + source-A -->|config-A| store -->|merge|source-A-config-A\nsource-B-config-B + source-B -->|config-B| store +``` + +In this example, `source-A` and `source-B` provide a single flag configuration, `config-A` and `config-B` respectively. The merge logic for this configuration is simple, both flag configurations are added to the `store`. +In most scenarios, these flag sources will be supplying `n` number of configurations, using a unique flag key for each configuration. However, as multiple sources are being used, there is the opportunity for keys to be duplicated, intentionally or not, between flag sources. In these situations `flagd` uses a merge priority order to ensure that its behavior is consistent. + +Merge order is dictated by the order that `sync-providers` and `uris` are defined, with the latest defined source taking precedence over those defined before it, as an example: + +```sh +./flagd start --uri file:source-A.json --uri file:source-B.json --uri file:source-C.json +``` + +When `flagd` is started with the command defined above, `source-B` takes priority over `source-A`, whilst `source-C` takes priority over both `source-B` and `source-A`. Using the above example, if a flag key is duplicated across all 3 sources, then the configuration from `source-C` would be the only one stored in the merged state. + +```mermaid +flowchart LR + source-A -->|config-A| store -->source-C-config-A + source-B -->|config-A| store + source-C -->|config-A| store +``` + +## State Resync Events + +Given the above example, the `source-A` and `source-B` 'versions' of flag configuration `config-A` have been discarded, so if a delete event in `source-C` results in the removal of `config-A`, there will no longer be any reference of` config-A` in flagd's store. As a result of this flagd will return `FLAG_NOT_FOUND` errors, and the OpenFeature SDK will always return the default value. + +To prevent flagd falling out of sync with its flag sources during delete events, resync events are used. When a delete event results in a flag configuration being removed from the merged state, the full set of configurations is requested from all flag sources, and the merged state is rebuilt. As a result, the value of `config-A` from `source-B` will be stored in the merged state, preventing flagd from returning `FLAG_NOT_FOUND` errors. + +```mermaid +flowchart LR + source-A -->|config-A| store -->source-C-config-A + source-B -->|config-A| store + source-C -->|config-A| store + source-C -->|delete config-A|source-C-config-A + source-C-config-A --> resync-event +``` +In the example above, a delete event results in a resync event being fired, as `source-C` has deleted its 'version' of `config-A`, this results in a new merge state being formed from the remaining configurations. + +```mermaid +flowchart LR + source-A -->|config-A| store -->source-B-config-A + source-B -->|config-A| store + source-C -->store + +``` + +Resync events may lead to further resync events if the returned flag configurations result in further delete events, however the state will eventually be resolved correctly. \ No newline at end of file From 408bb7c6420678706854932df3144253ada89f82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 14:51:44 -0500 Subject: [PATCH 4/7] chore(main): release 0.4.0 (#412) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :robot: I have created a release *beep* *boop* --- ## [0.4.0](https://github.com/open-feature/flagd/compare/v0.3.7...v0.4.0) (2023-03-02) ### โš  BREAKING CHANGES * Use OTel to export metrics (metric name changes) ([#419](https://github.com/open-feature/flagd/issues/419)) ### ๐Ÿงน Chore * add additional sections to the release notes ([#449](https://github.com/open-feature/flagd/issues/449)) ([798f71a](https://github.com/open-feature/flagd/commit/798f71a92d2e2f450a53cda93b44217cbb2ad7fd)) * attach image sbom to release artefacts ([#407](https://github.com/open-feature/flagd/issues/407)) ([fb4ee50](https://github.com/open-feature/flagd/commit/fb4ee502217e2262849df09258f3a0ffa7edec13)) * **deps:** update actions/configure-pages digest to fc89b04 ([#417](https://github.com/open-feature/flagd/issues/417)) ([04014e7](https://github.com/open-feature/flagd/commit/04014e7cb37e43f5ed3726dfd31da96202abc043)) * **deps:** update amannn/action-semantic-pull-request digest to b6bca70 ([#441](https://github.com/open-feature/flagd/issues/441)) ([ce0ebe1](https://github.com/open-feature/flagd/commit/ce0ebe13dd992688a3a0464ff401f2c40651da52)) * **deps:** update docker/login-action digest to ec9cdf0 ([#437](https://github.com/open-feature/flagd/issues/437)) ([2650670](https://github.com/open-feature/flagd/commit/2650670d35166e119f9a92613d3aca81523b9faa)) * **deps:** update docker/metadata-action digest to 3343011 ([#438](https://github.com/open-feature/flagd/issues/438)) ([e7ebf32](https://github.com/open-feature/flagd/commit/e7ebf32caf0eae7449e673da0c10998f97ebf781)) * **deps:** update github/codeql-action digest to 32dc499 ([#439](https://github.com/open-feature/flagd/issues/439)) ([f91d91b](https://github.com/open-feature/flagd/commit/f91d91bf020d330f96572c5ee11a210c0c7f4311)) * **deps:** update google-github-actions/release-please-action digest to d3c71f9 ([#406](https://github.com/open-feature/flagd/issues/406)) ([6e1ffb2](https://github.com/open-feature/flagd/commit/6e1ffb27fea5e91014a6991b2afca9a59f89117f)) * disable caching tests in CI ([#442](https://github.com/open-feature/flagd/issues/442)) ([28a35f6](https://github.com/open-feature/flagd/commit/28a35f62d618539362ae83a48f11af08ca2ae245)) * fix race condition on init read ([#409](https://github.com/open-feature/flagd/issues/409)) ([0c9eb23](https://github.com/open-feature/flagd/commit/0c9eb2322df99b4216d40afd1cb3b8873b0c59ff)) * integration test stability ([#432](https://github.com/open-feature/flagd/issues/432)) ([5a6a5d5](https://github.com/open-feature/flagd/commit/5a6a5d5887badd846cffe882c8c22a35b850fa06)) * integration tests ([#312](https://github.com/open-feature/flagd/issues/312)) ([6192ac8](https://github.com/open-feature/flagd/commit/6192ac8820b0f472672ba177b7c5838244b6e277)) * reorder release note sections ([df7bfce](https://github.com/open-feature/flagd/commit/df7bfce85ec7d6abaa987f87341c5af380904b51)) * use -short flag in benchmark tests ([#431](https://github.com/open-feature/flagd/issues/431)) ([e68a6aa](https://github.com/open-feature/flagd/commit/e68a6aadb3dac46676299ab94a34a0bcc39a67af)) ### ๐Ÿ› Bug Fixes * **deps:** update kubernetes packages to v0.26.2 ([#450](https://github.com/open-feature/flagd/issues/450)) ([2885227](https://github.com/open-feature/flagd/commit/28852270f34ff81c072337b29aa17f4b6634e9cc)) * **deps:** update module github.com/bufbuild/connect-go to v1.5.2 ([#416](https://github.com/open-feature/flagd/issues/416)) ([feb7f04](https://github.com/open-feature/flagd/commit/feb7f047365263758a63d8dffea936f621a4966d)) * **deps:** update module github.com/open-feature/go-sdk-contrib/providers/flagd to v0.1.9 ([#427](https://github.com/open-feature/flagd/issues/427)) ([42d2705](https://github.com/open-feature/flagd/commit/42d270558bf9badcff9a9b352fda35491c45aebe)) * **deps:** update module github.com/open-feature/open-feature-operator to v0.2.29 ([#429](https://github.com/open-feature/flagd/issues/429)) ([b7fae81](https://github.com/open-feature/flagd/commit/b7fae81b89b3a1a0793a688c32569c4284633c6a)) * **deps:** update module github.com/stretchr/testify to v1.8.2 ([#440](https://github.com/open-feature/flagd/issues/440)) ([ab3e674](https://github.com/open-feature/flagd/commit/ab3e6748abc7843c022afeaf7cb11193cdcf59c5)) * **deps:** update module golang.org/x/net to v0.7.0 ([#410](https://github.com/open-feature/flagd/issues/410)) ([c6133b6](https://github.com/open-feature/flagd/commit/c6133b6af61f3d73ae73d318a1a9f44db2540540)) * **deps:** update module sigs.k8s.io/controller-runtime to v0.14.5 ([#454](https://github.com/open-feature/flagd/issues/454)) ([f907f11](https://github.com/open-feature/flagd/commit/f907f114f23fa2efa2637e254e829e4d53a90b51)) * remove non-error error log from parseFractionalEvaluationData ([#446](https://github.com/open-feature/flagd/issues/446)) ([34aca79](https://github.com/open-feature/flagd/commit/34aca79e6ec9876a6cced0fe49e1ceea34d83696)) ### โœจ New Features * add debug logging for merge behaviour ([#456](https://github.com/open-feature/flagd/issues/456)) ([dc71e84](https://github.com/open-feature/flagd/commit/dc71e84f0704690b528e7f1c2b56cb4898374fbf)) * add Health and Readiness probes ([#418](https://github.com/open-feature/flagd/issues/418)) ([7f2358c](https://github.com/open-feature/flagd/commit/7f2358ce207527c890f4a2f46ce4b9e8bf697095)) * Add version to startup message ([#430](https://github.com/open-feature/flagd/issues/430)) ([8daf613](https://github.com/open-feature/flagd/commit/8daf613e7e4f4492df0c06e2ef464f4337cadaca)) * introduce flag merge behaviour ([#414](https://github.com/open-feature/flagd/issues/414)) ([524f65e](https://github.com/open-feature/flagd/commit/524f65ea7215466bb4ac24a8d0d5953dd1cfe9a0)) * introduce grpc sync for flagd ([#297](https://github.com/open-feature/flagd/issues/297)) ([33413f2](https://github.com/open-feature/flagd/commit/33413f25882a3f1cf4953da0f18e746bfb69faf4)) * refactor and improve K8s sync provider ([#443](https://github.com/open-feature/flagd/issues/443)) ([4c03bfc](https://github.com/open-feature/flagd/commit/4c03bfc812e7ceabcac0979290bd74d9efc9da15)) * Use OTel to export metrics (metric name changes) ([#419](https://github.com/open-feature/flagd/issues/419)) ([eb3982a](https://github.com/open-feature/flagd/commit/eb3982a1cb72d664022b5cb126b533cf61497001)) ### ๐Ÿ“š Documentation * add .net flagd provider ([73d7840](https://github.com/open-feature/flagd/commit/73d7840c9fdef9c62371c677e02c0d9773c85f95)) * configuration merge docs ([#455](https://github.com/open-feature/flagd/issues/455)) ([6cb66b1](https://github.com/open-feature/flagd/commit/6cb66b14d01b6ee1c270bbdd3e30d4016757eae5)) * documentation for creating a provider ([#413](https://github.com/open-feature/flagd/issues/413)) ([d0c099d](https://github.com/open-feature/flagd/commit/d0c099d9aba3ed4d760a1858381f5e29b6d49a9c)) * updated filepaths for schema store regex ([#344](https://github.com/open-feature/flagd/issues/344)) ([2d0e9d9](https://github.com/open-feature/flagd/commit/2d0e9d956fbc99f2775821cfecdceb2b016d2b78)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 55 +++++++++++++++++++++++++++++++++++ snap/snapcraft.yaml | 2 +- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 457d91e0f..da59f99ea 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.3.7" + ".": "0.4.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 97452e303..3772b46a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,60 @@ # Changelog +## [0.4.0](https://github.com/open-feature/flagd/compare/v0.3.7...v0.4.0) (2023-03-02) + + +### โš  BREAKING CHANGES + +* Use OTel to export metrics (metric name changes) ([#419](https://github.com/open-feature/flagd/issues/419)) + +### ๐Ÿงน Chore + +* add additional sections to the release notes ([#449](https://github.com/open-feature/flagd/issues/449)) ([798f71a](https://github.com/open-feature/flagd/commit/798f71a92d2e2f450a53cda93b44217cbb2ad7fd)) +* attach image sbom to release artefacts ([#407](https://github.com/open-feature/flagd/issues/407)) ([fb4ee50](https://github.com/open-feature/flagd/commit/fb4ee502217e2262849df09258f3a0ffa7edec13)) +* **deps:** update actions/configure-pages digest to fc89b04 ([#417](https://github.com/open-feature/flagd/issues/417)) ([04014e7](https://github.com/open-feature/flagd/commit/04014e7cb37e43f5ed3726dfd31da96202abc043)) +* **deps:** update amannn/action-semantic-pull-request digest to b6bca70 ([#441](https://github.com/open-feature/flagd/issues/441)) ([ce0ebe1](https://github.com/open-feature/flagd/commit/ce0ebe13dd992688a3a0464ff401f2c40651da52)) +* **deps:** update docker/login-action digest to ec9cdf0 ([#437](https://github.com/open-feature/flagd/issues/437)) ([2650670](https://github.com/open-feature/flagd/commit/2650670d35166e119f9a92613d3aca81523b9faa)) +* **deps:** update docker/metadata-action digest to 3343011 ([#438](https://github.com/open-feature/flagd/issues/438)) ([e7ebf32](https://github.com/open-feature/flagd/commit/e7ebf32caf0eae7449e673da0c10998f97ebf781)) +* **deps:** update github/codeql-action digest to 32dc499 ([#439](https://github.com/open-feature/flagd/issues/439)) ([f91d91b](https://github.com/open-feature/flagd/commit/f91d91bf020d330f96572c5ee11a210c0c7f4311)) +* **deps:** update google-github-actions/release-please-action digest to d3c71f9 ([#406](https://github.com/open-feature/flagd/issues/406)) ([6e1ffb2](https://github.com/open-feature/flagd/commit/6e1ffb27fea5e91014a6991b2afca9a59f89117f)) +* disable caching tests in CI ([#442](https://github.com/open-feature/flagd/issues/442)) ([28a35f6](https://github.com/open-feature/flagd/commit/28a35f62d618539362ae83a48f11af08ca2ae245)) +* fix race condition on init read ([#409](https://github.com/open-feature/flagd/issues/409)) ([0c9eb23](https://github.com/open-feature/flagd/commit/0c9eb2322df99b4216d40afd1cb3b8873b0c59ff)) +* integration test stability ([#432](https://github.com/open-feature/flagd/issues/432)) ([5a6a5d5](https://github.com/open-feature/flagd/commit/5a6a5d5887badd846cffe882c8c22a35b850fa06)) +* integration tests ([#312](https://github.com/open-feature/flagd/issues/312)) ([6192ac8](https://github.com/open-feature/flagd/commit/6192ac8820b0f472672ba177b7c5838244b6e277)) +* reorder release note sections ([df7bfce](https://github.com/open-feature/flagd/commit/df7bfce85ec7d6abaa987f87341c5af380904b51)) +* use -short flag in benchmark tests ([#431](https://github.com/open-feature/flagd/issues/431)) ([e68a6aa](https://github.com/open-feature/flagd/commit/e68a6aadb3dac46676299ab94a34a0bcc39a67af)) + + +### ๐Ÿ› Bug Fixes + +* **deps:** update kubernetes packages to v0.26.2 ([#450](https://github.com/open-feature/flagd/issues/450)) ([2885227](https://github.com/open-feature/flagd/commit/28852270f34ff81c072337b29aa17f4b6634e9cc)) +* **deps:** update module github.com/bufbuild/connect-go to v1.5.2 ([#416](https://github.com/open-feature/flagd/issues/416)) ([feb7f04](https://github.com/open-feature/flagd/commit/feb7f047365263758a63d8dffea936f621a4966d)) +* **deps:** update module github.com/open-feature/go-sdk-contrib/providers/flagd to v0.1.9 ([#427](https://github.com/open-feature/flagd/issues/427)) ([42d2705](https://github.com/open-feature/flagd/commit/42d270558bf9badcff9a9b352fda35491c45aebe)) +* **deps:** update module github.com/open-feature/open-feature-operator to v0.2.29 ([#429](https://github.com/open-feature/flagd/issues/429)) ([b7fae81](https://github.com/open-feature/flagd/commit/b7fae81b89b3a1a0793a688c32569c4284633c6a)) +* **deps:** update module github.com/stretchr/testify to v1.8.2 ([#440](https://github.com/open-feature/flagd/issues/440)) ([ab3e674](https://github.com/open-feature/flagd/commit/ab3e6748abc7843c022afeaf7cb11193cdcf59c5)) +* **deps:** update module golang.org/x/net to v0.7.0 ([#410](https://github.com/open-feature/flagd/issues/410)) ([c6133b6](https://github.com/open-feature/flagd/commit/c6133b6af61f3d73ae73d318a1a9f44db2540540)) +* **deps:** update module sigs.k8s.io/controller-runtime to v0.14.5 ([#454](https://github.com/open-feature/flagd/issues/454)) ([f907f11](https://github.com/open-feature/flagd/commit/f907f114f23fa2efa2637e254e829e4d53a90b51)) +* remove non-error error log from parseFractionalEvaluationData ([#446](https://github.com/open-feature/flagd/issues/446)) ([34aca79](https://github.com/open-feature/flagd/commit/34aca79e6ec9876a6cced0fe49e1ceea34d83696)) + + +### โœจ New Features + +* add debug logging for merge behaviour ([#456](https://github.com/open-feature/flagd/issues/456)) ([dc71e84](https://github.com/open-feature/flagd/commit/dc71e84f0704690b528e7f1c2b56cb4898374fbf)) +* add Health and Readiness probes ([#418](https://github.com/open-feature/flagd/issues/418)) ([7f2358c](https://github.com/open-feature/flagd/commit/7f2358ce207527c890f4a2f46ce4b9e8bf697095)) +* Add version to startup message ([#430](https://github.com/open-feature/flagd/issues/430)) ([8daf613](https://github.com/open-feature/flagd/commit/8daf613e7e4f4492df0c06e2ef464f4337cadaca)) +* introduce flag merge behaviour ([#414](https://github.com/open-feature/flagd/issues/414)) ([524f65e](https://github.com/open-feature/flagd/commit/524f65ea7215466bb4ac24a8d0d5953dd1cfe9a0)) +* introduce grpc sync for flagd ([#297](https://github.com/open-feature/flagd/issues/297)) ([33413f2](https://github.com/open-feature/flagd/commit/33413f25882a3f1cf4953da0f18e746bfb69faf4)) +* refactor and improve K8s sync provider ([#443](https://github.com/open-feature/flagd/issues/443)) ([4c03bfc](https://github.com/open-feature/flagd/commit/4c03bfc812e7ceabcac0979290bd74d9efc9da15)) +* Use OTel to export metrics (metric name changes) ([#419](https://github.com/open-feature/flagd/issues/419)) ([eb3982a](https://github.com/open-feature/flagd/commit/eb3982a1cb72d664022b5cb126b533cf61497001)) + + +### ๐Ÿ“š Documentation + +* add .net flagd provider ([73d7840](https://github.com/open-feature/flagd/commit/73d7840c9fdef9c62371c677e02c0d9773c85f95)) +* configuration merge docs ([#455](https://github.com/open-feature/flagd/issues/455)) ([6cb66b1](https://github.com/open-feature/flagd/commit/6cb66b14d01b6ee1c270bbdd3e30d4016757eae5)) +* documentation for creating a provider ([#413](https://github.com/open-feature/flagd/issues/413)) ([d0c099d](https://github.com/open-feature/flagd/commit/d0c099d9aba3ed4d760a1858381f5e29b6d49a9c)) +* updated filepaths for schema store regex ([#344](https://github.com/open-feature/flagd/issues/344)) ([2d0e9d9](https://github.com/open-feature/flagd/commit/2d0e9d956fbc99f2775821cfecdceb2b016d2b78)) + ## [0.3.7](https://github.com/open-feature/flagd/compare/v0.3.6...v0.3.7) (2023-02-13) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b89c1bf08..2533b0077 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: flagd base: core20 -version: "v0.3.7" # x-release-please-version +version: "v0.4.0" # x-release-please-version summary: A feature flag daemon with a Unix philosophy description: | Flagd is a simple command line tool for fetching and evaluating feature flags for services. It is designed to conform with the OpenFeature specification. From 05bb51c7ab30f6e976b87f54ca889e978f834211 Mon Sep 17 00:00:00 2001 From: Kavindu Dodanduwa Date: Thu, 2 Mar 2023 14:31:30 -0800 Subject: [PATCH 5/7] fix: fix broken image signing (#461) ## This PR Fixes image signing issue and sign with digest --------- Signed-off-by: Kavindu Dodanduwa --- .github/workflows/release-please.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index a70a60723..d80b7a93d 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -62,6 +62,7 @@ jobs: run: echo "::set-output name=date::$(date +'%Y-%m-%d')" - name: Build + id: build uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4 with: builder: ${{ steps.buildx.outputs.name }} @@ -77,13 +78,20 @@ jobs: VERSION=${{ needs.release-please.outputs.release_tag_name }} COMMIT=${{ github.sha }} DATE=${{ steps.date.outputs.date }} + outputs: + image_digest: ${{ steps.build.outputs.digest }} + container-signing: + needs: container-release + runs-on: ubuntu-latest + if: ${{ needs.release-please.outputs.release_created }} + steps: - name: Install Cosign - uses: sigstore/cosign-installer@main + uses: sigstore/cosign-installer@c3667d99424e7e6047999fb6246c0da843953c65 - name: Sign the image run: | - cosign sign --key env://COSIGN_PRIVATE_KEY ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-please.outputs.release_tag_name }} + cosign sign --yes --key env://COSIGN_PRIVATE_KEY ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.container-release.outputs.image_digest }} cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile ${{ env.PUBLIC_KEY_FILE }} env: COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} From cbdf9b07c30239d7d04ef770cf4461fb33422fe9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 03:17:49 +0000 Subject: [PATCH 6/7] fix(deps): update module github.com/open-feature/go-sdk-contrib/providers/flagd to v0.1.10 (#459) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/open-feature/go-sdk-contrib/providers/flagd](https://togithub.com/open-feature/go-sdk-contrib) | require | patch | `v0.1.9` -> `v0.1.10` | --- ### Configuration ๐Ÿ“… **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). ๐Ÿšฆ **Automerge**: Enabled. โ™ป **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. ๐Ÿ”• **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://app.renovatebot.com/dashboard#github/open-feature/flagd). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b8d301b1d..13ad59421 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/golang/mock v1.6.0 github.com/mattn/go-colorable v0.1.13 - github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.9 + github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.10 github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.1 github.com/open-feature/open-feature-operator v0.2.29 github.com/open-feature/schemas v0.2.8 diff --git a/go.sum b/go.sum index 3f25c0628..bb6134958 100644 --- a/go.sum +++ b/go.sum @@ -289,6 +289,8 @@ github.com/open-feature/go-sdk v1.2.0 h1:2xsUgNUUDITpryB9nFS43CI9gAF415I1He22Q1d github.com/open-feature/go-sdk v1.2.0/go.mod h1:UQJJXUptk92An4F6so2Vd0iRo6EEZ+QGa7HVyQ/GPi0= github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.9 h1:hHa7sjOzohj9ZhYR6ym+Xjk517ogb4q2QIE6ztdLZMg= github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.9/go.mod h1:IibpAPNmtUIJsJA6T4X1IcD4+BG1hCLw86luG8YQcqA= +github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.10 h1:0QD8xsx35Ip6k7PcSdx/MEQq3ETEANKgRdz/pXjKtt4= +github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.10/go.mod h1:IibpAPNmtUIJsJA6T4X1IcD4+BG1hCLw86luG8YQcqA= github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.1 h1:Tg712Egcqb5dsYxOGEaQbfD3g1mqPFdV4tSmKKKxDPk= github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.1/go.mod h1:zw/xpuDy9ziBEKVA1t4VoQtzFc80btAAQCiZkX6y9oQ= github.com/open-feature/open-feature-operator v0.2.29 h1:Ky/SMzwEiBV5x9qOfHTj1jl/CakPZNClRtoeSPqVbNo= From b4ee495dc8e00b032518ea42d272a36b3b662e95 Mon Sep 17 00:00:00 2001 From: Kavindu Dodanduwa Date: Fri, 3 Mar 2023 06:38:03 -0800 Subject: [PATCH 7/7] fix: fixing image delimeter (#463) ## This PR Fixes image version delimiter to match digest usage [1] `:tag` vs `@digest` [1]. - https://docs.docker.com/engine/reference/run/#general-form --------- Signed-off-by: Kavindu Dodanduwa Co-authored-by: Todd Baert --- .github/workflows/release-please.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index d80b7a93d..6459a8bf3 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -91,7 +91,7 @@ jobs: - name: Sign the image run: | - cosign sign --yes --key env://COSIGN_PRIVATE_KEY ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.container-release.outputs.image_digest }} + cosign sign --yes --key env://COSIGN_PRIVATE_KEY ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.container-release.outputs.image_digest }} cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile ${{ env.PUBLIC_KEY_FILE }} env: COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}}