Skip to content

Commit

Permalink
Add resourceCounts and incomplete state to BundleDeploymentStatus (#3203
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@aruiz14
aruiz14 authored Jan 16, 2025
1 parent bc7d1f1 commit 0348ad2
Show file tree
Hide file tree
Showing 5 changed files with 418 additions and 47 deletions.
41 changes: 41 additions & 0 deletions charts/fleet-crd/templates/crds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,6 +910,11 @@ spec:
nullable: true
type: string
type: object
incompleteState:
description: IncompleteState is true if there are more than 10 non-ready
or modified resources, meaning that the lists in those fields
have been truncated.
type: boolean
modifiedStatus:
items:
description: 'ModifiedStatus is used to report the status of a
Expand Down Expand Up @@ -998,6 +1003,42 @@ spec:
description: Release is the Helm release ID
nullable: true
type: string
resourceCounts:
description: ResourceCounts contains the number of resources in
each state.
properties:
desiredReady:
description: DesiredReady is the number of resources that should
be ready.
type: integer
missing:
description: Missing is the number of missing resources.
type: integer
modified:
description: Modified is the number of resources that have been
modified.
type: integer
notReady:
description: 'NotReady is the number of not ready resources.
Resources are not
ready if they do not match any other state.'
type: integer
orphaned:
description: Orphaned is the number of orphaned resources.
type: integer
ready:
description: Ready is the number of ready resources.
type: integer
unknown:
description: Unknown is the number of resources in an unknown
state.
type: integer
waitApplied:
description: WaitApplied is the number of resources that are
waiting to be applied.
type: integer
type: object
resources:
description: 'Resources lists the metadata of resources that were
deployed
Expand Down
152 changes: 105 additions & 47 deletions internal/cmd/agent/deployer/monitor/updatestatus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import (
"sort"
"strings"

"github.com/go-logr/logr"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -27,6 +25,9 @@ import (
fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
)

// limit the length of nonReady and modified resources
const resourcesDetailsMaxLength = 10

type Monitor struct {
client client.Client
desiredset *desiredset.Client
Expand Down Expand Up @@ -82,11 +83,12 @@ func ShouldUpdateStatus(bd *fleet.BundleDeployment) bool {

func (m *Monitor) UpdateStatus(ctx context.Context, bd *fleet.BundleDeployment, resources *helmdeployer.Resources) (fleet.BundleDeploymentStatus, error) {
logger := log.FromContext(ctx).WithName("update-status")
ctx = log.IntoContext(ctx, logger)

// updateFromResources mutates bd.Status, so copy it first
// updateFromPreviousDeployment mutates bd.Status, so copy it first
origStatus := *bd.Status.DeepCopy()
bd = bd.DeepCopy()
err := m.updateFromResources(ctx, logger, bd, resources)
err := m.updateFromPreviousDeployment(ctx, bd, resources)
if err != nil {

// Returning an error will cause UpdateStatus to requeue in a loop.
Expand Down Expand Up @@ -145,9 +147,9 @@ func readyError(status fleet.BundleDeploymentStatus) error {
return errors.New(msg)
}

// updateFromResources updates the status with information from the
// updateFromPreviousDeployment updates the status with information from the
// helm release history and an apply dry run.
func (m *Monitor) updateFromResources(ctx context.Context, logger logr.Logger, bd *fleet.BundleDeployment, resources *helmdeployer.Resources) error {
func (m *Monitor) updateFromPreviousDeployment(ctx context.Context, bd *fleet.BundleDeployment, resources *helmdeployer.Resources) error {
resourcesPreviousRelease, err := m.deployer.ResourcesFromPreviousReleaseVersion(bd.Name, bd.Status.Release)
if err != nil {
return err
Expand All @@ -172,71 +174,139 @@ func (m *Monitor) updateFromResources(ctx context.Context, logger logr.Logger, b
return err
}

bd.Status.NonReadyStatus = nonReady(logger, plan, bd.Spec.Options.IgnoreOptions)
bd.Status.ModifiedStatus = modified(ctx, m.client, logger, plan, resourcesPreviousRelease)
bd.Status.Ready = false
bd.Status.NonModified = false

if len(bd.Status.NonReadyStatus) == 0 {
bd.Status.Ready = true
}
if len(bd.Status.ModifiedStatus) == 0 {
bd.Status.NonModified = true
nonReadyResources := nonReady(ctx, plan, bd.Spec.Options.IgnoreOptions)
modifiedResources := modified(ctx, m.client, plan, resourcesPreviousRelease)
allResources, err := toBundleDeploymentResources(m.client, plan.Objects, resources.DefaultNamespace)
if err != nil {
return err
}

bd.Status.Resources = []fleet.BundleDeploymentResource{}
for _, obj := range plan.Objects {
updateFromResources(&bd.Status, allResources, nonReadyResources, modifiedResources)
return nil
}

func toBundleDeploymentResources(client client.Client, objs []runtime.Object, defaultNamespace string) ([]fleet.BundleDeploymentResource, error) {
res := make([]fleet.BundleDeploymentResource, 0, len(objs))
for _, obj := range objs {
ma, err := meta.Accessor(obj)
if err != nil {
return err
return nil, err
}

ns := ma.GetNamespace()
gvk := obj.GetObjectKind().GroupVersionKind()
if ns == "" && isNamespaced(m.client.RESTMapper(), gvk) {
ns = resources.DefaultNamespace
if ns == "" && isNamespaced(client.RESTMapper(), gvk) {
ns = defaultNamespace
}

version, kind := gvk.ToAPIVersionAndKind()
bd.Status.Resources = append(bd.Status.Resources, fleet.BundleDeploymentResource{
res = append(res, fleet.BundleDeploymentResource{
Kind: kind,
APIVersion: version,
Namespace: ns,
Name: ma.GetName(),
CreatedAt: ma.GetCreationTimestamp(),
})
}
return res, nil
}

return nil
func updateFromResources(bdStatus *fleet.BundleDeploymentStatus, resources []fleet.BundleDeploymentResource, nonReadyResources []fleet.NonReadyStatus, modifiedResources []fleet.ModifiedStatus) {
bdStatus.Ready = len(nonReadyResources) == 0
bdStatus.NonReadyStatus = nonReadyResources
if len(bdStatus.NonReadyStatus) > resourcesDetailsMaxLength {
bdStatus.IncompleteState = true
bdStatus.NonReadyStatus = nonReadyResources[:resourcesDetailsMaxLength]
}

bdStatus.NonModified = len(modifiedResources) == 0
bdStatus.ModifiedStatus = modifiedResources
if len(bdStatus.ModifiedStatus) > resourcesDetailsMaxLength {
bdStatus.IncompleteState = true
bdStatus.ModifiedStatus = modifiedResources[:resourcesDetailsMaxLength]
}

bdStatus.Resources = resources
bdStatus.ResourceCounts = calculateResourceCounts(resources, nonReadyResources, modifiedResources)
}

func calculateResourceCounts(all []fleet.BundleDeploymentResource, nonReady []fleet.NonReadyStatus, modified []fleet.ModifiedStatus) fleet.ResourceCounts {
// Create a map with all different resource keys, then remove modified or non-ready keys
resourceKeys := make(map[fleet.ResourceKey]struct{}, len(all))
for _, r := range all {
resourceKeys[fleet.ResourceKey{
Kind: r.Kind,
APIVersion: r.APIVersion,
Namespace: r.Namespace,
Name: r.Name,
}] = struct{}{}
}

// The agent must have enough visibility to determine the exact state of every resource.
// e.g. "WaitApplied" or "Unknown" states do not make sense in this context
counts := fleet.ResourceCounts{
DesiredReady: len(all),
}
for _, r := range modified {
if r.Create {
counts.Missing++
} else if r.Delete {
counts.Orphaned++
} else {
counts.Modified++
}
delete(resourceKeys, fleet.ResourceKey{
Kind: r.Kind,
APIVersion: r.APIVersion,
Namespace: r.Namespace,
Name: r.Name,
})
}
for _, r := range nonReady {
key := fleet.ResourceKey{
Kind: r.Kind,
APIVersion: r.APIVersion,
Namespace: r.Namespace,
Name: r.Name,
}
// If not present, it was already accounted for as "modified"
if _, ok := resourceKeys[key]; ok {
counts.NotReady++
delete(resourceKeys, key)
}
}

// Remaining keys are considered ready
counts.Ready = len(resourceKeys)

return counts
}

func nonReady(logger logr.Logger, plan desiredset.Plan, ignoreOptions fleet.IgnoreOptions) (result []fleet.NonReadyStatus) {
func nonReady(ctx context.Context, plan desiredset.Plan, ignoreOptions fleet.IgnoreOptions) (result []fleet.NonReadyStatus) {
logger := log.FromContext(ctx)
defer func() {
sort.Slice(result, func(i, j int) bool {
return result[i].UID < result[j].UID
})
}()

for _, obj := range plan.Objects {
if len(result) >= 10 {
return result
}
if u, ok := obj.(*unstructured.Unstructured); ok {
if ignoreOptions.Conditions != nil {
if err := excludeIgnoredConditions(u, ignoreOptions); err != nil {
logger.Error(err, "failed to ignore conditions")
}
}

summary := summary.Summarize(u)
if !summary.IsReady() {
sum := summary.Summarize(u)
if !sum.IsReady() {
result = append(result, fleet.NonReadyStatus{
UID: u.GetUID(),
Kind: u.GetKind(),
APIVersion: u.GetAPIVersion(),
Namespace: u.GetNamespace(),
Name: u.GetName(),
Summary: summary,
Summary: sum,
})
}
}
Expand All @@ -249,20 +319,16 @@ func nonReady(logger logr.Logger, plan desiredset.Plan, ignoreOptions fleet.Igno
// The function iterates through the plan's create, delete, and update actions and constructs a modified status
// for each resource.
// If the number of modified statuses exceeds 10, the function stops and returns the current result.
func modified(ctx context.Context, c client.Client, logger logr.Logger, plan desiredset.Plan, resourcesPreviousRelease *helmdeployer.Resources) (result []fleet.ModifiedStatus) {
func modified(ctx context.Context, c client.Client, plan desiredset.Plan, resourcesPreviousRelease *helmdeployer.Resources) (result []fleet.ModifiedStatus) {
logger := log.FromContext(ctx)
defer func() {
sort.Slice(result, func(i, j int) bool {
return sortKey(result[i]) < sortKey(result[j])
})
}()
for gvk, keys := range plan.Create {
apiVersion, kind := gvk.ToAPIVersionAndKind()
for _, key := range keys {
if len(result) >= 10 {
return result
}

apiVersion, kind := gvk.ToAPIVersionAndKind()

obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(gvk)
key := client.ObjectKey{
Expand Down Expand Up @@ -296,12 +362,8 @@ func modified(ctx context.Context, c client.Client, logger logr.Logger, plan des
}

for gvk, keys := range plan.Delete {
apiVersion, kind := gvk.ToAPIVersionAndKind()
for _, key := range keys {
if len(result) >= 10 {
return result
}

apiVersion, kind := gvk.ToAPIVersionAndKind()
// Check if resource was in a previous release. It is possible that some operators copy the
// objectset.rio.cattle.io/hash label into a dynamically created objects. We need to skip these resources
// because they are not part of the release, and they would appear as orphaned.
Expand All @@ -319,12 +381,8 @@ func modified(ctx context.Context, c client.Client, logger logr.Logger, plan des
}

for gvk, patches := range plan.Update {
apiVersion, kind := gvk.ToAPIVersionAndKind()
for key, patch := range patches {
if len(result) >= 10 {
break
}

apiVersion, kind := gvk.ToAPIVersionAndKind()
result = append(result, fleet.ModifiedStatus{
Kind: kind,
APIVersion: apiVersion,
Expand Down
Loading

0 comments on commit 0348ad2

Please sign in to comment.