diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 3eac247a6a..305cfb3f0f 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -76,13 +76,6 @@ jobs: type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - name: Generate static deployment - run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag} - working-directory: ./tests - - name: Build binary uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 with: @@ -151,7 +144,6 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./tests - name: Run conformance tests diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 5e4eafff89..574cc274a3 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -31,7 +31,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name={{ .Values.nginxGateway.gatewayControllerName }} - --gatewayclass={{ .Values.nginxGateway.gatewayClassName }} - --config={{ include "nginx-gateway.config-name" . }} diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index 5652a9f2dd..efa990e29d 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -21,7 +21,6 @@ import ( ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/nginx/nginx-gateway-fabric/internal/framework/file" - "github.com/nginx/nginx-gateway-fabric/internal/mode/provisioner" "github.com/nginx/nginx-gateway-fabric/internal/mode/static" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/licensing" @@ -53,7 +52,7 @@ func createRootCommand() *cobra.Command { return rootCmd } -func createStaticModeCommand() *cobra.Command { +func createControllerCommand() *cobra.Command { // flag names const ( gatewayFlag = "gateway" @@ -145,8 +144,8 @@ func createStaticModeCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "static-mode", - Short: "Configure NGINX in the scope of a single Gateway resource", + Use: "controller", + Short: "Run the NGINX Gateway Fabric control plane", RunE: func(cmd *cobra.Command, _ []string) error { atom := zap.NewAtomicLevel() @@ -155,7 +154,7 @@ func createStaticModeCommand() *cobra.Command { commit, date, dirty := getBuildInfo() logger.Info( - "Starting NGINX Gateway Fabric in static mode", + "Starting the NGINX Gateway Fabric control plane", "version", version, "commit", commit, "date", date, @@ -443,56 +442,6 @@ func createStaticModeCommand() *cobra.Command { return cmd } -func createProvisionerModeCommand() *cobra.Command { - var ( - gatewayCtlrName = stringValidatingValue{ - validator: validateGatewayControllerName, - } - gatewayClassName = stringValidatingValue{ - validator: validateResourceName, - } - ) - - cmd := &cobra.Command{ - Use: "provisioner-mode", - Short: "Provision a static-mode NGINX Gateway Fabric Deployment per Gateway resource", - Hidden: true, - RunE: func(_ *cobra.Command, _ []string) error { - logger := ctlrZap.New() - commit, date, dirty := getBuildInfo() - logger.Info( - "Starting NGINX Gateway Fabric Provisioner", - "version", version, - "commit", commit, - "date", date, - "dirty", dirty, - ) - - return provisioner.StartManager(provisioner.Config{ - Logger: logger, - GatewayClassName: gatewayClassName.value, - GatewayCtlrName: gatewayCtlrName.value, - }) - }, - } - - cmd.Flags().Var( - &gatewayCtlrName, - gatewayCtlrNameFlag, - fmt.Sprintf(gatewayCtlrNameUsageFmt, domain), - ) - utilruntime.Must(cmd.MarkFlagRequired(gatewayCtlrNameFlag)) - - cmd.Flags().Var( - &gatewayClassName, - gatewayClassFlag, - gatewayClassNameUsage, - ) - utilruntime.Must(cmd.MarkFlagRequired(gatewayClassFlag)) - - return cmd -} - // FIXME(pleshakov): Remove this command once NGF min supported Kubernetes version supports sleep action in // preStop hook. // See https://github.com/kubernetes/enhancements/tree/4ec371d92dcd4f56a2ab18c8ba20bb85d8d20efe/keps/sig-node/3960-pod-lifecycle-sleep-action diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index e89a5a91dd..61459455f6 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -122,13 +122,9 @@ func TestCommonFlagsValidation(t *testing.T) { } for _, test := range tests { - t.Run(test.name+"_static_mode", func(t *testing.T) { + t.Run(test.name+"_controller", func(t *testing.T) { t.Parallel() - testFlag(t, createStaticModeCommand(), test) - }) - t.Run(test.name+"_provisioner_mode", func(t *testing.T) { - t.Parallel() - testFlag(t, createProvisionerModeCommand(), test) + testFlag(t, createControllerCommand(), test) }) } } @@ -439,28 +435,12 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - cmd := createStaticModeCommand() + cmd := createControllerCommand() testFlag(t, cmd, test) }) } } -func TestProvisionerModeCmdFlagValidation(t *testing.T) { - t.Parallel() - testCase := flagTestCase{ - name: "valid flags", - args: []string{ - "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", // common and required flag - "--gatewayclass=nginx", // common and required flag - }, - wantErr: false, - } - - // common flags validation is tested separately - - testFlag(t, createProvisionerModeCommand(), testCase) -} - func TestSleepCmdFlagValidation(t *testing.T) { t.Parallel() tests := []flagTestCase{ diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index fc2a5949c7..203385b732 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -21,8 +21,7 @@ func main() { rootCmd := createRootCommand() rootCmd.AddCommand( - createStaticModeCommand(), - createProvisionerModeCommand(), + createControllerCommand(), createInitializeCommand(), createSleepCommand(), ) diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml deleted file mode 100644 index 7c1b2df7e9..0000000000 --- a/config/tests/static-deployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- -# Source: nginx-gateway-fabric/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-gateway - namespace: nginx-gateway - labels: - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/version: "edge" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/instance: nginx-gateway - template: - metadata: - labels: - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/instance: nginx-gateway - spec: - containers: - - args: - - static-mode - - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - - --gatewayclass=nginx - - --config=nginx-gateway-config - - --service=nginx-gateway - - --metrics-disable - - --health-port=8081 - - --leader-election-lock-name=nginx-gateway-leader-election - - --product-telemetry-disable - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: INSTANCE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['app.kubernetes.io/instance'] - - name: IMAGE_NAME - value: ghcr.io/nginx/nginx-gateway-fabric:edge - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: nginx-gateway - ports: - - name: agent-grpc - containerPort: 8443 - - name: health - containerPort: 8081 - readinessProbe: - httpGet: - path: /readyz - port: health - initialDelaySeconds: 3 - periodSeconds: 1 - securityContext: - seccompProfile: - type: RuntimeDefault - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsUser: 101 - runAsGroup: 1001 - terminationGracePeriodSeconds: 30 - serviceAccountName: nginx-gateway - securityContext: - fsGroup: 1001 - runAsNonRoot: true diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index d49067708b..c6309d395e 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -200,7 +200,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index e7d0ef976d..88ce668193 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -200,7 +200,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 42cc36de4a..b73922cdae 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -200,7 +200,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 9ac24a81da..23d4223234 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -204,7 +204,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 15311817cc..7ae7821f26 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -204,7 +204,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 6d6c1ca848..b6b6b1ca58 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -200,7 +200,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index fa29623d9a..206401aac1 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -200,7 +200,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index e2701bf885..0a62309fcc 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -210,7 +210,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 88b9371440..a25c7e1aa3 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -202,7 +202,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index dfe78332b5..288058bac2 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -202,7 +202,7 @@ spec: spec: containers: - args: - - static-mode + - controller - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - --gatewayclass=nginx - --config=nginx-gateway-config diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index 993c994509..8267af8b9b 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -44,9 +44,8 @@ To create a new release, follow these steps: 1. Kick off the [longevity tests](https://github.com/nginx/nginx-gateway-fabric/blob/main/tests/README.md#longevity-testing) for both OSS and Plus. You'll need to create two clusters and VMs for this. Before running, update your `vars.env` file with the proper image tag and prefixes. NGF and nginx images will be available from `ghcr.io`, and nginx plus will be available in GCP (`us-docker.pkg.dev//nginx-gateway-fabric/nginx-plus`). These tests need to run for 4 days before releasing. The results should be committed to the main branch and then cherry-picked to the release branch. 2. Kick off the [NFR workflow](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/nfr.yml) in the browser. For `image_tag`, use `release-X.X-rc`, and for `version`, use the upcoming `X.Y.Z` NGF version. Run the workflow on the new release branch. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging. Once merged, be sure to cherry-pick the commit to the main branch as well (the original PR targets the release branch). 5. Run the [Release PR](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete: - 1. Update the version tag used in the [provisioner manifest](/tests/conformance/provisioner/provisioner.yaml) and [getting started guide](/site/content/get-started.md). - 2. Update the [README](/README.md) to include information about the release. - 3. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. + 1. Update the [README](/README.md) to include information about the release. + 2. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. - At the top there will be a list of all PRs that are labeled with `release-notes`. The changelog includes only important (from the user perspective) changes to NGF. This is in contrast with the autogenerated full changelog, which is created in the next diff --git a/docs/proposals/nginx-extensions.md b/docs/proposals/nginx-extensions.md index 497de1355f..6bf72fd300 100644 --- a/docs/proposals/nginx-extensions.md +++ b/docs/proposals/nginx-extensions.md @@ -155,7 +155,7 @@ spec: name: my-annotation ``` -Infrastructure labels and annotations should be applied to all resources created in response to the Gateway. This only applies to _automated deployments_ (i.e., provisioner mode), implementations that automatically deploy the data plane based on a Gateway. +Infrastructure labels and annotations should be applied to all resources created in response to the Gateway. Other use cases for this API are Service type, Service IP, CPU memory requests, affinity rules, and Gateway routability (public, private, and cluster). ### TLS Options diff --git a/embedded.go b/embedded.go deleted file mode 100644 index 0147f76f0e..0000000000 --- a/embedded.go +++ /dev/null @@ -1,11 +0,0 @@ -package embeddedfiles - -import _ "embed" - -// StaticModeDeploymentYAML contains the YAML manifest of the Deployment resource for the static mode. -// We put this in the root of the repo because goembed doesn't support relative/absolute paths and symlinks, -// and we want to keep the static mode deployment manifest for the provisioner in the config/tests/ -// directory. -// -//go:embed config/tests/static-deployment.yaml -var StaticModeDeploymentYAML []byte diff --git a/internal/mode/provisioner/deployment.go b/internal/mode/provisioner/deployment.go deleted file mode 100644 index 091052245e..0000000000 --- a/internal/mode/provisioner/deployment.go +++ /dev/null @@ -1,43 +0,0 @@ -package provisioner - -import ( - "fmt" - "strings" - - v1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/yaml" -) - -// prepareDeployment prepares a new the static mode Deployment based on the YAML manifest. -// It will use the specified id to set unique parts of the deployment, so it must be unique among all Deployments for -// Gateways. -// It will configure the Deployment to use the Gateway with the given NamespacedName. -func prepareDeployment(depYAML []byte, id string, gwNsName types.NamespacedName) (*v1.Deployment, error) { - dep := &v1.Deployment{} - if err := yaml.Unmarshal(depYAML, dep); err != nil { - return nil, fmt.Errorf("failed to unmarshal deployment: %w", err) - } - - dep.ObjectMeta.Name = id - dep.Spec.Selector.MatchLabels["app"] = id - dep.Spec.Template.ObjectMeta.Labels["app"] = id - - finalArgs := []string{ - "--gateway=" + gwNsName.String(), - "--update-gatewayclass-status=false", - } - - for _, arg := range dep.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(arg, "leader-election-lock-name") { - lockNameArg := "--leader-election-lock-name=" + gwNsName.Name - finalArgs = append(finalArgs, lockNameArg) - } else { - finalArgs = append(finalArgs, arg) - } - } - - dep.Spec.Template.Spec.Containers[0].Args = finalArgs - - return dep, nil -} diff --git a/internal/mode/provisioner/doc.go b/internal/mode/provisioner/doc.go deleted file mode 100644 index 589ab68527..0000000000 --- a/internal/mode/provisioner/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -/* -Package provisioner contains all the packages that relate to the provisioner-mode implementation of NGF. -Provisioner-mode implements data plane provisioning for NGINX Gateway Fabric (NGF): it creates an NGF static mode -Deployment for each Gateway that belongs to the provisioner GatewayClass. -*/ -package provisioner diff --git a/internal/mode/provisioner/handler.go b/internal/mode/provisioner/handler.go deleted file mode 100644 index b31145bd37..0000000000 --- a/internal/mode/provisioner/handler.go +++ /dev/null @@ -1,186 +0,0 @@ -package provisioner - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - v1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - "github.com/nginx/nginx-gateway-fabric/internal/framework/conditions" - "github.com/nginx/nginx-gateway-fabric/internal/framework/events" - "github.com/nginx/nginx-gateway-fabric/internal/framework/gatewayclass" - "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" - "github.com/nginx/nginx-gateway-fabric/internal/framework/status" -) - -type timeNowFunc func() metav1.Time - -// eventHandler ensures each Gateway for the specific GatewayClass has a corresponding Deployment -// of NGF configured to use that specific Gateway. -// -// eventHandler implements events.Handler interface. -type eventHandler struct { - gcName string - store *store - - // provisions maps NamespacedName of Gateway to its corresponding Deployment - provisions map[types.NamespacedName]*v1.Deployment - - statusUpdater *status.Updater - k8sClient client.Client - timeNow timeNowFunc - - staticModeDeploymentYAML []byte - - gatewayNextID int64 -} - -func newEventHandler( - gcName string, - statusUpdater *status.Updater, - k8sClient client.Client, - staticModeDeploymentYAML []byte, - timeNow timeNowFunc, -) *eventHandler { - return &eventHandler{ - store: newStore(), - provisions: make(map[types.NamespacedName]*v1.Deployment), - statusUpdater: statusUpdater, - gcName: gcName, - k8sClient: k8sClient, - staticModeDeploymentYAML: staticModeDeploymentYAML, - gatewayNextID: 1, - timeNow: timeNow, - } -} - -func (h *eventHandler) setGatewayClassStatuses(ctx context.Context) { - var reqs []status.UpdateRequest - - var gcExists bool - - for nsname, gc := range h.store.gatewayClasses { - // The order of conditions matters. Default conditions are added first so that any additional conditions will - // override them, which is ensured by DeduplicateConditions. - conds := conditions.NewDefaultGatewayClassConditions() - - if gc.Name == h.gcName { - gcExists = true - } else { - conds = append(conds, conditions.NewGatewayClassConflict()) - } - - // We ignore the boolean return value here because the provisioner only sets status, - // it does not generate config. - supportedVersionConds, _ := gatewayclass.ValidateCRDVersions(h.store.crdMetadata) - conds = append(conds, supportedVersionConds...) - - reqs = append(reqs, status.UpdateRequest{ - NsName: nsname, - ResourceType: &gatewayv1.GatewayClass{}, - Setter: func(obj client.Object) bool { - gc := helpers.MustCastObject[*gatewayv1.GatewayClass](obj) - - gcs := gatewayv1.GatewayClassStatus{ - Conditions: conditions.ConvertConditions(conditions.DeduplicateConditions(conds), gc.Generation, h.timeNow()), - } - - if status.ConditionsEqual(gc.Status.Conditions, gcs.Conditions) { - return false - } - - gc.Status = gcs - - return true - }, - }) - } - - if !gcExists { - panic(fmt.Errorf("GatewayClass %s must exist", h.gcName)) - } - - h.statusUpdater.Update(ctx, reqs...) -} - -func (h *eventHandler) ensureDeploymentsMatchGateways(ctx context.Context, logger logr.Logger) { - var gwsWithoutDeps, removedGwsWithDeps []types.NamespacedName - - for nsname, gw := range h.store.gateways { - if string(gw.Spec.GatewayClassName) != h.gcName { - continue - } - if _, exist := h.provisions[nsname]; exist { - continue - } - - gwsWithoutDeps = append(gwsWithoutDeps, nsname) - } - - for nsname := range h.provisions { - if _, exist := h.store.gateways[nsname]; exist { - continue - } - - removedGwsWithDeps = append(removedGwsWithDeps, nsname) - } - - // Create new deployments - - for _, nsname := range gwsWithoutDeps { - deployment, err := prepareDeployment(h.staticModeDeploymentYAML, h.generateDeploymentID(), nsname) - if err != nil { - panic(fmt.Errorf("failed to prepare deployment: %w", err)) - } - - if err = h.k8sClient.Create(ctx, deployment); err != nil { - panic(fmt.Errorf("failed to create deployment: %w", err)) - } - - h.provisions[nsname] = deployment - - logger.Info( - "Created deployment", - "deployment", client.ObjectKeyFromObject(deployment), - "gateway", nsname, - ) - } - - // Remove unnecessary deployments - - for _, nsname := range removedGwsWithDeps { - deployment := h.provisions[nsname] - - if err := h.k8sClient.Delete(ctx, deployment); err != nil { - panic(fmt.Errorf("failed to delete deployment: %w", err)) - } - - delete(h.provisions, nsname) - - logger.Info( - "Deleted deployment", - "deployment", client.ObjectKeyFromObject(deployment), - "gateway", nsname, - ) - } -} - -func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, batch events.EventBatch) { - h.store.update(batch) - h.setGatewayClassStatuses(ctx) - h.ensureDeploymentsMatchGateways(ctx, logger) -} - -func (h *eventHandler) generateDeploymentID() string { - // This approach will break if the provisioner is restarted, because the existing Gateways might get - // IDs different from the previous replica of the provisioner. - id := h.gatewayNextID - h.gatewayNextID++ - - return fmt.Sprintf("nginx-gateway-%d", id) -} diff --git a/internal/mode/provisioner/handler_test.go b/internal/mode/provisioner/handler_test.go deleted file mode 100644 index 97c870179e..0000000000 --- a/internal/mode/provisioner/handler_test.go +++ /dev/null @@ -1,570 +0,0 @@ -package provisioner - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - v1 "k8s.io/api/apps/v1" - apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - . "github.com/onsi/gomega" - - embeddedfiles "github.com/nginx/nginx-gateway-fabric" - "github.com/nginx/nginx-gateway-fabric/internal/framework/conditions" - "github.com/nginx/nginx-gateway-fabric/internal/framework/events" - "github.com/nginx/nginx-gateway-fabric/internal/framework/gatewayclass" - "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" - "github.com/nginx/nginx-gateway-fabric/internal/framework/status" -) - -var _ = Describe("handler", func() { - const ( - gcName = "test-gc" - ) - var ( - handler *eventHandler - - statusUpdater *status.Updater - k8sclient client.Client - crd *metav1.PartialObjectMetadata - gc *gatewayv1.GatewayClass - - fakeTimeNow timeNowFunc - ) - - BeforeEach(OncePerOrdered, func() { - scheme := runtime.NewScheme() - - Expect(gatewayv1.Install(scheme)).Should(Succeed()) - Expect(v1.AddToScheme(scheme)).Should(Succeed()) - Expect(apiext.AddToScheme(scheme)).Should(Succeed()) - - k8sclient = fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource( - &gatewayv1.Gateway{}, - &gatewayv1.GatewayClass{}, - ). - Build() - - fakeTime := helpers.PrepareTimeForFakeClient(metav1.Now()) - fakeTimeNow = func() metav1.Time { - return fakeTime - } - - statusUpdater = status.NewUpdater(k8sclient, logr.Discard()) - - // Add GatewayClass CRD to the cluster - crd = &metav1.PartialObjectMetadata{ - TypeMeta: metav1.TypeMeta{ - Kind: "CustomResourceDefinition", - APIVersion: "apiextensions.k8s.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "gatewayclasses.gateway.networking.k8s.io", - Annotations: map[string]string{ - gatewayclass.BundleVersionAnnotation: gatewayclass.SupportedVersion, - }, - }, - } - - err := k8sclient.Create(context.Background(), crd) - Expect(err).ToNot(HaveOccurred()) - - gc = &gatewayv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: gcName, - }, - } - }) - - createGateway := func(gwNsName types.NamespacedName) *gatewayv1.Gateway { - return &gatewayv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: gwNsName.Namespace, - Name: gwNsName.Name, - }, - Spec: gatewayv1.GatewaySpec{ - GatewayClassName: gcName, - }, - } - } - - itShouldUpsertGatewayClass := func() { - // Add GatewayClass to the cluster - - err := k8sclient.Create(context.Background(), gc) - Expect(err).ToNot(HaveOccurred()) - - // UpsertGatewayClass and CRD - - batch := []interface{}{ - &events.UpsertEvent{ - Resource: gc, - }, - &events.UpsertEvent{ - Resource: crd, - }, - } - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - // Ensure GatewayClass is accepted - - clusterGc := &gatewayv1.GatewayClass{} - err = k8sclient.Get(context.Background(), client.ObjectKeyFromObject(gc), clusterGc) - - Expect(err).ToNot(HaveOccurred()) - - expectedConditions := []metav1.Condition{ - { - Type: string(gatewayv1.GatewayClassConditionStatusAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: "Accepted", - Message: "GatewayClass is accepted", - }, - { - Type: string(gatewayv1.GatewayClassReasonSupportedVersion), - Status: metav1.ConditionTrue, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: "SupportedVersion", - Message: "Gateway API CRD versions are supported", - }, - } - - Expect(clusterGc.Status.Conditions).To(Equal(expectedConditions)) - } - - itShouldUpsertGateway := func(gwNsName types.NamespacedName, seqNumber int64) { - batch := []interface{}{ - &events.UpsertEvent{ - Resource: createGateway(gwNsName), - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - depNsName := types.NamespacedName{ - Namespace: "nginx-gateway", - Name: fmt.Sprintf("nginx-gateway-%d", seqNumber), - } - - dep := &v1.Deployment{} - err := k8sclient.Get(context.Background(), depNsName, dep) - - Expect(err).ToNot(HaveOccurred()) - - Expect(dep.ObjectMeta.Namespace).To(Equal("nginx-gateway")) - Expect(dep.ObjectMeta.Name).To(Equal(depNsName.Name)) - Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement("static-mode")) - expectedGwFlag := fmt.Sprintf("--gateway=%s", gwNsName.String()) - Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement(expectedGwFlag)) - Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--update-gatewayclass-status=false")) - expectedLockFlag := fmt.Sprintf("--leader-election-lock-name=%s", gwNsName.Name) - Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement(expectedLockFlag)) - } - - itShouldUpsertCRD := func(version string, accepted bool) { - updatedCRD := crd - updatedCRD.Annotations[gatewayclass.BundleVersionAnnotation] = version - - err := k8sclient.Update(context.Background(), updatedCRD) - Expect(err).ToNot(HaveOccurred()) - - batch := []interface{}{ - &events.UpsertEvent{ - Resource: updatedCRD, - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - updatedGC := &gatewayv1.GatewayClass{} - - err = k8sclient.Get(context.Background(), client.ObjectKeyFromObject(gc), updatedGC) - Expect(err).ToNot(HaveOccurred()) - - var expConds []metav1.Condition - if !accepted { - expConds = []metav1.Condition{ - { - Type: string(gatewayv1.GatewayClassConditionStatusAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: string(gatewayv1.GatewayClassReasonUnsupportedVersion), - Message: fmt.Sprintf("Gateway API CRD versions are not supported. "+ - "Please install version %s", gatewayclass.SupportedVersion), - }, - { - Type: string(gatewayv1.GatewayClassReasonSupportedVersion), - Status: metav1.ConditionFalse, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: string(gatewayv1.GatewayClassReasonUnsupportedVersion), - Message: fmt.Sprintf("Gateway API CRD versions are not supported. "+ - "Please install version %s", gatewayclass.SupportedVersion), - }, - } - } else { - expConds = []metav1.Condition{ - { - Type: string(gatewayv1.GatewayClassConditionStatusAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: string(gatewayv1.GatewayClassReasonAccepted), - Message: "GatewayClass is accepted", - }, - { - Type: string(gatewayv1.GatewayClassReasonSupportedVersion), - Status: metav1.ConditionFalse, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: string(gatewayv1.GatewayClassReasonUnsupportedVersion), - Message: fmt.Sprintf("Gateway API CRD versions are not recommended. "+ - "Recommended version is %s", gatewayclass.SupportedVersion), - }, - } - } - - Expect(updatedGC.Status.Conditions).To(Equal(expConds)) - } - - itShouldPanicWhenUpsertingGateway := func(gwNsName types.NamespacedName) { - batch := []interface{}{ - &events.UpsertEvent{ - Resource: createGateway(gwNsName), - }, - } - - handle := func() { - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - } - - Expect(handle).Should(Panic()) - } - - Describe("Core cases", Ordered, func() { - var gwNsName1, gwNsName2 types.NamespacedName - - BeforeAll(func() { - gwNsName1 = types.NamespacedName{ - Namespace: "test-ns-1", - Name: "test-gw-1", - } - gwNsName2 = types.NamespacedName{ - Namespace: "test-ns-2", - Name: "test-gw-2", - } - - handler = newEventHandler( - gcName, - statusUpdater, - k8sclient, - embeddedfiles.StaticModeDeploymentYAML, - fakeTimeNow, - ) - }) - - When("upserting GatewayClass", func() { - It("should make GatewayClass Accepted", func() { - itShouldUpsertGatewayClass() - }) - }) - - When("upserting first Gateway", func() { - It("should create first Deployment", func() { - itShouldUpsertGateway(gwNsName1, 1) - }) - }) - - When("upserting first Gateway again", func() { - It("must retain Deployment", func() { - itShouldUpsertGateway(gwNsName1, 1) - }) - }) - - When("upserting second Gateway", func() { - It("should create second Deployment", func() { - itShouldUpsertGateway(gwNsName2, 2) - }) - }) - - When("deleting first Gateway", func() { - It("should remove first Deployment", func() { - batch := []interface{}{ - &events.DeleteEvent{ - Type: &gatewayv1.Gateway{}, - NamespacedName: gwNsName1, - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - deps := &v1.DeploymentList{} - - err := k8sclient.List(context.Background(), deps) - - Expect(err).ToNot(HaveOccurred()) - Expect(deps.Items).To(HaveLen(1)) - Expect(deps.Items[0].ObjectMeta.Name).To(Equal("nginx-gateway-2")) - }) - }) - - When("deleting second Gateway", func() { - It("should remove second Deployment", func() { - batch := []interface{}{ - &events.DeleteEvent{ - Type: &gatewayv1.Gateway{}, - NamespacedName: gwNsName2, - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - deps := &v1.DeploymentList{} - - err := k8sclient.List(context.Background(), deps) - - Expect(err).ToNot(HaveOccurred()) - Expect(deps.Items).To(BeEmpty()) - }) - }) - - When("upserting Gateway for a different GatewayClass", func() { - It("should not create Deployment", func() { - gw := &gatewayv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-gw-3", - Namespace: "test-ns-3", - }, - Spec: gatewayv1.GatewaySpec{ - GatewayClassName: "some-class", - }, - } - - batch := []interface{}{ - &events.UpsertEvent{ - Resource: gw, - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - deps := &v1.DeploymentList{} - err := k8sclient.List(context.Background(), deps) - - Expect(err).ToNot(HaveOccurred()) - Expect(deps.Items).To(BeEmpty()) - }) - }) - - When("upserting GatewayClass that is not set in command-line argument", func() { - It("should set the proper status if this controller is referenced", func() { - newGC := &gatewayv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "unknown-gc", - }, - Spec: gatewayv1.GatewayClassSpec{ - ControllerName: "test.example.com", - }, - } - - err := k8sclient.Create(context.Background(), newGC) - Expect(err).ToNot(HaveOccurred()) - - batch := []interface{}{ - &events.UpsertEvent{ - Resource: newGC, - }, - &events.UpsertEvent{ - Resource: crd, - }, - } - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - unknownGC := &gatewayv1.GatewayClass{} - err = k8sclient.Get(context.Background(), client.ObjectKeyFromObject(newGC), unknownGC) - Expect(err).ToNot(HaveOccurred()) - - expectedConditions := []metav1.Condition{ - { - Type: string(gatewayv1.GatewayClassReasonSupportedVersion), - Status: metav1.ConditionTrue, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: "SupportedVersion", - Message: "Gateway API CRD versions are supported", - }, - { - Type: string(gatewayv1.GatewayClassConditionStatusAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: 0, - LastTransitionTime: fakeTimeNow(), - Reason: string(conditions.GatewayClassReasonGatewayClassConflict), - Message: conditions.GatewayClassMessageGatewayClassConflict, - }, - } - Expect(unknownGC.Status.Conditions).To(Equal(expectedConditions)) - }) - }) - - When("upserting Gateway API CRD that is not a supported major version", func() { - It("should set the SupportedVersion and Accepted statuses to false on GatewayClass", func() { - itShouldUpsertCRD("v99.0.0", false /* accepted */) - }) - }) - - When("upserting Gateway API CRD that is not a supported minor version", func() { - It("should set the SupportedVersion status to false and Accepted status to true on GatewayClass", func() { - itShouldUpsertCRD("1.99.0", true /* accepted */) - }) - }) - }) - - Describe("Edge cases", func() { - var gwNsName types.NamespacedName - - BeforeEach(func() { - gwNsName = types.NamespacedName{ - Namespace: "test-ns", - Name: "test-gw", - } - - handler = newEventHandler( - gcName, - statusUpdater, - k8sclient, - embeddedfiles.StaticModeDeploymentYAML, - fakeTimeNow, - ) - }) - - DescribeTable("Edge cases for events", - func(e interface{}) { - batch := []interface{}{e} - - handle := func() { - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - } - - Expect(handle).Should(Panic()) - }, - Entry("should panic for an unknown event type", - &struct{}{}), - Entry("should panic for an unknown type of resource in upsert event", - &events.UpsertEvent{ - Resource: &gatewayv1.HTTPRoute{}, - }), - Entry("should panic for an unknown type of resource in delete event", - &events.DeleteEvent{ - Type: &gatewayv1.HTTPRoute{}, - }), - ) - - When("upserting Gateway when GatewayClass doesn't exist", func() { - It("should panic", func() { - itShouldPanicWhenUpsertingGateway(gwNsName) - }) - }) - - When("upserting Gateway when Deployment can't be created", func() { - It("should panic", func() { - itShouldUpsertGatewayClass() - - // Create a deployment so that the Handler will fail to create it because it already exists. - - dep := &v1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx-gateway-1", - }, - } - - err := k8sclient.Create(context.Background(), dep) - Expect(err).ToNot(HaveOccurred()) - - itShouldPanicWhenUpsertingGateway(gwNsName) - }) - }) - - When("deleting Gateway when Deployment can't be deleted", func() { - It("should panic", func() { - itShouldUpsertGatewayClass() - itShouldUpsertGateway(gwNsName, 1) - - // Delete the deployment so that the Handler will fail to delete it because it doesn't exist. - - dep := &v1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx-gateway-1", - }, - } - - err := k8sclient.Delete(context.Background(), dep) - Expect(err).ToNot(HaveOccurred()) - - batch := []interface{}{ - &events.DeleteEvent{ - Type: &gatewayv1.Gateway{}, - NamespacedName: gwNsName, - }, - } - - handle := func() { - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - } - - Expect(handle).Should(Panic()) - }) - }) - - When("deleting GatewayClass", func() { - It("should panic", func() { - itShouldUpsertGatewayClass() - - batch := []interface{}{ - &events.DeleteEvent{ - Type: &gatewayv1.GatewayClass{}, - NamespacedName: types.NamespacedName{ - Name: gcName, - }, - }, - } - - handle := func() { - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - } - - Expect(handle).Should(Panic()) - }) - }) - - When("upserting Gateway with broken static Deployment YAML", func() { - It("it should panic", func() { - handler = newEventHandler( - gcName, - statusUpdater, - k8sclient, - []byte("broken YAML"), - fakeTimeNow, - ) - - itShouldUpsertGatewayClass() - itShouldPanicWhenUpsertingGateway(types.NamespacedName{Namespace: "test-ns", Name: "test-gw"}) - }) - }) - }) -}) diff --git a/internal/mode/provisioner/manager.go b/internal/mode/provisioner/manager.go deleted file mode 100644 index bb2b93b6e5..0000000000 --- a/internal/mode/provisioner/manager.go +++ /dev/null @@ -1,152 +0,0 @@ -package provisioner - -import ( - "fmt" - - "github.com/go-logr/logr" - v1 "k8s.io/api/apps/v1" - apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - ctlr "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" - gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - - embeddedfiles "github.com/nginx/nginx-gateway-fabric" - "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" - "github.com/nginx/nginx-gateway-fabric/internal/framework/controller/predicate" - "github.com/nginx/nginx-gateway-fabric/internal/framework/events" - "github.com/nginx/nginx-gateway-fabric/internal/framework/gatewayclass" - "github.com/nginx/nginx-gateway-fabric/internal/framework/status" - ngftypes "github.com/nginx/nginx-gateway-fabric/internal/framework/types" -) - -// Config is configuration for the provisioner mode. -type Config struct { - Logger logr.Logger - GatewayClassName string - GatewayCtlrName string -} - -// StartManager starts a Manager for the provisioner mode, which provisions -// a Deployment of NGF (static mode) for each Gateway of the provisioner GatewayClass. -// -// The provisioner mode is introduced to allow running Gateway API conformance tests for NGF, which expects -// an independent data plane instance being provisioned for each Gateway. -// -// The provisioner mode is not intended to be used in production (in the short term), as it lacks support for -// many important features. See https://github.com/nginx/nginx-gateway-fabric/issues/634 for more details. -func StartManager(cfg Config) error { - scheme := runtime.NewScheme() - utilruntime.Must(gatewayv1.Install(scheme)) - utilruntime.Must(v1.AddToScheme(scheme)) - utilruntime.Must(apiext.AddToScheme(scheme)) - - options := manager.Options{ - Scheme: scheme, - Logger: cfg.Logger, - } - clusterCfg := ctlr.GetConfigOrDie() - - mgr, err := manager.New(clusterCfg, options) - if err != nil { - return fmt.Errorf("cannot build runtime manager: %w", err) - } - - crdWithGVK := apiext.CustomResourceDefinition{} - crdWithGVK.SetGroupVersionKind( - schema.GroupVersionKind{Group: apiext.GroupName, Version: "v1", Kind: "CustomResourceDefinition"}, - ) - - // Note: for any new object type or a change to the existing one, - // make sure to also update firstBatchPreparer creation below - controllerRegCfgs := []struct { - objectType ngftypes.ObjectType - options []controller.Option - }{ - { - objectType: &gatewayv1.GatewayClass{}, - options: []controller.Option{ - controller.WithK8sPredicate(predicate.GatewayClassPredicate{ControllerName: cfg.GatewayCtlrName}), - }, - }, - { - objectType: &gatewayv1.Gateway{}, - }, - { - objectType: &crdWithGVK, - options: []controller.Option{ - controller.WithOnlyMetadata(), - controller.WithK8sPredicate( - predicate.AnnotationPredicate{Annotation: gatewayclass.BundleVersionAnnotation}, - ), - }, - }, - } - - ctx := ctlr.SetupSignalHandler() - eventCh := make(chan interface{}) - - for _, regCfg := range controllerRegCfgs { - if err := controller.Register( - ctx, - regCfg.objectType, - regCfg.objectType.GetObjectKind().GroupVersionKind().Kind, - mgr, - eventCh, - regCfg.options..., - ); err != nil { - return fmt.Errorf("cannot register controller for %T: %w", regCfg.objectType, err) - } - } - - partialObjectMetadataList := &metav1.PartialObjectMetadataList{} - partialObjectMetadataList.SetGroupVersionKind( - schema.GroupVersionKind{ - Group: apiext.GroupName, - Version: "v1", - Kind: "CustomResourceDefinition", - }, - ) - - firstBatchPreparer := events.NewFirstEventBatchPreparerImpl( - mgr.GetCache(), - []client.Object{ - &gatewayv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: cfg.GatewayClassName}}, - }, - []client.ObjectList{ - &gatewayv1.GatewayList{}, - partialObjectMetadataList, - }, - ) - - statusUpdater := status.NewUpdater( - mgr.GetClient(), - cfg.Logger.WithName("statusUpdater"), - ) - - handler := newEventHandler( - cfg.GatewayClassName, - statusUpdater, - mgr.GetClient(), - embeddedfiles.StaticModeDeploymentYAML, - metav1.Now, - ) - - eventLoop := events.NewEventLoop( - eventCh, - cfg.Logger.WithName("eventLoop"), - handler, - firstBatchPreparer, - ) - - if err := mgr.Add(eventLoop); err != nil { - return fmt.Errorf("cannot register event loop: %w", err) - } - - cfg.Logger.Info("Starting manager") - return mgr.Start(ctx) -} diff --git a/internal/mode/provisioner/provisioner_suite_test.go b/internal/mode/provisioner/provisioner_suite_test.go deleted file mode 100644 index 1435a2230e..0000000000 --- a/internal/mode/provisioner/provisioner_suite_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package provisioner - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestProvisioner(t *testing.T) { - t.Parallel() - RegisterFailHandler(Fail) - RunSpecs(t, "Provisioner Suite") -} diff --git a/internal/mode/provisioner/store.go b/internal/mode/provisioner/store.go deleted file mode 100644 index ebc6afcc17..0000000000 --- a/internal/mode/provisioner/store.go +++ /dev/null @@ -1,58 +0,0 @@ -package provisioner - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - v1 "sigs.k8s.io/gateway-api/apis/v1" - - "github.com/nginx/nginx-gateway-fabric/internal/framework/events" -) - -// store stores the cluster state needed by the provisioner and allows to update it from the events. -type store struct { - gatewayClasses map[types.NamespacedName]*v1.GatewayClass - gateways map[types.NamespacedName]*v1.Gateway - crdMetadata map[types.NamespacedName]*metav1.PartialObjectMetadata -} - -func newStore() *store { - return &store{ - gatewayClasses: make(map[types.NamespacedName]*v1.GatewayClass), - gateways: make(map[types.NamespacedName]*v1.Gateway), - crdMetadata: make(map[types.NamespacedName]*metav1.PartialObjectMetadata), - } -} - -func (s *store) update(batch events.EventBatch) { - for _, event := range batch { - switch e := event.(type) { - case *events.UpsertEvent: - switch obj := e.Resource.(type) { - case *v1.GatewayClass: - s.gatewayClasses[client.ObjectKeyFromObject(obj)] = obj - case *v1.Gateway: - s.gateways[client.ObjectKeyFromObject(obj)] = obj - case *metav1.PartialObjectMetadata: - s.crdMetadata[client.ObjectKeyFromObject(obj)] = obj - default: - panic(fmt.Errorf("unknown resource type %T", e.Resource)) - } - case *events.DeleteEvent: - switch e.Type.(type) { - case *v1.GatewayClass: - delete(s.gatewayClasses, e.NamespacedName) - case *v1.Gateway: - delete(s.gateways, e.NamespacedName) - case *metav1.PartialObjectMetadata: - delete(s.crdMetadata, e.NamespacedName) - default: - panic(fmt.Errorf("unknown resource type %T", e.Type)) - } - default: - panic(fmt.Errorf("unknown event type %T", e)) - } - } -} diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 930be5d01e..3459a7134f 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -150,18 +150,13 @@ func StartManager(cfg config.Config) error { if cfg.MetricsConfig.Enabled { constLabels := map[string]string{"class": cfg.GatewayClassName} - ngxruntimeCollector := collectors.NewManagerMetricsCollector(constLabels) handlerCollector = collectors.NewControllerCollector(constLabels) - handlerCollector, ok := handlerCollector.(prometheus.Collector) if !ok { return fmt.Errorf("handlerCollector is not a prometheus.Collector: %w", frameworkStatus.ErrFailedAssert) } - metrics.Registry.MustRegister( - ngxruntimeCollector, - handlerCollector, - ) + metrics.Registry.MustRegister(handlerCollector) } statusUpdater := frameworkStatus.NewUpdater( diff --git a/internal/mode/static/metrics/collectors/nginx_runtime.go b/internal/mode/static/metrics/collectors/nginx_runtime.go deleted file mode 100644 index c762171d76..0000000000 --- a/internal/mode/static/metrics/collectors/nginx_runtime.go +++ /dev/null @@ -1,108 +0,0 @@ -package collectors - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/nginx/nginx-gateway-fabric/internal/mode/static/metrics" -) - -// MetricsCollector is an interface for the metrics of the NGINX runtime manager. -// -//counterfeiter:generate . MetricsCollector -type MetricsCollector interface { - IncReloadCount() - IncReloadErrors() - ObserveLastReloadTime(ms time.Duration) -} - -// NginxRuntimeCollector implements runtime.Collector interface and prometheus.Collector interface. -type NginxRuntimeCollector struct { - // Metrics - reloadsTotal prometheus.Counter - reloadsError prometheus.Counter - configStale prometheus.Gauge - reloadsDuration prometheus.Histogram -} - -// NewManagerMetricsCollector creates a new NginxRuntimeCollector. -func NewManagerMetricsCollector(constLabels map[string]string) *NginxRuntimeCollector { - nc := &NginxRuntimeCollector{ - reloadsTotal: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "nginx_reloads_total", - Namespace: metrics.Namespace, - Help: "Number of successful NGINX reloads", - ConstLabels: constLabels, - }), - reloadsError: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "nginx_reload_errors_total", - Namespace: metrics.Namespace, - Help: "Number of unsuccessful NGINX reloads", - ConstLabels: constLabels, - }, - ), - configStale: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "nginx_stale_config", - Namespace: metrics.Namespace, - Help: "Indicates if NGINX is not serving the latest configuration.", - ConstLabels: constLabels, - }, - ), - reloadsDuration: prometheus.NewHistogram( - prometheus.HistogramOpts{ - Name: "nginx_reloads_milliseconds", - Namespace: metrics.Namespace, - Help: "Duration in milliseconds of NGINX reloads", - ConstLabels: constLabels, - Buckets: []float64{500, 1000, 5000, 10000, 30000}, - }, - ), - } - return nc -} - -// IncReloadCount increments the counter of successful NGINX reloads and sets the stale config status to false. -func (c *NginxRuntimeCollector) IncReloadCount() { - c.reloadsTotal.Inc() - c.updateConfigStaleStatus(false) -} - -// IncReloadErrors increments the counter of NGINX reload errors and sets the stale config status to true. -func (c *NginxRuntimeCollector) IncReloadErrors() { - c.reloadsError.Inc() - c.updateConfigStaleStatus(true) -} - -// updateConfigStaleStatus updates the last NGINX reload status metric. -func (c *NginxRuntimeCollector) updateConfigStaleStatus(stale bool) { - var status float64 - if stale { - status = 1.0 - } - c.configStale.Set(status) -} - -// ObserveLastReloadTime adds the last NGINX reload time to the histogram. -func (c *NginxRuntimeCollector) ObserveLastReloadTime(duration time.Duration) { - c.reloadsDuration.Observe(float64(duration / time.Millisecond)) -} - -// Describe implements prometheus.Collector interface Describe method. -func (c *NginxRuntimeCollector) Describe(ch chan<- *prometheus.Desc) { - c.reloadsTotal.Describe(ch) - c.reloadsError.Describe(ch) - c.configStale.Describe(ch) - c.reloadsDuration.Describe(ch) -} - -// Collect implements the prometheus.Collector interface Collect method. -func (c *NginxRuntimeCollector) Collect(ch chan<- prometheus.Metric) { - c.reloadsTotal.Collect(ch) - c.reloadsError.Collect(ch) - c.configStale.Collect(ch) - c.reloadsDuration.Collect(ch) -} diff --git a/scripts/generate-manifests.sh b/scripts/generate-manifests.sh index f52743e382..ca3e9c2041 100755 --- a/scripts/generate-manifests.sh +++ b/scripts/generate-manifests.sh @@ -30,10 +30,3 @@ done # For OpenShift, we don't need a Helm example so we generate the manifests from the default values.yaml generate_manifests openshift - -# FIXME(lucacome): Implement a better way to generate the static deployment file -# https://github.com/nginx/nginx-gateway-fabric/issues/2326 -helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set nginxGateway.metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml >config/tests/static-deployment.yaml -sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' config/tests/static-deployment.yaml -sed -i.bak '/helm.sh/d' config/tests/static-deployment.yaml -rm -f config/tests/static-deployment.yaml.bak diff --git a/tests/Makefile b/tests/Makefile index a7e8e5717f..dd82633d76 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,7 +13,6 @@ GW_SVC_GKE_INTERNAL = false NGF_VERSION ?= edge## NGF version to be tested PULL_POLICY = Never## Pull policy for the images NGINX_CONF_DIR = internal/mode/static/nginx/conf -PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation,GatewayInfrastructurePropagation STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS @@ -74,9 +73,6 @@ cleanup-conformance-tests: ## Clean up conformance tests fixtures kubectl delete pod conformance kubectl delete -f conformance/conformance-rbac.yaml -.PHONY: build -build: generate-static-deployment - .PHONY: reset-go-modules reset-go-modules: ## Reset the go modules changes git checkout -- ../go.mod ../go.sum @@ -169,16 +165,7 @@ delete-gke-cluster: ## Delete the GKE cluster add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks ./scripts/add-local-ip-auth-networks.sh -HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set nginx.service.type=ClusterIP --skip-schema-validation - -.PHONY: deploy-updated-provisioner -deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster - yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f - - yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f - - -.PHONY: generate-static-deployment -generate-static-deployment: - helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) > $(SELF_DIR)config/tests/static-deployment.yaml +HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginx.service.type=ClusterIP --skip-schema-validation # this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job) # it overrides the target in the main Makefile when the GW_API_VERSION is set to main @@ -187,27 +174,15 @@ install-gateway-crds: kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f - endif -.PHONY: install-ngf-local-build -install-ngf-local-build: deploy-updated-provisioner - .PHONY: install-ngf-local-no-build -install-ngf-local-no-build: load-images helm-install-local deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster but do not build the NGF image - -.PHONY: install-ngf-local-build-with-plus -install-ngf-local-build-with-plus: deploy-updated-provisioner +install-ngf-local-no-build: load-images helm-install-local ## Install NGF from local build on configured kind cluster but do not build the NGF image .PHONY: install-ngf-local-no-build-with-plus -install-ngf-local-no-build-with-plus: load-images-with-plus helm-install-local-with-plus deploy-updated-provisioner ## Install NGF with Plus from local build with provisioner on configured kind cluster but do not build the NGF image - -.PHONY: install-ngf-edge -install-ngf-edge: load-images helm-install-local ## Install NGF with provisioner from edge on configured kind cluster - kubectl apply -f $(PROVISIONER_MANIFEST) +install-ngf-local-no-build-with-plus: load-images-with-plus helm-install-local-with-plus ## Install NGF with Plus from local build on configured kind cluster but do not build the NGF image .PHONY: uninstall-ngf -uninstall-ngf: ## Uninstall NGF on configured kind cluster and undo manifest changes +uninstall-ngf: ## Uninstall NGF on configured kind cluster -helm uninstall nginx-gateway -n nginx-gateway -make uninstall-gateway-crds - -kubectl delete clusterrole nginx-gateway-provisioner - -kubectl delete clusterrolebinding nginx-gateway-provisioner -kubectl delete namespace nginx-gateway -kubectl kustomize ../config/crd | kubectl delete -f - diff --git a/tests/README.md b/tests/README.md index 1db86ebc3b..16df0bc36a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -19,7 +19,6 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide - [Step 1 - Install NGINX Gateway Fabric to configured kind cluster](#step-1---install-nginx-gateway-fabric-to-configured-kind-cluster) - [Option 1 - Build and install NGINX Gateway Fabric from local to configured kind cluster](#option-1---build-and-install-nginx-gateway-fabric-from-local-to-configured-kind-cluster) - [Option 2 - Install NGINX Gateway Fabric from local already built image to configured kind cluster](#option-2---install-nginx-gateway-fabric-from-local-already-built-image-to-configured-kind-cluster) - - [Option 3 - Install NGINX Gateway Fabric from edge to configured kind cluster](#option-3---install-nginx-gateway-fabric-from-edge-to-configured-kind-cluster) - [Step 2 - Build conformance test runner image](#step-2---build-conformance-test-runner-image) - [Step 3 - Run Gateway conformance tests](#step-3---run-gateway-conformance-tests) - [Step 4 - Cleanup the conformance test fixtures and uninstall NGINX Gateway Fabric](#step-4---cleanup-the-conformance-test-fixtures-and-uninstall-nginx-gateway-fabric) @@ -158,15 +157,6 @@ Or, to install NGF with NGINX Plus enabled: make install-ngf-local-no-build-with-plus ``` -#### Option 3 - Install NGINX Gateway Fabric from edge to configured kind cluster - -You can also skip the build NGF image step and prepare the environment to instead use the `edge` image. Note that this -option does not currently support installing with NGINX Plus enabled. - -```makefile -make install-ngf-edge -``` - ### Step 2 - Build conformance test runner image > Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following diff --git a/tests/conformance/provisioner/README.md b/tests/conformance/provisioner/README.md deleted file mode 100644 index c9f5bc36ae..0000000000 --- a/tests/conformance/provisioner/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Provisioner - -Provisioner implements data plane provisioning for NGINX Gateway Fabric (NGF): it creates an NGF static mode -Deployment for each Gateway that belongs to the provisioner GatewayClass. - -```text -Usage: - gateway provisioner-mode [flags] - -Flags: - -h, --help help for provisioner-mode - -Global Flags: - --gateway-ctlr-name string The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is 'gateway.nginx.org' (default "") - --gatewayclass string The name of the GatewayClass resource. Every NGINX Gateway Fabric must have a unique corresponding GatewayClass resource. (default "") -``` - -> Note: Provisioner is not ready for production yet (see this issue for more details -https://github.com/nginx/nginx-gateway-fabric/issues/634). However, it can be used in the Gateway API conformance -tests, which expect a Gateway API implementation to provision an independent data plane per Gateway. -> -> Note: Provisioner uses [this manifest](https://github.com/nginx/nginx-gateway-fabric/blob/main/config/tests/static-deployment.yaml) -to create an NGF static mode Deployment. -> This manifest gets included into the NGF binary during the NGF build. To customize the Deployment, modify the -manifest and **re-build** NGF. - -How to deploy: - -1. Follow the [installation](https://docs.nginx.com/nginx-gateway-fabric/installation/) instructions up until the Deploy the NGINX Gateway Fabric step - to deploy prerequisites for both the static mode Deployments and the provisioner. -1. Deploy provisioner: - - ```shell - kubectl apply -f provisioner.yaml - ``` - -1. Confirm the provisioner is running in nginx-gateway namespace: - - ```shell - kubectl get pods -n nginx-gateway - ``` - - ```text - - NAME READY STATUS RESTARTS AGE - nginx-gateway-provisioner-6c9d9fdcb8-b2pf8 1/1 Running 0 11m - ``` diff --git a/tests/conformance/provisioner/provisioner.yaml b/tests/conformance/provisioner/provisioner.yaml deleted file mode 100644 index 07ebae4a45..0000000000 --- a/tests/conformance/provisioner/provisioner.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-gateway-provisioner - namespace: nginx-gateway ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nginx-gateway-provisioner -rules: -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses - - gateways - verbs: - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/status - verbs: - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nginx-gateway-provisioner -subjects: -- kind: ServiceAccount - name: nginx-gateway-provisioner - namespace: nginx-gateway -roleRef: - kind: ClusterRole - name: nginx-gateway-provisioner - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-gateway-provisioner - namespace: nginx-gateway -spec: - replicas: 1 - selector: - matchLabels: - app: nginx-gateway-provisioner - template: - metadata: - labels: - app: nginx-gateway-provisioner - spec: - serviceAccountName: nginx-gateway-provisioner - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: nginx-gateway-provisioner - securityContext: - runAsUser: 1001 - args: - - provisioner-mode - - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller - - --gatewayclass=nginx