SRIOV + Performance operators integration

Dockerfile

@@ -34,8 +34,8 @@ FROM ${OPERATOR_BASE_IMAGE}
 ARG DEST_ROOT=/dest-root
 
 LABEL   com.redhat.component="compute-node-operator-container" \
-        name="cn-osp/compute-node-operator" \
-        version="0.0.1" \
+        name="compute-node-operator" \
+        version="1.0" \
         summary="Compute Node Operator" \
         io.k8s.name="compute-node-operator" \
         io.k8s.description="This image includes the compute-node-operator"

api/v1alpha1/computenodeopenstack_types.go

@@ -30,8 +30,6 @@ type ComputeNodeOpenStackSpec struct {
 	BaseWorkerMachineSetName string `json:"baseWorkerMachineSetName"`
 	// Number of workers
 	Workers int32 `json:"workers,omitempty"`
-	// Cores Pinning
-	CorePinning string `json:"corePinning,omitempty"`
 	// Make or not the Node dedicated to OSP workloads (does not account for infra pods)
 	Dedicated bool `json:"dedicated,omitempty"`
 	// Make the nodes to be Network Gateways
@@ -50,6 +48,8 @@ type ComputeNodeOpenStackSpec struct {
 	Compute NovaCompute `json:"compute,omitempty"`
 	// Network/Neutron configuration
 	Network NeutronNetwork `json:"network,omitempty"`
+	// Performance configuration
+	Performance PerformanceConfig `json:"performance,omitempty"`
 	// Manage selinux - Defaults to false
 	SelinuxDisabled bool `json:"selinuxDisabled,omitempty"`
 	// Cell the computes are assigned to, default cell1
@@ -81,6 +81,8 @@ type NovaCompute struct {
 	NovaComputeCPUDedicatedSet string `json:"novaComputeCPUDedicatedSet,omitempty"`
 	// CPU Shared Set
 	NovaComputeCPUSharedSet string `json:"novaComputeCPUSharedSet,omitempty"`
+	// Memory reserved for the host
+	NovaReservedHostMemory int32 `json:"novaReservedHostMemory,omitempty"`
 	// sshd migration port
 	SshdPort int32 `json:"sshdPort,omitempty"`
 	// Nova secret containing the needed password, default "nova-secret"
@@ -93,18 +95,40 @@ type NovaCompute struct {
 	TransportURLSecret string `json:"transportURLSecret,omitempty"`
 }
 
+// PerformanceConfig defines the OS tuning parameters
+type PerformanceConfig struct {
+	HugePages HugepagesConfig `json:"hugepages,omitempty"`
+	CPU       CPUConfig       `json:"cpu,omitempty"`
+}
+
+// HugepagesConfig defines the hugepages parameters
+type HugepagesConfig struct {
+	DefaultHugePagesSize string              `json:"defaultHugePagesSize"`
+	Pages                []map[string]string `json:"pages,omitempty"`
+}
+
+// CPUConfig defines the cpu isolation parameters
+type CPUConfig struct {
+	Isolated string `json:"isolated"`
+	Reserved string `json:"reserved"`
+}
+
 // NeutronNetwork defines neutron configuration parameters
 type NeutronNetwork struct {
-	Nic              string      `json:"nic"`
-	BridgeMappings   string      `json:"bridgeMappings,omitempty"`
-	MechanishDrivers string      `json:"mechanismDrivers,omitempty"`
-	ServicePlugings  string      `json:"servicePlugins,omitempty"`
-	Sriov            SriovConfig `json:"sriov,omitempty"`
+	Nic              string        `json:"nic"`
+	BridgeMappings   string        `json:"bridgeMappings,omitempty"`
+	MechanishDrivers string        `json:"mechanismDrivers,omitempty"`
+	ServicePlugings  string        `json:"servicePlugins,omitempty"`
+	Sriov            []SriovConfig `json:"sriov,omitempty"`
 }
 
 // SriovConfig defines SRIOV config parameters, such as nic information.
 type SriovConfig struct {
-	DevName string `json:"devName"`
+	Interface  string `json:"interface"`
+	NumVfs     int32  `json:"numVfs,omitempty"`
+	Network    string `json:"network"`
+	DeviceType string `json:"deviceType"`
+	Mtu        int32  `json:"mtu,omitempty"`
 }
 
 // NodeToDelete defines the name of the node to delete and if automatic drain is needed

bindata/nova/005-compute.yaml

@@ -8,7 +8,14 @@ spec:
   neutronSecret: {{ .NeutronSecret }}
   placementSecret: {{ .PlacementSecret }}
   transportURLSecret: {{ .TransportURLSecret }}
+  {{ if not (eq .NovaComputeCPUDedicatedSet "") }}
   novaComputeCpuDedicatedSet: {{ .NovaComputeCPUDedicatedSet }}
+  {{ end }}
+  {{ if not (eq .NovaComputeCPUSharedSet "") }}
   novaComputeCpuSharedSet: {{ .NovaComputeCPUSharedSet }}
-  novaComputeImage: docker.io/tripleomaster/centos-binary-nova-compute:current-tripleo
-  roleName: {{ .WorkerOspRole }}
+  {{ end }}
+  {{ if not (eq .NovaReservedHostMemory 0) }}
+  novaReservedHostMemory: {{ .NovaReservedHostMemory }}
+  {{ end }}
+  novaComputeImage: docker.io/tripleotrain/rhel-binary-nova-compute:current-tripleo
+  roleName: {{ .WorkerOspRole }}

bindata/worker-osp/004-worker-osp-performance-profile.yaml

@@ -0,0 +1,29 @@
+{{ if or .Isolcpus .EnableHugePages }}
+apiVersion: performance.openshift.io/v1alpha1
+kind: PerformanceProfile
+metadata:
+  name: performance-profile-{{ .WorkerOspRole }}
+spec:
+  {{ if and (not (eq .CPUIsolated "")) (not (eq .CPUReserved ""))}}
+  cpu:
+    isolated: "{{ .CPUIsolated }}"
+    reserved: "{{ .CPUReserved }}"
+  {{ end }}
+  hugepages:
+  {{ if not (eq .HugePagesDefaultSize "") }}
+    defaultHugepagesSize: "{{ .HugePagesDefaultSize }}"
+  {{ end }}
+  {{ if not (eq (len .HugePagesPages) 0) }}
+    pages:
+    {{ range $i, $m := .HugePagesPages }}
+    - size: "{{ $m.size }}"
+      count: {{ $m.count }}
+    {{ end }}
+  {{ else }}
+    pages: []
+  {{ end }}
+  realTimeKernel:
+    enabled: false 
+  nodeSelector:
+    node-role.kubernetes.io/{{ .WorkerOspRole }}: ""
+{{ end }} 

bindata/worker-osp/009-worker-osp-sriov-network-node-policy.yaml

@@ -0,0 +1,27 @@
+{{ if not (eq .AvailableNodeCount 0) }}
+  {{ $workerOspRole := .WorkerOspRole }}
+  {{ $workerOspRoleFormatted := .WorkerOspRole | replace "-" "_" }}
+  {{ range $i, $e := .Sriov }}
+    {{ if not (eq $i 0) }}
+---
+    {{ end }}
+apiVersion: sriovnetwork.openshift.io/v1
+kind: SriovNetworkNodePolicy
+metadata:
+  name: sriov-policy-{{ $workerOspRole }}-{{ $e.Interface }}
+  namespace: openstack
+  labels:
+    compute-node.openstack.org/sriov-policy: "{{ $workerOspRole }}"
+spec:
+  deviceType: {{ $e.DeviceType }}
+  mtu: {{ $e.Mtu }}
+  nicSelector:
+    pfNames:
+    - {{ $e.Interface }}
+  nodeSelector:
+    node-role.kubernetes.io/{{ $workerOspRole }}: ""
+  numVfs: {{ $e.NumVfs }}
+  priority: 5
+  resourceName: sriovnic_{{ $workerOspRoleFormatted }}_{{ $e.Interface }}
+  {{ end }}
+{{ end }}

config/crd/bases/compute-node.openstack.org_computenodeopenstacks.yaml

@@ -59,6 +59,10 @@ spec:
                 novaComputeCPUSharedSet:
                   description: CPU Shared Set
                   type: string
+                novaReservedHostMemory:
+                  description: Memory reserved for the host
+                  format: int32
+                  type: integer
                 novaSecrets:
                   description: Nova secret containing the needed password, default
                     "nova-secret"
@@ -76,9 +80,6 @@ spec:
                     default nova-<CELL>-transport-url
                   type: string
               type: object
-            corePinning:
-              description: Cores Pinning
-              type: string
             dedicated:
               description: Make or not the Node dedicated to OSP workloads (does not
                 account for infra pods)
@@ -127,14 +128,28 @@ spec:
                 servicePlugins:
                   type: string
                 sriov:
-                  description: SriovConfig defines SRIOV config parameters, such as
-                    nic information.
-                  properties:
-                    devName:
-                      type: string
-                  required:
-                  - devName
-                  type: object
+                  items:
+                    description: SriovConfig defines SRIOV config parameters, such
+                      as nic information.
+                    properties:
+                      deviceType:
+                        type: string
+                      interface:
+                        type: string
+                      mtu:
+                        format: int32
+                        type: integer
+                      network:
+                        type: string
+                      numVfs:
+                        format: int32
+                        type: integer
+                    required:
+                    - deviceType
+                    - interface
+                    - network
+                    type: object
+                  type: array
               required:
               - nic
               type: object
@@ -164,6 +179,35 @@ spec:
               description: openstackclient configmap which holds information to connect
                 to OpenStack API
               type: string
+            performance:
+              description: Performance configuration
+              properties:
+                cpu:
+                  description: CPUConfig defines the cpu isolation parameters
+                  properties:
+                    isolated:
+                      type: string
+                    reserved:
+                      type: string
+                  required:
+                  - isolated
+                  - reserved
+                  type: object
+                hugepages:
+                  description: HugepagesConfig defines the hugepages parameters
+                  properties:
+                    defaultHugePagesSize:
+                      type: string
+                    pages:
+                      items:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      type: array
+                  required:
+                  - defaultHugePagesSize
+                  type: object
+              type: object
             roleName:
               description: Name of the worker role created for OSP computes
               type: string

config/rbac/role.yaml

@@ -94,6 +94,7 @@ rules:
 - apiGroups:
   - machine.openshift.io
   - machineconfiguration.openshift.io
+  - sriovnetwork.openshift.io
   resources:
   - '*'
   verbs: