Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support kubeflow.org/pytorchjob #995

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ test: generate gotestsum ## Run tests.
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out

.PHONY: test-integration
test-integration: manifests generate envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd ## Run tests.
test-integration: manifests generate envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \
$(GINKGO) $(GINKGO_ARGS) --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)

Expand Down Expand Up @@ -320,6 +320,12 @@ mpi-operator-crd:
mkdir -p $(PROJECT_DIR)/dep-crds/mpi-operator/
cp -f $(MPIROOT)/manifests/base/* $(PROJECT_DIR)/dep-crds/mpi-operator/

KFTRAININGROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" github.com/kubeflow/training-operator)
.PHONY: kf-training-operator-crd
kf-training-operator-crd:
mkdir -p $(PROJECT_DIR)/dep-crds/training-operator/
cp -f $(KFTRAININGROOT)/manifests/base/crds/* $(PROJECT_DIR)/dep-crds/training-operator/

RAYROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator)
.PHONY: ray-operator-crd
ray-operator-crd:
Expand All @@ -331,4 +337,3 @@ JOBSETROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" sigs.k8s.io/jobset)
jobset-operator-crd:
mkdir -p $(PROJECT_DIR)/dep-crds/jobset-operator/
cp -f $(JOBSETROOT)/config/components/crd/bases/* $(PROJECT_DIR)/dep-crds/jobset-operator/

1 change: 1 addition & 0 deletions apis/config/v1beta1/configuration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,5 +222,6 @@ type Integrations struct {
// - "kubeflow.org/mpijob"
// - "ray.io/rayjob"
// - "jobset.x-k8s.io/jobset"
// - "kubeflow.org/pytorchjob"
Frameworks []string `json:"frameworks,omitempty"`
}
27 changes: 27 additions & 0 deletions charts/kueue/templates/rbac/pytorchjob_editor_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# permissions for end users to edit pytorchjobs.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: '{{ include "kueue.fullname" . }}-pytorchjob-editor-role'
labels:
rbac.kueue.x-k8s.io/batch-admin: "true"
rbac.kueue.x-k8s.io/batch-user: "true"
rules:
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
23 changes: 23 additions & 0 deletions charts/kueue/templates/rbac/pytorchjob_viewer_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# permissions for end users to view jobs.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: '{{ include "kueue.fullname" . }}-pytorchjob-viewer-role'
labels:
rbac.kueue.x-k8s.io/batch-admin: "true"
rbac.kueue.x-k8s.io/batch-user: "true"
rules:
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- get
- list
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
17 changes: 17 additions & 0 deletions charts/kueue/templates/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,23 @@ rules:
verbs:
- get
- update
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
- update
- apiGroups:
- kueue.x-k8s.io
resources:
Expand Down
1 change: 1 addition & 0 deletions charts/kueue/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ managerConfig:
- "batch/job"
- "kubeflow.org/mpijob"
- "ray.io/rayjob"
- "kubeflow.org/pytorchjob"
# ports definition for metricsService and webhookService.
metricsService:
ports:
Expand Down
1 change: 1 addition & 0 deletions config/components/manager/controller_manager_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,4 @@ integrations:
- "kubeflow.org/mpijob"
- "ray.io/rayjob"
- "jobset.x-k8s.io/jobset"
- "kubeflow.org/pytorchjob"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same should be done in charts/kueue/values.yaml

integrations:
frameworks:
- "batch/job"
- "kubeflow.org/mpijob"
- "ray.io/rayjob"

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

2 changes: 2 additions & 0 deletions config/components/rbac/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,5 @@ resources:
- mpijob_viewer_role.yaml
- rayjob_editor_role.yaml
- rayjob_viewer_role.yaml
- pytorchjob_editor_role.yaml
- pytorchjob_viewer_role.yaml
27 changes: 27 additions & 0 deletions config/components/rbac/pytorchjob_editor_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# permissions for end users to edit pytorchjobs.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: pytorchjob-editor-role
labels:
rbac.kueue.x-k8s.io/batch-admin: "true"
rbac.kueue.x-k8s.io/batch-user: "true"
rules:
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
23 changes: 23 additions & 0 deletions config/components/rbac/pytorchjob_viewer_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# permissions for end users to view jobs.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: pytorchjob-viewer-role
labels:
rbac.kueue.x-k8s.io/batch-admin: "true"
rbac.kueue.x-k8s.io/batch-user: "true"
rules:
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- get
- list
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
17 changes: 17 additions & 0 deletions config/components/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,23 @@ rules:
verbs:
- get
- update
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- kubeflow.org
resources:
- pytorchjobs/status
verbs:
- get
- update
Comment on lines +118 to +134
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

two extra role files should be created config/components/rbac/pytorchjob_editor_role.yaml and config/components/rbac/pytorchjob_viewer_role.yaml (check the content of config/components/rbac/jobset_XXX_role.yaml ) and referenced in config/components/rbac/kustomization.yaml

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a good point. Thanks.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

- apiGroups:
- kueue.x-k8s.io
resources:
Expand Down
39 changes: 39 additions & 0 deletions config/components/webhook/manifests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,25 @@ webhooks:
resources:
- jobsets
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
service:
name: webhook-service
namespace: system
path: /mutate-kubeflow-org-v1-pytorchjob
failurePolicy: Fail
name: mpytorchjob.kb.io
rules:
- apiGroups:
- kubeflow.org
apiVersions:
- v1
operations:
- CREATE
resources:
- pytorchjobs
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
Expand Down Expand Up @@ -184,6 +203,26 @@ webhooks:
resources:
- jobsets
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
service:
name: webhook-service
namespace: system
path: /validate-kubeflow-org-v1-pytorchjob
failurePolicy: Fail
name: vpytorchjob.kb.io
rules:
- apiGroups:
- kubeflow.org
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- pytorchjobs
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/google/go-cmp v0.5.9
github.com/kubeflow/common v0.4.7
github.com/kubeflow/mpi-operator v0.4.0
github.com/kubeflow/training-operator v1.7.0-rc.0
github.com/onsi/ginkgo/v2 v2.11.0
github.com/onsi/gomega v1.27.10
github.com/open-policy-agent/cert-controller v0.8.0
Expand Down Expand Up @@ -60,6 +61,7 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ github.com/kubeflow/common v0.4.7 h1:zz6QS4k2u2FY838M/FjOtwjJq39MRZVZcvPahRYL97M
github.com/kubeflow/common v0.4.7/go.mod h1:43MAof/uhpJA2C0urynqatE3oKFQc7m2HLmJty7waqY=
github.com/kubeflow/mpi-operator v0.4.0 h1:PS4jLoMuRyrk/DHuYkI0D46sQQYpQt375HjOV4KVMFs=
github.com/kubeflow/mpi-operator v0.4.0/go.mod h1:/A4mTy/RYh2UIgaGUiXUaW70eThjsogu80WbbcZpuMg=
github.com/kubeflow/training-operator v1.7.0-rc.0 h1:FsOxEKg23XyGD2rgO06rIobdDOqtsP6R6o9ust8Jwi8=
github.com/kubeflow/training-operator v1.7.0-rc.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
Expand Down Expand Up @@ -152,6 +154,8 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230613204710-aeed3cdcbdcc h
github.com/ray-project/kuberay/ray-operator v0.0.0-20230613204710-aeed3cdcbdcc/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
Expand Down Expand Up @@ -233,6 +237,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
2 changes: 1 addition & 1 deletion main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ integrations:
{
name: "bad integrations config",
configFile: badIntegrationsConfig,
wantError: fmt.Errorf("integrations.frameworks: Unsupported value: \"unregistered/jobframework\": supported values: \"batch/job\", \"jobset.x-k8s.io/jobset\", \"kubeflow.org/mpijob\", \"ray.io/rayjob\""),
wantError: fmt.Errorf("integrations.frameworks: Unsupported value: \"unregistered/jobframework\": supported values: \"batch/job\", \"jobset.x-k8s.io/jobset\", \"kubeflow.org/mpijob\", \"kubeflow.org/pytorchjob\", \"ray.io/rayjob\""),
},
}

Expand Down
1 change: 1 addition & 0 deletions pkg/controller/jobs/jobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package jobs
import (
_ "sigs.k8s.io/kueue/pkg/controller/jobs/job"
_ "sigs.k8s.io/kueue/pkg/controller/jobs/jobset"
_ "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs"
_ "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob"
_ "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob"
)
22 changes: 22 additions & 0 deletions pkg/controller/jobs/kubeflow/jobs/jobs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
Copyright 2023 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jobs

// Reference the job framework integration packages to ensure linking.
import (
_ "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob"
)
Loading