From df7ac8e1097cfcc9a2e26f90bb9dee728301c061 Mon Sep 17 00:00:00 2001 From: Kam Kasravi Date: Wed, 4 Sep 2019 14:26:23 -0700 Subject: [PATCH 1/2] sync with kubeflow v0.6.0-rc.0-82-g489e916f (#16) --- go.mod | 14 +- go.sum | 25 +- pkg/kfapp/gcp/gcp.go | 34 +-- pkg/kfapp/gcp/gcp_test.go | 34 +-- pkg/kfapp/kustomize/kustomize.go | 188 ++------------- pkg/utils/k8utils.go | 397 ++++++++++++++++--------------- 6 files changed, 303 insertions(+), 389 deletions(-) diff --git a/go.mod b/go.mod index 6142b6ca..6de8440a 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,16 @@ module github.com/kubeflow/kfctl/v3 require ( cloud.google.com/go v0.38.0 + github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/aws/aws-sdk-go v1.15.78 github.com/cenkalti/backoff v2.1.1+incompatible + github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect github.com/deckarep/golang-set v1.7.1 + github.com/docker/docker v1.13.1 // indirect + github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect + github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fatih/camelcase v1.0.0 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-kit/kit v0.8.0 github.com/go-openapi/jsonpointer v0.19.2 // indirect @@ -17,16 +23,19 @@ require ( github.com/golang/protobuf v1.3.1 github.com/hashicorp/go-getter v1.0.2 github.com/imdario/mergo v0.3.7 + github.com/jonboulle/clockwork v0.1.0 // indirect github.com/kr/pty v1.1.3 // indirect github.com/kubeflow/kubeflow/components/profile-controller v0.0.0-20190614045418-7ca3cfb39368 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe // indirect github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/onrik/logrus v0.2.1 github.com/otiai10/copy v1.0.1 github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.2 github.com/prometheus/common v0.2.0 + github.com/russross/blackfriday v1.5.2 // indirect github.com/sirupsen/logrus v1.3.0 github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.3 @@ -41,13 +50,16 @@ require ( k8s.io/api v0.0.0-20190222213804-5cb15d344471 k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236 k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 + k8s.io/cli-runtime v0.0.0-20190228180923-a9e421a79326 k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4 - k8s.io/klog v0.3.0 // indirect + k8s.io/kubernetes v1.13.4 + k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a // indirect sigs.k8s.io/controller-runtime v0.1.12 sigs.k8s.io/kustomize v2.0.3+incompatible ) replace ( + git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 github.com/Azure/go-autorest => github.com/Azure/go-autorest v9.1.0+incompatible github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.0.5 github.com/go-openapi/jsonpointer => github.com/go-openapi/jsonpointer v0.17.0 diff --git a/go.sum b/go.sum index 3344a6b5..e854ae10 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,9 @@ dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e h1:eb0Pzkt15Bm7f2FFYv7sjY7NPFi3cPkS3tv1CcrFBWA= +github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -22,6 +23,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -33,6 +35,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 h1:HD4PLRzjuCVW79mQ0/pdsalOLHJ+FaEoqJLxfltpb2U= +github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -44,12 +48,20 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/emicklei/go-restful v2.9.0+incompatible h1:YKhDcF/NL19iSAQcyCATL1MkFXCzxfdaTiuJKr18Ank= github.com/emicklei/go-restful v2.9.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/evanphx/json-patch v4.0.0+incompatible h1:xregGRMLBeuRcwiOTHRCsPPuzCQlqhxUPbqdw+zNkLc= github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -132,6 +144,8 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -162,6 +176,8 @@ github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnG github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -209,6 +225,7 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190219184716-e4d4a2206da0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2-0.20180428102519-11635eb403ff h1:bxenFOpdnKzbA1dhcJpgiwjSw7yqvWWY6huCpmsBfv0= github.com/russross/blackfriday v1.5.2-0.20180428102519-11635eb403ff/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -395,6 +412,8 @@ k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236 h1:JfFtjaElBIg k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg= k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/cli-runtime v0.0.0-20190228180923-a9e421a79326 h1:uMY/EHQt0VHYgRMPWwPl/vQ0K0fPlt5zxTEGAdt8pDg= +k8s.io/cli-runtime v0.0.0-20190228180923-a9e421a79326/go.mod h1:qWnH3/b8sp/l7EvlDh7ulDU3UWA4P4N1NFbEEP791tM= k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4 h1:aE8wOCKuoRs2aU0OP/Rz8SXiAB0FTTku3VtGhhrkSmc= k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/code-generator v0.0.0-20190215220509-095ce2f23e83/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= @@ -404,6 +423,10 @@ k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190215190454-ea82251f3668 h1:M80qeWaBNOX2Uc4plRHcb6k+3YE5VWMaJXKZo+tX9aU= k8s.io/kube-openapi v0.0.0-20190215190454-ea82251f3668/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kubernetes v1.13.4 h1:gQqFv/pH8hlbznLXQUsi8s5zqYnv0slmUDl/yVA0EWc= +k8s.io/kubernetes v1.13.4/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a h1:uy5HAgt4Ha5rEMbhZA+aM1j2cq5LmR6LQ71EYC2sVH4= +k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/controller-runtime v0.1.12 h1:ovDq28E64PeY1yR+6H7DthakIC09soiDCrKvfP2tPYo= sigs.k8s.io/controller-runtime v0.1.12/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= sigs.k8s.io/controller-tools v0.1.9/go.mod h1:6g08p9m9G/So3sBc1AOQifHfhxH/mb6Sc4z0LMI8XMw= diff --git a/pkg/kfapp/gcp/gcp.go b/pkg/kfapp/gcp/gcp.go index 4df6e6b6..5df1f4ae 100644 --- a/pkg/kfapp/gcp/gcp.go +++ b/pkg/kfapp/gcp/gcp.go @@ -1688,31 +1688,31 @@ func generatePodDefault(group string, version string, kind string, namespace str "name": "add-gcp-secret", "namespace": namespace, }, - "desc": "add gcp credential", "spec": map[string]interface{}{ "selector": map[string]interface{}{ "matchLabels": map[string]interface{}{ "add-gcp-secret": "true", }, }, - }, - "env": []interface{}{ - map[string]interface{}{ - "name": "GOOGLE_APPLICATION_CREDENTIALS", - "value": "/secret/gcp/user-gcp-sa.json", + "desc": "add gcp credential", + "env": []interface{}{ + map[string]interface{}{ + "name": "GOOGLE_APPLICATION_CREDENTIALS", + "value": "/secret/gcp/user-gcp-sa.json", + }, }, - }, - "volumeMounts": []interface{}{ - map[string]interface{}{ - "name": "secret-volume", - "mountPath": "/secret/gcp", + "volumeMounts": []interface{}{ + map[string]interface{}{ + "name": "secret-volume", + "mountPath": "/secret/gcp", + }, }, - }, - "volumes": []interface{}{ - map[string]interface{}{ - "name": "secret-volume", - "secret": map[string]interface{}{ - "secretName": USER_SECRET_NAME, + "volumes": []interface{}{ + map[string]interface{}{ + "name": "secret-volume", + "secret": map[string]interface{}{ + "secretName": USER_SECRET_NAME, + }, }, }, }, diff --git a/pkg/kfapp/gcp/gcp_test.go b/pkg/kfapp/gcp/gcp_test.go index 79abe1f9..38c466e7 100644 --- a/pkg/kfapp/gcp/gcp_test.go +++ b/pkg/kfapp/gcp/gcp_test.go @@ -380,31 +380,31 @@ func TestGcp_setPodDefault(t *testing.T) { "name": "add-gcp-secret", "namespace": namespace, }, - "desc": "add gcp credential", "spec": map[string]interface{}{ "selector": map[string]interface{}{ "matchLabels": map[string]interface{}{ "add-gcp-secret": "true", }, }, - }, - "env": []interface{}{ - map[string]interface{}{ - "name": "GOOGLE_APPLICATION_CREDENTIALS", - "value": "/secret/gcp/user-gcp-sa.json", + "desc": "add gcp credential", + "env": []interface{}{ + map[string]interface{}{ + "name": "GOOGLE_APPLICATION_CREDENTIALS", + "value": "/secret/gcp/user-gcp-sa.json", + }, }, - }, - "volumeMounts": []interface{}{ - map[string]interface{}{ - "name": "secret-volume", - "mountPath": "/secret/gcp", + "volumeMounts": []interface{}{ + map[string]interface{}{ + "name": "secret-volume", + "mountPath": "/secret/gcp", + }, }, - }, - "volumes": []interface{}{ - map[string]interface{}{ - "name": "secret-volume", - "secret": map[string]interface{}{ - "secretName": "user-gcp-sa", + "volumes": []interface{}{ + map[string]interface{}{ + "name": "secret-volume", + "secret": map[string]interface{}{ + "secretName": "user-gcp-sa", + }, }, }, }, diff --git a/pkg/kfapp/kustomize/kustomize.go b/pkg/kfapp/kustomize/kustomize.go index c7cb47af..904d8c53 100644 --- a/pkg/kfapp/kustomize/kustomize.go +++ b/pkg/kfapp/kustomize/kustomize.go @@ -27,31 +27,22 @@ import ( kfapisv3 "github.com/kubeflow/kfctl/v3/pkg/apis" kftypesv3 "github.com/kubeflow/kfctl/v3/pkg/apis/apps" kfdefsv3 "github.com/kubeflow/kfctl/v3/pkg/apis/apps/kfdef/v1alpha1" - profilev2 "github.com/kubeflow/kubeflow/components/profile-controller/pkg/apis/kubeflow/v1alpha1" - "github.com/kubeflow/kfctl/v3/pkg/utils" + profilev2 "github.com/kubeflow/kubeflow/components/profile-controller/pkg/apis/kubeflow/v1alpha1" "github.com/otiai10/copy" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "io/ioutil" - "k8s.io/api/core/v1" rbacv2 "k8s.io/api/rbac/v1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/discovery" - "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" "os" "path" "path/filepath" - "regexp" "sigs.k8s.io/kustomize/k8sdeps" "sigs.k8s.io/kustomize/pkg/fs" "sigs.k8s.io/kustomize/pkg/image" @@ -110,7 +101,8 @@ type kustomize struct { } const ( - outputDir = "kustomize" + defaultUserId = "anonymous" + outputDir = "kustomize" ) // Setter defines an interface for modifying the plugin. @@ -252,27 +244,9 @@ func (kustomize *kustomize) initK8sClients() error { // Apply deploys kustomize generated resources to the kubenetes api server func (kustomize *kustomize) Apply(resources kftypesv3.ResourceEnum) error { - if err := kustomize.initK8sClients(); err != nil { - return &kfapisv3.KfError{ - Code: int(kfapisv3.INVALID_ARGUMENT), - Message: fmt.Sprintf("Error: kustomize plugin couldn't initialize a K8s client %v", err), - } - } - clientset := kftypesv3.GetClientset(kustomize.restConfig) - namespace := kustomize.kfDef.ObjectMeta.Namespace - log.Infof(string(kftypesv3.NAMESPACE)+": %v", namespace) - _, nsMissingErr := clientset.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}) - if nsMissingErr != nil { - log.Infof("Creating namespace: %v", namespace) - nsSpec := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, nsErr := clientset.CoreV1().Namespaces().Create(nsSpec) - if nsErr != nil { - return &kfapisv3.KfError{ - Code: int(kfapisv3.INVALID_ARGUMENT), - Message: fmt.Sprintf("couldn't create %v %v Error: %v", - string(kftypesv3.NAMESPACE), namespace, nsErr), - } - } + apply, err := utils.NewApply(kustomize.kfDef.ObjectMeta.Namespace) + if err != nil { + return err } kustomizeDir := path.Join(kustomize.kfDef.Spec.AppDir, outputDir) @@ -285,6 +259,7 @@ func (kustomize *kustomize) Apply(resources kftypesv3.ResourceEnum) error { Message: fmt.Sprintf("error evaluating kustomization manifest for %v Error %v", app.Name, err), } } + //TODO this should be streamed data, err := resMap.EncodeAsYaml() if err != nil { return &kfapisv3.KfError{ @@ -292,18 +267,21 @@ func (kustomize *kustomize) Apply(resources kftypesv3.ResourceEnum) error { Message: fmt.Sprintf("can not encode component %v as yaml Error %v", app.Name, err), } } - resourcesErr := kustomize.deployResources(kustomize.restConfig, data) - if resourcesErr != nil { - return &kfapisv3.KfError{ - Code: int(kfapisv3.INTERNAL_ERROR), - Message: fmt.Sprintf("couldn't create resources from %v Error: %v", app.Name, resourcesErr), - } + err = apply.Apply(data) + if err != nil { + return err } } + // Create default profile + // When user identity available, the user will be owner of the profile + // Otherwise the profile would be a public one. if kustomize.kfDef.Spec.Email != "" { - // Profile name is also the namespace created. - defaultProfileNamespace := kftypesv3.EmailToDefaultName(kustomize.kfDef.Spec.Email) + userId := defaultUserId + // Use user email as user id if available. + // When platform == GCP, same user email is also identity in requests through IAP. + userId = kustomize.kfDef.Spec.Email + defaultProfileNamespace := kftypesv3.EmailToDefaultName(userId) profile := &profilev2.Profile{ TypeMeta: metav1.TypeMeta{ Kind: "Profile", @@ -315,31 +293,27 @@ func (kustomize *kustomize) Apply(resources kftypesv3.ResourceEnum) error { Spec: profilev2.ProfileSpec{ Owner: rbacv2.Subject{ Kind: "User", - Name: kustomize.kfDef.Spec.Email, + Name: userId, }, }, } - _, nsMissingErr := clientset.CoreV1().Namespaces().Get(defaultProfileNamespace, metav1.GetOptions{}) - if nsMissingErr != nil { + + if !apply.DefaultProfileNamespace(defaultProfileNamespace) { body, err := json.Marshal(profile) if err != nil { return err } - resourcesErr := kustomize.deployResources(kustomize.restConfig, body) - if resourcesErr != nil { - return &kfapisv3.KfError{ - Code: int(kfapisv3.INTERNAL_ERROR), - Message: fmt.Sprintf("couldn't create default profile from %v Error: %v", profile, resourcesErr), - } + err = apply.Apply(body) + if err != nil { + return err } b := backoff.NewExponentialBackOff() b.InitialInterval = 3 * time.Second b.MaxInterval = 30 * time.Second b.MaxElapsedTime = 5 * time.Minute return backoff.Retry(func() error { - _, nsErr := clientset.CoreV1().Namespaces().Get(defaultProfileNamespace, metav1.GetOptions{}) - if nsErr != nil { - msg := fmt.Sprintf("Could not find namespace %v, wait and retry: %v", defaultProfileNamespace, nsErr) + if !apply.DefaultProfileNamespace(defaultProfileNamespace) { + msg := fmt.Sprintf("Could not find namespace %v, wait and retry", defaultProfileNamespace) log.Warnf(msg) return &kfapisv3.KfError{ Code: int(kfapisv3.INVALID_ARGUMENT), @@ -356,116 +330,6 @@ func (kustomize *kustomize) Apply(resources kftypesv3.ResourceEnum) error { return nil } -// deployResourcesFromFile creates resources from a file, just like `kubectl create -f filename` -// TODO based on bootstrap/v2/app/k8sUtil.go. Need to merge. -// TODO: it can't handle "kind: list" yet. -func (kustomize *kustomize) deployResourcesFromFile(config *rest.Config, filename string) error { - data, err := ioutil.ReadFile(filename) - if err != nil { - return err - } - return kustomize.deployResources(config, data) -} - -// deployResources creates resources with byte array. -func (kustomize *kustomize) deployResources(config *rest.Config, data []byte) error { - // Create a restmapper to determine the resource type. - _discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) - if err != nil { - return err - } - _cached := cached.NewMemCacheClient(_discoveryClient) - _cached.Invalidate() - mapper := restmapper.NewDeferredDiscoveryRESTMapper(_cached) - - splitter := regexp.MustCompile(kftypesv3.YamlSeparator) - objects := splitter.Split(string(data), -1) - - for _, object := range objects { - var o map[string]interface{} - if err = yaml.Unmarshal([]byte(object), &o); err != nil { - return err - } - a := o["apiVersion"] - if a == nil { - log.Warnf("Unknown resource: %v", object) - continue - } - apiVersion := strings.Split(a.(string), "/") - var group, version string - if len(apiVersion) == 1 { - // core v1, no group. e.g. namespace - group, version = "", apiVersion[0] - } else { - group, version = apiVersion[0], apiVersion[1] - } - metadata := o["metadata"].(map[string]interface{}) - var namespace string - if metadata["namespace"] != nil { - namespace = metadata["namespace"].(string) - } else { - namespace = "" - } - kind := o["kind"].(string) - gk := schema.GroupKind{ - Group: group, - Kind: kind, - } - mapping, retryErr := mapper.RESTMapping(gk, version) - if retryErr != nil { - return retryErr - } - // build config for restClient - c := rest.CopyConfig(config) - c.GroupVersion = &schema.GroupVersion{ - Group: group, - Version: version, - } - c.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} - if group == "" { - c.APIPath = "/api" - } else { - c.APIPath = "/apis" - } - restClient, err := rest.RESTClientFor(c) - if err != nil { - return err - } - - // build the request - if metadata["name"] != nil { - name := metadata["name"].(string) - log.Infof("creating %v/%v\n", kind, name) - body, err := json.Marshal(o) - if err != nil { - return err - } - - request := restClient.Post().Resource(mapping.Resource.Resource).Body(body) - if mapping.Scope.Name() == "namespace" { - request = request.Namespace(namespace) - } - result := request.Do() - if result.Error() != nil { - statusCode := 200 - result.StatusCode(&statusCode) - switch statusCode { - case 200: - fallthrough - case 409: - continue - default: - return result.Error() - } - } - } else { - log.Warnf("object with kind %v has no name\n", metadata["kind"]) - } - - } - return nil -} - // deleteGlobalResources is called from Delete and deletes CRDs, ClusterRoles, ClusterRoleBindings func (kustomize *kustomize) deleteGlobalResources() error { if err := kustomize.initK8sClients(); err != nil { diff --git a/pkg/utils/k8utils.go b/pkg/utils/k8utils.go index 2f6a661b..89d2c585 100644 --- a/pkg/utils/k8utils.go +++ b/pkg/utils/k8utils.go @@ -16,21 +16,26 @@ package utils import ( "context" "fmt" - "github.com/cenkalti/backoff" "github.com/ghodss/yaml" configtypes "github.com/kubeflow/kfctl/v3/config" kfapis "github.com/kubeflow/kfctl/v3/pkg/apis" + kftypes "github.com/kubeflow/kfctl/v3/pkg/apis/apps" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "io/ioutil" + "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericclioptions/printers" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + kubectlapply "k8s.io/kubernetes/pkg/kubectl/cmd/apply" + kubectldelete "k8s.io/kubernetes/pkg/kubectl/cmd/delete" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "os" "regexp" "sigs.k8s.io/controller-runtime/pkg/client" "strings" @@ -40,195 +45,10 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" ) -// RecommendedConfigPathEnvVar is a environment variable for path configuration -const RecommendedConfigPathEnvVar = "KUBECONFIG" - const ( - maxRetries = 5 - backoffInterval = 5 * time.Second - YamlSeparator = "(?m)^---[ \t]*$" + YamlSeparator = "(?m)^---[ \t]*$" ) -func getRESTClient(config *rest.Config, group string, version string) (*rest.RESTClient, error) { - c := rest.CopyConfig(config) - c.GroupVersion = &schema.GroupVersion{ - Group: group, - Version: version, - } - c.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} - if group == "" { - c.APIPath = "/api" - } else { - c.APIPath = "/apis" - } - return rest.RESTClientFor(c) -} - -// TODO: Might need to return response if needed. -func getResource(mapping *meta.RESTMapping, config *rest.Config, group string, - version string, namespace string, name string) error { - restClient, err := getRESTClient(config, group, version) - if err != nil { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("getResource error: %v", err), - } - } - - if _, err = restClient. - Get(). - Resource(mapping.Resource.Resource). - NamespaceIfScoped(namespace, mapping.Scope.Name() == "namespace"). - Name(name). - Do(). - Get(); err == nil { - return nil - } else { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("getResource error: %v", err), - } - } -} - -// TODO(#2391): kubectl is hard to be used as library - it's deeply integrated with -// Cobra. Currently using RESTClient with `kubectl create` has some issues with YAML -// generated with `ks show`. -func patchResource(mapping *meta.RESTMapping, config *rest.Config, group string, - version string, namespace string, data []byte) error { - restClient, err := getRESTClient(config, group, version) - if err != nil { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("patchResource error: %v", err), - } - } - - if _, err = restClient. - Patch(k8stypes.JSONPatchType). - Resource(mapping.Resource.Resource). - NamespaceIfScoped(namespace, mapping.Scope.Name() == "namespace"). - Body(data). - Do(). - Get(); err == nil { - return nil - } else { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("patchResource error: %v", err), - } - } -} - -func deleteResource(mapping *meta.RESTMapping, config *rest.Config, group string, - version string, namespace string, name string) error { - restClient, err := getRESTClient(config, group, version) - if err != nil { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("deleteResource error: %v", err), - } - } - - _, err = restClient. - Delete(). - Resource(mapping.Resource.Resource). - NamespaceIfScoped(namespace, mapping.Scope.Name() == "namespace"). - Name(name). - Do(). - Get() - - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } else { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("Resource deletion error: %v", err), - } - } - } - - return backoff.Retry(func() error { - getErr := getResource(mapping, config, group, version, namespace, name) - if k8serrors.IsNotFound(getErr) { - log.Infof("%v in namespace %v is deleted ...", name, namespace) - return nil - } else { - msg := fmt.Sprintf("%v in namespace %v is being deleted ...", - name, namespace) - if getErr != nil { - msg = msg + getErr.Error() - } - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: msg, - } - } - }, backoff.NewExponentialBackOff()) -} - -func createResource(mapping *meta.RESTMapping, config *rest.Config, group string, - version string, namespace string, data []byte) error { - restClient, err := getRESTClient(config, group, version) - if err != nil { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("createResource error: %v", err), - } - } - - if _, err = restClient. - Post(). - Resource(mapping.Resource.Resource). - NamespaceIfScoped(namespace, mapping.Scope.Name() == "namespace"). - Body(data). - Do(). - Get(); err == nil { - return nil - } else { - return &kfapis.KfError{ - Code: int(kfapis.INVALID_ARGUMENT), - Message: fmt.Sprintf("createResource error: %v", err), - } - } -} - -// TODO(#2585): Should try to have 3 way merge functionality. -func patchOrCreate(mapping *meta.RESTMapping, config *rest.Config, group string, - version string, namespace string, name string, data []byte) error { - log.Infof("Applying resource configuration for %v", name) - err := getResource(mapping, config, group, version, namespace, name) - if err != nil { - log.Infof("getResource error, treating as not found: %v", err) - err = createResource(mapping, config, group, version, namespace, data) - } else { - log.Infof("getResource succeeds, treating as found.") - err = patchResource(mapping, config, group, version, namespace, data) - } - - for i := 1; i < maxRetries && k8serrors.IsConflict(err); i++ { - time.Sleep(backoffInterval) - - log.Infof("Retrying patchOrCreate at %v attempt ...", i) - err = getResource(mapping, config, group, version, namespace, name) - if err != nil { - return err - } - err = patchResource(mapping, config, group, version, namespace, data) - } - - if err != nil && (k8serrors.IsConflict(err) || k8serrors.IsInvalid(err) || - k8serrors.IsMethodNotSupported(err)) { - log.Infof("Trying delete and create as last resort ...") - if err = deleteResource(mapping, config, group, version, namespace, name); err != nil { - return err - } - err = createResource(mapping, config, group, version, namespace, data) - } - return err -} - func CreateResourceFromFile(config *rest.Config, filename string, elems ...configtypes.NameValue) error { elemsMap := make(map[string]configtypes.NameValue) for _, nv := range elems { @@ -325,3 +145,198 @@ func DeleteResourceFromFile(config *rest.Config, filename string) error { } return nil } + +type Apply struct { + matchVersionKubeConfigFlags *cmdutil.MatchVersionFlags + factory cmdutil.Factory + clientset *kubernetes.Clientset + options *kubectlapply.ApplyOptions + tmpfile *os.File + stdin *os.File +} + +func NewApply(namespace string) (*Apply, error) { + apply := &Apply{ + matchVersionKubeConfigFlags: cmdutil.NewMatchVersionFlags(genericclioptions.NewConfigFlags()), + } + apply.factory = cmdutil.NewFactory(apply.matchVersionKubeConfigFlags) + clientset, err := apply.factory.KubernetesClientSet() + if err != nil { + return nil, &kfapis.KfError{ + Code: int(kfapis.INTERNAL_ERROR), + Message: fmt.Sprintf("could not get clientset Error %v", err), + } + } + apply.clientset = clientset + err = apply.namespace(namespace) + if err != nil { + return nil, err + } + return apply, nil +} + +func (a *Apply) DefaultProfileNamespace(name string) bool { + _, nsMissingErr := a.clientset.CoreV1().Namespaces().Get(name, metav1.GetOptions{}) + if nsMissingErr != nil { + return false + } + return true +} + +func (a *Apply) Apply(data []byte) error { + a.tmpfile = a.tempFile(data) + a.stdin = os.Stdin + os.Stdin = a.tmpfile + defer a.cleanup() + ioStreams := genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} + a.options = kubectlapply.NewApplyOptions(ioStreams) + a.options.DeleteFlags = a.deleteFlags("that contains the configuration to apply") + initializeErr := a.init() + if initializeErr != nil { + return &kfapis.KfError{ + Code: int(kfapis.INTERNAL_ERROR), + Message: fmt.Sprintf("could not initialize Error %v", initializeErr), + } + } + err := a.run() + if err != nil { + return err + } + return nil +} + +func (a *Apply) run() error { + resourcesErr := a.options.Run() + if resourcesErr != nil { + cmdutil.CheckErr(resourcesErr) + return &kfapis.KfError{ + Code: int(kfapis.INTERNAL_ERROR), + Message: fmt.Sprintf("Apply.Run Error %v", resourcesErr), + } + } + return nil +} + +func (a *Apply) cleanup() error { + os.Stdin = a.stdin + if a.tmpfile != nil { + if err := a.tmpfile.Close(); err != nil { + return err + } + return os.Remove(a.tmpfile.Name()) + } + return nil +} + +func (a *Apply) init() error { + var err error + var o = a.options + var f = a.factory + // allow for a success message operation to be specified at print time + o.ToPrinter = func(operation string) (printers.ResourcePrinter, error) { + o.PrintFlags.NamePrintFlags.Operation = operation + if o.DryRun { + err = o.PrintFlags.Complete("%s (dry run)") + if err != nil { + return nil, err + } + } + if o.ServerDryRun { + err = o.PrintFlags.Complete("%s (server dry run)") + if err != nil { + return nil, err + } + } + return o.PrintFlags.ToPrinter() + } + o.DiscoveryClient, err = f.ToDiscoveryClient() + if err != nil { + return err + } + dynamicClient, err := f.DynamicClient() + if err != nil { + return err + } + o.DeleteOptions = o.DeleteFlags.ToOptions(dynamicClient, o.IOStreams) + o.ShouldIncludeUninitialized = false + o.OpenApiPatch = true + o.OpenAPISchema, _ = f.OpenAPISchema() + o.Validator, err = f.Validator(false) + o.Builder = f.NewBuilder() + o.Mapper, err = f.ToRESTMapper() + if err != nil { + return err + } + o.DynamicClient, err = f.DynamicClient() + if err != nil { + return err + } + o.Namespace, o.EnforceNamespace, err = f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + return nil +} + +func (a *Apply) namespace(namespace string) error { + log.Infof(string(kftypes.NAMESPACE)+": %v", namespace) + _, nsMissingErr := a.clientset.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}) + if nsMissingErr != nil { + log.Infof("Creating namespace: %v", namespace) + nsSpec := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + _, nsErr := a.clientset.CoreV1().Namespaces().Create(nsSpec) + if nsErr != nil { + return &kfapis.KfError{ + Code: int(kfapis.INVALID_ARGUMENT), + Message: fmt.Sprintf("couldn't create %v %v Error: %v", + string(kftypes.NAMESPACE), namespace, nsErr), + } + } + } + return nil +} + +func (a *Apply) tempFile(data []byte) *os.File { + tmpfile, err := ioutil.TempFile("/tmp", "kout") + if err != nil { + log.Fatal(err) + } + if _, err := tmpfile.Write(data); err != nil { + log.Fatal(err) + } + if _, err := tmpfile.Seek(0, 0); err != nil { + log.Fatal(err) + } + return tmpfile +} + +func (a *Apply) deleteFlags(usage string) *kubectldelete.DeleteFlags { + cascade := true + gracePeriod := -1 + // setup command defaults + all := false + force := false + ignoreNotFound := false + now := false + output := "" + labelSelector := "" + fieldSelector := "" + timeout := time.Duration(0) + wait := true + filenames := []string{a.tmpfile.Name()} + recursive := false + return &kubectldelete.DeleteFlags{ + FileNameFlags: &genericclioptions.FileNameFlags{Usage: usage, Filenames: &filenames, Recursive: &recursive}, + LabelSelector: &labelSelector, + FieldSelector: &fieldSelector, + Cascade: &cascade, + GracePeriod: &gracePeriod, + All: &all, + Force: &force, + IgnoreNotFound: &ignoreNotFound, + Now: &now, + Timeout: &timeout, + Wait: &wait, + Output: &output, + } +} From 45bf84175517794491934bda10d7a88d4e1e34c4 Mon Sep 17 00:00:00 2001 From: Kam Kasravi Date: Mon, 9 Sep 2019 05:51:18 -0700 Subject: [PATCH 2/2] prow fix to run in the right repo (#17) * prow fix to run in the right repo * point to kubeflow/kubeflow for config * add kubeflow/kubeflow as required repo * change to testing/e2e * add deploy_utils.py * relocating util scripts * leave srcDir pointing to kubeflow/kubeflow but change kubeflowPy * set kubeflowPy back go srcDir * expand PYTHONPATH * add semi-colon --- pkg/kfapp/kustomize/kustomize.go | 12 ++ testing/deploy_utils.py | 202 ++++++++++++++++++ testing/e2e/conftest.py | 76 +++++++ testing/e2e/endpoint_ready_test.py | 36 ++++ testing/e2e/kf_is_ready_test.py | 104 +++++++++ testing/e2e/kfctl_delete_test.py | 74 +++++++ testing/e2e/kfctl_go_test.py | 158 ++++++++++++++ testing/vm_util.py | 128 +++++++++++ .../components/kfctl_go_test.jsonnet | 22 +- .../components/kubeflow_workflow.libsonnet | 10 +- testing/workflows/run.sh | 27 +++ 11 files changed, 834 insertions(+), 15 deletions(-) create mode 100644 testing/deploy_utils.py create mode 100644 testing/e2e/conftest.py create mode 100644 testing/e2e/endpoint_ready_test.py create mode 100644 testing/e2e/kf_is_ready_test.py create mode 100644 testing/e2e/kfctl_delete_test.py create mode 100644 testing/e2e/kfctl_go_test.py create mode 100644 testing/vm_util.py create mode 100755 testing/workflows/run.sh diff --git a/pkg/kfapp/kustomize/kustomize.go b/pkg/kfapp/kustomize/kustomize.go index 904d8c53..2b445bfd 100644 --- a/pkg/kfapp/kustomize/kustomize.go +++ b/pkg/kfapp/kustomize/kustomize.go @@ -18,6 +18,7 @@ package kustomize import ( "bufio" + "encoding/hex" "encoding/json" "fmt" "github.com/cenkalti/backoff" @@ -40,6 +41,7 @@ import ( corev1 "k8s.io/client-go/kubernetes/typed/core/v1" rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/rest" + "math/rand" "os" "path" "path/filepath" @@ -671,6 +673,16 @@ func MergeKustomization(compDir string, targetDir string, kfDef *kfdefsv3.KfDef, for i, param := range params { paramName := strings.Split(param, "=")[0] if val, ok := paramMap[paramName]; ok && val != "" { + switch paramName { + case "generateName": + arr := strings.Split(param, "=") + if len(arr) == 1 || arr[1] == "" { + b := make([]byte, 4) //equals 8 charachters + rand.Read(b) + s := hex.EncodeToString(b) + val += s + } + } params[i] = paramName + "=" + val } else { switch paramName { diff --git a/testing/deploy_utils.py b/testing/deploy_utils.py new file mode 100644 index 00000000..a50a2478 --- /dev/null +++ b/testing/deploy_utils.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +import argparse +import datetime +import json +import logging +import os +import shutil +import ssl +import tempfile +import time +import uuid + +import requests +import yaml +from googleapiclient import discovery, errors +from kubernetes import client as k8s_client +from kubernetes.client import rest +from kubernetes.config import kube_config +from oauth2client.client import GoogleCredentials + +from kubeflow.testing import test_util, util # pylint: disable=no-name-in-module # noqa: E501 +from testing import vm_util + + +def get_gcp_identity(): + identity = util.run(["gcloud", "config", "get-value", "account"]) + logging.info("Current GCP account: %s", identity) + return identity + + +def create_k8s_client(): + # We need to load the kube config so that we can have credentials to + # talk to the APIServer. + util.load_kube_config(persist_config=False) + + # Create an API client object to talk to the K8s master. + api_client = k8s_client.ApiClient() + + return api_client + + +def _setup_test(api_client, run_label): + """Create the namespace for the test. + + Returns: + test_dir: The local test directory. + """ + + api = k8s_client.CoreV1Api(api_client) + namespace = k8s_client.V1Namespace() + namespace.api_version = "v1" + namespace.kind = "Namespace" + namespace.metadata = k8s_client.V1ObjectMeta( + name=run_label, labels={ + "app": "kubeflow-e2e-test", + }) + + try: + logging.info("Creating namespace %s", namespace.metadata.name) + namespace = api.create_namespace(namespace) + logging.info("Namespace %s created.", namespace.metadata.name) + except rest.ApiException as e: + if e.status == 409: + logging.info("Namespace %s already exists.", namespace.metadata.name) + else: + raise + + return namespace + + +def setup_kubeflow_ks_app(dir, namespace, github_token, api_client): + """Create a ksonnet app for Kubeflow""" + util.makedirs(dir) + + logging.info("Using test directory: %s", dir) + + namespace_name = namespace + + namespace = _setup_test(api_client, namespace_name) + logging.info("Using namespace: %s", namespace) + if github_token: + logging.info("Setting GITHUB_TOKEN to %s.", github_token) + # Set a GITHUB_TOKEN so that we don't rate limited by GitHub; + # see: https://github.com/ksonnet/ksonnet/issues/233 + os.environ["GITHUB_TOKEN"] = github_token + + if not os.getenv("GITHUB_TOKEN"): + logging.warning("GITHUB_TOKEN not set; you will probably hit Github API " + "limits.") + # Initialize a ksonnet app. + app_name = "kubeflow-test-" + uuid.uuid4().hex[0:4] + util.run([ + "ks", + "init", + app_name, + ], cwd=dir) + + app_dir = os.path.join(dir, app_name) + + # Set the default namespace. + util.run(["ks", "env", "set", "default", "--namespace=" + namespace_name], + cwd=app_dir) + + kubeflow_registry = "github.com/kubeflow/kubeflow/tree/master/kubeflow" + util.run(["ks", "registry", "add", "kubeflow", kubeflow_registry], + cwd=app_dir) + + # Install required packages + packages = [ + "kubeflow/common", "kubeflow/gcp", "kubeflow/jupyter", + "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/tf-training", + "kubeflow/pytorch-job", "kubeflow/argo", "kubeflow/katib" + ] + + # Instead of installing packages we edit the app.yaml file directly for p in + # packages: + # util.run(["ks", "pkg", "install", p], cwd=app_dir) + app_file = os.path.join(app_dir, "app.yaml") + with open(app_file) as f: + app_yaml = yaml.load(f) + + libraries = {} + for pkg in packages: + pkg = pkg.split("/")[1] + libraries[pkg] = { + 'gitVersion': { + 'commitSha': 'fake', + 'refSpec': 'fake' + }, + 'name': pkg, + 'registry': "kubeflow" + } + app_yaml['libraries'] = libraries + + with open(app_file, "w") as f: + yaml.dump(app_yaml, f) + + # Create vendor directory with a symlink to the src so that we use the code + # at the desired commit. + target_dir = os.path.join(app_dir, "vendor", "kubeflow") + + REPO_ORG = "kubeflow" + REPO_NAME = "kubeflow" + REGISTRY_PATH = "kubeflow" + source = os.path.join(dir, "src", REPO_ORG, REPO_NAME, REGISTRY_PATH) + logging.info("Creating link %s -> %s", target_dir, source) + os.symlink(source, target_dir) + + return app_dir + + +def log_operation_status(operation): + """A callback to use with wait_for_operation.""" + name = operation.get("name", "") + status = operation.get("status", "") + logging.info("Operation %s status %s", name, status) + + +def wait_for_operation(client, + project, + op_id, + timeout=datetime.timedelta(hours=1), + polling_interval=datetime.timedelta(seconds=5), + status_callback=log_operation_status): + """Wait for the specified operation to complete. + + Args: + client: Client for the API that owns the operation. + project: project + op_id: Operation id. + timeout: A datetime.timedelta expressing the amount of time to wait before + giving up. + polling_interval: A datetime.timedelta to represent the amount of time to + wait between requests polling for the operation status. + + Returns: + op: The final operation. + + Raises: + TimeoutError: if we timeout waiting for the operation to complete. + """ + endtime = datetime.datetime.now() + timeout + while True: + try: + op = client.operations().get(project=project, operation=op_id).execute() + + if status_callback: + status_callback(op) + + status = op.get("status", "") + # Need to handle other status's + if status == "DONE": + return op + except ssl.SSLError as e: + logging.error("Ignoring error %s", e) + if datetime.datetime.now() > endtime: + raise TimeoutError( + "Timed out waiting for op: {0} to complete.".format(op_id)) + time.sleep(polling_interval.total_seconds()) + + # Linter complains if we don't have a return here even though its unreachable + return None diff --git a/testing/e2e/conftest.py b/testing/e2e/conftest.py new file mode 100644 index 00000000..fbc70d55 --- /dev/null +++ b/testing/e2e/conftest.py @@ -0,0 +1,76 @@ +import pytest + +def pytest_addoption(parser): + parser.addoption( + "--app_path", action="store", default="", + help="Path where the KF application should be stored") + + parser.addoption( + "--app_name", action="store", default="", + help="Name of the KF application") + + parser.addoption( + "--kfctl_path", action="store", default="", + help="Path to kfctl.") + + parser.addoption( + "--namespace", action="store", default="kubeflow", + help="Namespace to use.") + + parser.addoption( + "--project", action="store", default="kubeflow-ci-deployment", + help="GCP project to deploy Kubeflow to") + + parser.addoption( + "--config_path", action="store", default="", + help="The config to use for kfctl init") + + parser.addoption( + "--use_basic_auth", action="store", default="False", + help="Use basic auth.") + + parser.addoption( + "--use_istio", action="store", default="False", + help="Use istio.") + +@pytest.fixture +def app_path(request): + return request.config.getoption("--app_path") + +@pytest.fixture +def app_name(request): + return request.config.getoption("--app_name") + +@pytest.fixture +def kfctl_path(request): + return request.config.getoption("--kfctl_path") + +@pytest.fixture +def namespace(request): + return request.config.getoption("--namespace") + +@pytest.fixture +def project(request): + return request.config.getoption("--project") + +@pytest.fixture +def config_path(request): + return request.config.getoption("--config_path") + +@pytest.fixture +def use_basic_auth(request): + value = request.config.getoption("--use_basic_auth").lower() + + if value in ["t", "true"]: + return True + else: + return False + +@pytest.fixture +def use_istio(request): + value = request.config.getoption("--use_istio").lower() + + if value in ["t", "true"]: + return True + else: + return False \ No newline at end of file diff --git a/testing/e2e/endpoint_ready_test.py b/testing/e2e/endpoint_ready_test.py new file mode 100644 index 00000000..96c9a346 --- /dev/null +++ b/testing/e2e/endpoint_ready_test.py @@ -0,0 +1,36 @@ +import datetime +import logging +import os +import subprocess +import tempfile +import uuid +from retrying import retry + +import pytest + +from kubeflow.testing import util +from testing import deploy_utils +from testing import gcp_util + +def test_endpoint_is_ready(project, app_name): + """Test that Kubeflow was successfully deployed. + + Args: + project: The gcp project that we deployed kubeflow + app_name: The name of the kubeflow deployment + """ + # Owned by project kubeflow-ci-deployment. + os.environ["CLIENT_ID"] = "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g.apps.googleusercontent.com" + if not gcp_util.endpoint_is_ready( + "https://{}.endpoints.{}.cloud.goog".format(app_name, project), + wait_min=25): + raise Exception("Endpoint not ready") + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format=('%(levelname)s|%(asctime)s' + '|%(pathname)s|%(lineno)d| %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S', + ) + logging.getLogger().setLevel(logging.INFO) + pytest.main() diff --git a/testing/e2e/kf_is_ready_test.py b/testing/e2e/kf_is_ready_test.py new file mode 100644 index 00000000..9762979b --- /dev/null +++ b/testing/e2e/kf_is_ready_test.py @@ -0,0 +1,104 @@ +import datetime +import logging +import os +import subprocess +import tempfile +import uuid +from retrying import retry + +import pytest + +from kubeflow.testing import util +from testing import deploy_utils + +def test_kf_is_ready(namespace, use_basic_auth, use_istio): + """Test that Kubeflow was successfully deployed. + + Args: + namespace: The namespace Kubeflow is deployed to. + """ + + logging.info("Using namespace %s", namespace) + + # Need to activate account for scopes. + if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"): + util.run(["gcloud", "auth", "activate-service-account", + "--key-file=" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]]) + + api_client = deploy_utils.create_k8s_client() + + util.load_kube_config() + + # Verify that components are actually deployed. + # TODO(jlewi): We need to parameterize this list based on whether + # we are using IAP or basic auth. + deployment_names = [ + "argo-ui", + "centraldashboard", + "cloud-endpoints-controller", + "jupyter-web-app-deployment", + "metadata-db", + "metadata-deployment", + "metadata-ui", + "ml-pipeline", + "ml-pipeline-scheduledworkflow", + "ml-pipeline-ui", + "notebook-controller-deployment", + "tf-job-operator", + "pytorch-operator", + "katib-controller", + "workflow-controller", + ] + + stateful_set_names = [ + "kfserving-controller-manager", + ] + + ingress_related_deployments = [] + ingress_related_stateful_sets = [] + + if use_basic_auth: + deployment_names.extend(["basic-auth-login"]) + ingress_related_stateful_sets.extend(["backend-updater"]) + else: + ingress_related_deployments.extend(["iap-enabler"]) + ingress_related_stateful_sets.extend(["backend-updater"]) + + # TODO(jlewi): Might want to parallelize this. + for deployment_name in deployment_names: + logging.info("Verifying that deployment %s started...", deployment_name) + util.wait_for_deployment(api_client, namespace, deployment_name, 10) + + for stateful_set_name in stateful_set_names: + logging.info("Verifying that stateful set %s started...", stateful_set_name) + util.wait_for_statefulset(api_client, namespace, stateful_set_name) + + ingress_namespace = "istio-system" if use_istio else namespace + for deployment_name in ingress_related_deployments: + logging.info("Verifying that deployment %s started...", deployment_name) + util.wait_for_deployment(api_client, ingress_namespace, deployment_name, 10) + + for name in ingress_related_stateful_sets: + logging.info("Verifying that statefulset %s started...", name) + util.wait_for_statefulset(api_client, ingress_namespace, name) + + # TODO(jlewi): We should verify that the ingress is created and healthy. + + knative_namespace = "knative-serving" + knative_related_deployments = [ + "activator", + "autoscaler", + "controller", + ] + for deployment_name in knative_related_deployments: + logging.info("Verifying that deployment %s started...", deployment_name) + util.wait_for_deployment(api_client, knative_namespace, deployment_name, 10) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format=('%(levelname)s|%(asctime)s' + '|%(pathname)s|%(lineno)d| %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S', + ) + logging.getLogger().setLevel(logging.INFO) + pytest.main() diff --git a/testing/e2e/kfctl_delete_test.py b/testing/e2e/kfctl_delete_test.py new file mode 100644 index 00000000..91ce0a9b --- /dev/null +++ b/testing/e2e/kfctl_delete_test.py @@ -0,0 +1,74 @@ +"""Run kfctl delete as a pytest. + +We use this in order to generate a junit_xml file. +""" +import datetime +import logging +import os +import subprocess +import tempfile +import uuid +from retrying import retry + +import pytest + +from kubeflow.testing import util +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +# TODO(gabrielwen): Move this to a separate test "kfctl_go_check_post_delete" +def get_endpoints_list(project): + cred = GoogleCredentials.get_application_default() + services_mgt = discovery.build('servicemanagement', 'v1', credentials=cred) + services = services_mgt.services() + next_page_token = None + endpoints = [] + + while True: + results = services.list(producerProjectId=project, + pageToken=next_page_token).execute() + + for s in results.get("services", {}): + name = s.get("serviceName", "") + endpoints.append(name) + if not "nextPageToken" in results: + break + next_page_token = results["nextPageToken"] + + return endpoints + +def test_kfctl_delete(kfctl_path, app_path, project): + if not kfctl_path: + raise ValueError("kfctl_path is required") + + if not app_path: + raise ValueError("app_path is required") + + logging.info("Using kfctl path %s", kfctl_path) + logging.info("Using app path %s", app_path) + + util.run([kfctl_path, "delete", "all", "--delete_storage", "-V"], + cwd=app_path) + + # Use services.list instead of services.get because error returned is not + # 404, it's 403 which is confusing. + name = os.path.basename(app_path) + endpoint_name = "{deployment}.endpoints.{project}.cloud.goog".format( + deployment=name, + project=project) + logging.info("Verify endpoint service is deleted: " + endpoint_name) + if endpoint_name in get_endpoints_list(project): + msg = "Endpoint is not deleted: " + endpoint_name + logging.error(msg) + raise AssertionError(msg) + else: + logging.info("Verified endpoint service is deleted.") + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format=('%(levelname)s|%(asctime)s' + '|%(pathname)s|%(lineno)d| %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S', + ) + logging.getLogger().setLevel(logging.INFO) + pytest.main() diff --git a/testing/e2e/kfctl_go_test.py b/testing/e2e/kfctl_go_test.py new file mode 100644 index 00000000..18afc5fa --- /dev/null +++ b/testing/e2e/kfctl_go_test.py @@ -0,0 +1,158 @@ +import datetime +import logging +import os +import subprocess +import tempfile +import uuid +from retrying import retry +import yaml + +import pytest + +from kubeflow.testing import util + + +# retry 4 times, waiting 3 minutes between retries +@retry(stop_max_attempt_number=4, wait_fixed=180000) +def run_with_retries(*args, **kwargs): + util.run(*args, **kwargs) + + +def verify_kubeconfig(project, zone, app_path): + name = os.path.basename(app_path) + context = util.run(["kubectl", "config", "current-context"]).strip() + if name == context: + logging.info("KUBECONFIG current context name matches app name: " + name) + else: + msg = "KUBECONFIG not having expected context: {expected} v.s. {actual}".format( + expected=name, actual=context) + logging.error(msg) + raise RuntimeError(msg) + + +def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_path): + """Test building and deploying Kubeflow. + + Args: + app_path: The path to the Kubeflow app. + project: The GCP project to use. + """ + if not app_path: + logging.info("--app_path not specified") + stamp = datetime.datetime.now().strftime("%H%M") + parent_dir = tempfile.gettempdir() + app_path = os.path.join( + parent_dir, "kfctl-{0}-{1}".format(stamp, + uuid.uuid4().hex[0:4])) + else: + parent_dir = os.path.dirname(app_path) + + logging.info("Using app path %s", app_path) + this_dir = os.path.dirname(__file__) + build_dir = os.path.abspath(os.path.join(this_dir, "..", "..")) + zone = 'us-central1-a' + + # Need to activate account for scopes. + if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"): + util.run([ + "gcloud", "auth", "activate-service-account", + "--key-file=" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] + ]) + + # We need to use retry builds because when building in the test cluster + # we see intermittent failures pulling dependencies + run_with_retries(["make", "build-kfctl"], cwd=build_dir) + kfctl_path = os.path.join(build_dir, "bin", "kfctl") + + # Set ENV for basic auth username/password. + init_args = [] + if use_basic_auth: + os.environ["KUBEFLOW_USERNAME"] = "kf-test-user" + os.environ["KUBEFLOW_PASSWORD"] = str(uuid.uuid4().hex) + init_args = ["--use_basic_auth"] + else: + # Owned by project kubeflow-ci-deployment. + os.environ["CLIENT_SECRET"] = "CJ4qVPLTi0j0GJMkONj7Quwt" + os.environ["CLIENT_ID"] = ( + "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g" + ".apps.googleusercontent.com") + + if use_istio: + init_args.append("--use_istio") + else: + init_args.append("--use_istio=false") + + version = "master" + if os.getenv("REPO_NAME") != "manifests": + if os.getenv("PULL_NUMBER"): + version = "pull/{0}".format(os.getenv("PULL_NUMBER")) + pull_manifests = "@master" + if os.getenv("REPO_NAME") == "manifests": + if os.getenv("PULL_PULL_SHA"): + pull_manifests = "@" + os.getenv("PULL_PULL_SHA") + + # We need to specify a valid email because + # 1. We need to create appropriate RBAC rules to allow the current user + # to create the required K8s resources. + # 2. Setting the IAM policy will fail if the email is invalid. + email = util.run(["gcloud", "config", "get-value", "account"]) + + if not email: + raise ValueError("Could not determine GCP account being used.") + + # username and password are passed as env vars and won't appear in the logs + # TODO(https://github.com/kubeflow/kubeflow/issues/2831): Once kfctl + # supports loading version from a URI we should use that so that we + # pull the configs from the repo we checked out. + # + # We don't run with retries because if kfctl init exits with an error + # but creates app.yaml then rerunning init will fail because app.yaml + # already exists. So retrying ends up masking the original error message + with open(config_path, 'r') as f: + config_spec = yaml.load(f) + config_spec["spec"]["project"] = project + config_spec["spec"]["email"] = email + config_spec["spec"] = filterSpartakus(config_spec["spec"]) + repos = config_spec["spec"]["repos"] + if os.getenv("REPO_NAME") == "manifests": + for repo in repos: + for key, value in repo.items(): + if value == "https://github.com/kubeflow/manifests/archive/master.tar.gz": + repo["uri"] = str("https://github.com/kubeflow/manifests/archive/pull/"+str(os.getenv("PULL_NUMBER"))+"/head.tar.gz") + logging.info(str(config_spec)) + with open(os.path.join(parent_dir, "tmp.yaml"), "w") as f: + yaml.dump(config_spec, f) + util.run([ + kfctl_path, "init", app_path, "-V", + "--config=" + os.path.join(parent_dir, "tmp.yaml")], cwd=parent_dir) + util.run(["cat", "app.yaml"], cwd=app_path) + + run_with_retries([ + kfctl_path, "generate", "-V", "all", "--email=" + email, "--zone=" + zone + ], + cwd=app_path) + + # We need to use retries because if we don't we see random failures + # where kfctl just appears to die. + # + # Do not run with retries since it masks errors + util.run([kfctl_path, "apply", "-V", "all"], cwd=app_path) + + verify_kubeconfig(project, zone, app_path) + +def filterSpartakus(spec): + for i, app in enumerate(spec["applications"]): + if app["name"] == "spartakus": + spec["applications"].pop(i) + break + return spec + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format=('%(levelname)s|%(asctime)s' + '|%(pathname)s|%(lineno)d| %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S', + ) + logging.getLogger().setLevel(logging.INFO) + pytest.main() diff --git a/testing/vm_util.py b/testing/vm_util.py new file mode 100644 index 00000000..92eb04bb --- /dev/null +++ b/testing/vm_util.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +"""Utilities for working with VMs as part of our tests.""" + +import datetime +import logging +import os +import socket +import ssl +import subprocess +import time +import uuid + +from kubeflow.testing import util + +# TODO(jlewi): Should we move this to kubeflow/testing + + +def wait_for_operation(client, + project, + zone, + op_id, + timeout=datetime.timedelta(hours=1), + polling_interval=datetime.timedelta(seconds=5)): + """ + Wait for the specified operation to complete. + + Args: + client: Client for the API that owns the operation. + project: project + zone: Zone. Set to none if its a global operation + op_id: Operation id/name. + timeout: A datetime.timedelta expressing the amount of time to wait before + giving up. + polling_interval: A datetime.timedelta to represent the amount of time to + wait between requests polling for the operation status. + + Returns: + op: The final operation. + + Raises: + TimeoutError: if we timeout waiting for the operation to complete. + """ + endtime = datetime.datetime.now() + timeout + while True: + try: + if zone: + op = client.zoneOperations().get( + project=project, zone=zone, operation=op_id).execute() + else: + op = client.globalOperations().get( + project=project, operation=op_id).execute() + except socket.error as e: + logging.error("Ignoring error %s", e) + continue + except ssl.SSLError as e: + logging.error("Ignoring error %s", e) + continue + status = op.get("status", "") + # Need to handle other status's + if status == "DONE": + return op + if datetime.datetime.now() > endtime: + raise TimeoutError( + "Timed out waiting for op: {0} to complete.".format(op_id)) + time.sleep(polling_interval.total_seconds()) + + +def wait_for_vm(project, + zone, + vm, + timeout=datetime.timedelta(minutes=5), + polling_interval=datetime.timedelta(seconds=10)): + """ + Wait for the VM to be ready. This is measured by trying to ssh into the VM + + timeout: A datetime.timedelta expressing the amount of time to wait + before giving up. + polling_interval: A datetime.timedelta to represent the amount of time + to wait between requests polling for the operation status. + Raises: + TimeoutError: if we timeout waiting for the operation to complete. + """ + endtime = datetime.datetime.now() + timeout + while True: + try: + util.run([ + "gcloud", "compute", "--project=" + project, "ssh", "--zone=" + zone, + vm, "--", "echo hello world" + ]) + logging.info("VM is ready") + return + except subprocess.CalledProcessError: + pass + + if datetime.datetime.now() > endtime: + raise util.TimeoutError( + ("Timed out waiting for VM to {0} be sshable. Check firewall" + "rules aren't blocking ssh.").format(vm)) + + time.sleep(polling_interval.total_seconds()) + + +def execute(project, zone, vm, commands): + """Execute the supplied commands on the VM.""" + util.run([ + "gcloud", "compute", "--project=" + project, "ssh", "--zone=" + zone, vm, + "--", " && ".join(commands) + ]) + + +def execute_script(project, zone, vm, script): + """Execute the specified script on the VM.""" + + target_path = os.path.join( + "/tmp", + os.path.basename(script) + "." + uuid.uuid4().hex[0:4]) + + target = "{0}:{1}".format(vm, target_path) + logging.info("Copying %s to %s", script, target) + util.run([ + "gcloud", "compute", "--project=" + project, "scp", script, target, + "--zone=" + zone + ]) + + util.run([ + "gcloud", "compute", "--project=" + project, "ssh", "--zone=" + zone, vm, + "--", "chmod a+rx " + target_path + " && " + target_path + ]) diff --git a/testing/workflows/components/kfctl_go_test.jsonnet b/testing/workflows/components/kfctl_go_test.jsonnet index 74ab7fae..77b4eb38 100644 --- a/testing/workflows/components/kfctl_go_test.jsonnet +++ b/testing/workflows/components/kfctl_go_test.jsonnet @@ -1,4 +1,4 @@ -// Uses test from kubeflow/kubeflow/testing/workflows/components/kfctl_go_test.jsonnet +// Uses test from kubeflow/kfctl/testing/workflows/components/kfctl_go_test.jsonnet // Any changes should reflect here and there // E2E test for the new go based version of kfctl. @@ -29,11 +29,12 @@ local outputDir = testDir + "/output"; local artifactsDir = outputDir + "/artifacts"; // Source directory where all repos should be checked out local srcRootDir = testDir + "/src"; -// The directory containing the kubeflow/kubeflow repo -local srcDir = srcRootDir + "/kubeflow/kubeflow"; +// The directory containing the kubeflow/kfctl repo +local srcDir = srcRootDir + "/kubeflow/kfctl"; +local configDir = srcRootDir + "/kubeflow/kubeflow"; local runPath = srcDir + "/testing/workflows/run.sh"; -local kfCtlPath = srcDir + "/bootstrap/bin/kfctl"; +local kfCtlPath = srcDir + "/bin/kfctl"; local kubeConfig = testDir + "/kfctl_test/.kube/kubeconfig"; // Name for the Kubeflow app. @@ -53,7 +54,8 @@ local testing_image = "gcr.io/kubeflow-ci/kubeflow-testing"; local nfsVolumeClaim = "nfs-external"; // The name to use for the volume to use to contain test data. local dataVolume = "kubeflow-test-volume"; -local kubeflowPy = srcDir; +local kfctlPy = srcDir; +local kubeflowPy = srcRootDir + "/kubeflow/kubeflow"; // The directory within the kubeflow_testing submodule containing // py scripts to use. local kubeflowTestingPy = srcRootDir + "/kubeflow/testing/py"; @@ -100,7 +102,7 @@ local buildTemplate(step_name, command, working_dir=null, env_vars=[], sidecars= { // Add the source directories to the python path. name: "PYTHONPATH", - value: kubeflowPy + ":" + kubeflowTestingPy, + value: kfctlPy + ":" + kubeflowPy + ":" + kubeflowTestingPy, }, { name: "GOOGLE_APPLICATION_CREDENTIALS", @@ -228,7 +230,7 @@ local dagTemplates = [ "-s", "--use_basic_auth=" + params.useBasicAuth, "--use_istio=" + params.useIstio, - "--config_path=" + srcDir + "/" + params.configPath, + "--config_path=" + configDir + "/" + params.configPath, // Increase the log level so that info level log statements show up. "--log-cli-level=info", "--junitxml=" + artifactsDir + "/junit_kfctl-build-test" + nameSuffix + ".xml", @@ -236,7 +238,7 @@ local dagTemplates = [ "-o", "junit_suite_name=test_kfctl_go_deploy_" + nameSuffix, "--app_path=" + appDir, ], - working_dir=srcDir+ "/testing/kfctl", + working_dir=srcDir+ "/testing/e2e", ), dependencies: ["checkout"], }, @@ -259,7 +261,7 @@ local dagTemplates = [ "-o", "junit_suite_name=test_kf_is_ready_" + nameSuffix, "--app_path=" + appDir, ], - working_dir=srcDir+ "/testing/kfctl", + working_dir=srcDir+ "/testing/e2e", ), dependencies: ["kfctl-build-deploy"], }, @@ -292,7 +294,7 @@ local deleteStep = if deleteKubeflow then "--app_path=" + appDir, "--kfctl_path=" + kfCtlPath, ], - working_dir=srcDir+ "/testing/kfctl", + working_dir=srcDir+ "/testing/e2e", ), dependencies: null, }] diff --git a/testing/workflows/components/kubeflow_workflow.libsonnet b/testing/workflows/components/kubeflow_workflow.libsonnet index ad466b38..00b19914 100644 --- a/testing/workflows/components/kubeflow_workflow.libsonnet +++ b/testing/workflows/components/kubeflow_workflow.libsonnet @@ -79,6 +79,8 @@ srcRootDir: self.testDir + "/src", // The directory containing the kubeflow/kubeflow repo srcDir: self.srcRootDir + "/kubeflow/kubeflow", + // The directory containing the kubeflow/kfctl repo + kfctlDir: self.srcRootDir + "/kubeflow/kfctl", image: "gcr.io/kubeflow-ci/test-worker:latest", // value of KUBECONFIG environment variable. This should be a full path. @@ -409,11 +411,9 @@ local artifactsDir = outputDir + "/artifacts"; // Source directory where all repos should be checked out local srcRootDir = testDir + "/src"; - // The directory containing the kubeflow/kubeflow repo - local srcDir = srcRootDir + "/kubeflow/kubeflow"; - local bootstrapDir = srcDir + "/bootstrap"; + // The directory containing the kubeflow/kfctl repo + local srcDir = srcRootDir + "/kubeflow/kfctl"; local image = "gcr.io/kubeflow-ci/test-worker:latest"; - local bootstrapperImage = "gcr.io/kubeflow-ci/bootstrapper:" + name; // The last 4 digits of the name should be a unique id. local deploymentName = "e2e-" + std.substr(name, std.length(name) - 4, 4); local v1alpha1Suffix = "-v1alpha1"; @@ -653,7 +653,7 @@ ["/usr/local/bin/checkout.sh", srcRootDir], env_vars=[{ name: "EXTRA_REPOS", - value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD", + value: "kubeflow/kubeflow@HEAD;kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD", }], ), buildTemplate("test-dir-delete", [ diff --git a/testing/workflows/run.sh b/testing/workflows/run.sh new file mode 100755 index 00000000..3b92615a --- /dev/null +++ b/testing/workflows/run.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# A simple wrapper script to run a command in the e2e tests. +# This script performs common functions like +# activating the service account. +set -ex +gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} + +echo Working Directory=$(pwd) +# Execute the actual command. +# TODO(jlewi): We should add retries on error. + +# Retry up to 3 times +for i in $(seq 1 3); do + set +e + "$@" + result=$? + set -e + if [[ ${result} -eq 0 ]]; then + echo command ran successfully + exit 0 + fi + + echo Command failed: "$@" +done +echo "command didn't succeed" +exit 1