Skip to content

Commit

Permalink
update samples with dependency requirement in the component image bui…
Browse files Browse the repository at this point in the history 
…ld (#378)
  • Loading branch information
@gaoning777 @k8s-ci-robot
gaoning777 authored and k8s-ci-robot committed Nov 27, 2018
1 parent b528f50 commit 29c2b4b
Showing 1 changed file with 148 additions and 16 deletions.
164 changes: 148 additions & 16 deletions samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,64 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {
"scrolled": false
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz\n",
"\u001b[?25l Downloading https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz (68kB)\n",
"\u001b[K 100% |████████████████████████████████| 71kB 9.4MB/s eta 0:00:01\n",
"\u001b[?25hRequirement already satisfied, skipping upgrade: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.22)\n",
"Requirement already satisfied, skipping upgrade: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.11.0)\n",
"Requirement already satisfied, skipping upgrade: certifi in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2018.10.15)\n",
"Requirement already satisfied, skipping upgrade: python-dateutil in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2.7.3)\n",
"Requirement already satisfied, skipping upgrade: PyYAML in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (3.13)\n",
"Requirement already satisfied, skipping upgrade: google-cloud-storage==1.13.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.13.0)\n",
"Requirement already satisfied, skipping upgrade: kubernetes==8.0.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (8.0.0)\n",
"Requirement already satisfied, skipping upgrade: google-api-core<2.0.0dev,>=0.1.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (1.5.2)\n",
"Requirement already satisfied, skipping upgrade: google-cloud-core<0.29dev,>=0.28.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.28.1)\n",
"Requirement already satisfied, skipping upgrade: google-resumable-media>=0.3.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.3.1)\n",
"Requirement already satisfied, skipping upgrade: requests-oauthlib in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.0.0)\n",
"Requirement already satisfied, skipping upgrade: adal>=1.0.2 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.2.0)\n",
"Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (2.18.4)\n",
"Requirement already satisfied, skipping upgrade: google-auth>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.6.1)\n",
"Requirement already satisfied, skipping upgrade: setuptools>=21.0.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (38.4.0)\n",
"Requirement already satisfied, skipping upgrade: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (0.54.0)\n",
"Requirement already satisfied, skipping upgrade: protobuf>=3.4.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (3.6.1)\n",
"Requirement already satisfied, skipping upgrade: pytz in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (2018.7)\n",
"Requirement already satisfied, skipping upgrade: googleapis-common-protos!=1.5.4,<2.0dev,>=1.5.3 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (1.5.5)\n",
"Requirement already satisfied, skipping upgrade: oauthlib>=0.6.2 in /opt/conda/lib/python3.6/site-packages (from requests-oauthlib->kubernetes==8.0.0->kfp==0.1) (2.1.0)\n",
"Requirement already satisfied, skipping upgrade: cryptography>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (2.1.4)\n",
"Requirement already satisfied, skipping upgrade: PyJWT>=1.0.0 in /opt/conda/lib/python3.6/site-packages (from adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (1.6.4)\n",
"Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->kubernetes==8.0.0->kfp==0.1) (3.0.4)\n",
"Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->kubernetes==8.0.0->kfp==0.1) (2.6)\n",
"Requirement already satisfied, skipping upgrade: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (3.0.0)\n",
"Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (0.2.2)\n",
"Requirement already satisfied, skipping upgrade: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (4.0)\n",
"Requirement already satisfied, skipping upgrade: asn1crypto>=0.21.0 in /opt/conda/lib/python3.6/site-packages (from cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (0.24.0)\n",
"Requirement already satisfied, skipping upgrade: cffi>=1.7 in /opt/conda/lib/python3.6/site-packages (from cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (1.11.4)\n",
"Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (0.4.4)\n",
"Requirement already satisfied, skipping upgrade: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi>=1.7->cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (2.18)\n",
"Building wheels for collected packages: kfp\n",
" Running setup.py bdist_wheel for kfp ... \u001b[?25ldone\n",
"\u001b[?25h Stored in directory: /tmp/pip-ephem-wheel-cache-bctiow25/wheels/f9/43/15/db34c8d6291d495360ac6eea475e6bf473750320547f837caf\n",
"Successfully built kfp\n",
"Installing collected packages: kfp\n",
" Found existing installation: kfp 0.1\n",
" Uninstalling kfp-0.1:\n",
" Successfully uninstalled kfp-0.1\n",
"Successfully installed kfp-0.1\n"
]
}
],
"source": [
"# Install Pipeline SDK\n",
"!pip3 install https://storage.googleapis.com/ml-pipeline/release/0.1.1/kfp.tar.gz --upgrade\n",
" \n"
"!pip3 install https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz --upgrade"
]
},
{
Expand Down Expand Up @@ -97,7 +146,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand All @@ -117,13 +166,13 @@
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Run link <a href=\"/pipeline/#/runs/details/97280443-e3ae-11e8-860c-42010a800274\" target=\"_blank\" >here</a>"
"Run link <a href=\"/pipeline/#/runs/details/67b216a8-f1ad-11e8-927c-42010a8000f7\" target=\"_blank\" >here</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
Expand All @@ -148,7 +197,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -302,13 +351,13 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Run link <a href=\"/pipeline/#/runs/details/7df8f63f-e2c4-11e8-93d0-42010a800048\" target=\"_blank\" >here</a>"
"Run link <a href=\"/pipeline/#/runs/details/d1a380ae-f1ad-11e8-927c-42010a8000f7\" target=\"_blank\" >here</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
Expand Down Expand Up @@ -346,17 +395,34 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: google-api-python-client in /opt/conda/lib/python3.6/site-packages (1.7.4)\n",
"Requirement already satisfied: google-auth>=1.4.1 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (1.6.1)\n",
"Requirement already satisfied: httplib2<1dev,>=0.9.2 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (0.11.3)\n",
"Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (3.0.0)\n",
"Requirement already satisfied: six<2dev,>=1.6.1 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (1.11.0)\n",
"Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (0.0.3)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (0.2.2)\n",
"Requirement already satisfied: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (3.0.0)\n",
"Requirement already satisfied: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (4.0)\n",
"Requirement already satisfied: pyasn1<0.5.0,>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.4.1->google-api-python-client) (0.4.4)\n"
]
}
],
"source": [
"# in order to run it locally we need a python package\n",
"!pip3 install google-api-python-client"
]
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -421,9 +487,75 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Build a Pipeline Step With the Above Function\n",
"### Build a Pipeline Step With the Above Function(Note: run either of the two options below)\n",
"#### Option One: Specify the dependency directly\n",
"Now that we've tested the function locally, we want to build a component that can run as a step in the pipeline. "
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2018-11-26 19:29:13:INFO:Build an image that is based on gcr.io/ml-pipeline-dogfood/pusherbase:dev and push the image to gcr.io/ml-pipeline-dogfood/pusher:dev\n",
"2018-11-26 19:29:13:INFO:Checking path: gs://ngao-bugbash...\n",
"2018-11-26 19:29:13:INFO:Generate entrypoint and serialization codes.\n",
"2018-11-26 19:29:13:INFO:Generate build files.\n",
"2018-11-26 19:29:13:INFO:Start a kaniko job for build.\n",
"2018-11-26 19:29:18:INFO:5 seconds: waiting for job to complete\n",
"2018-11-26 19:29:23:INFO:10 seconds: waiting for job to complete\n",
"2018-11-26 19:29:28:INFO:15 seconds: waiting for job to complete\n",
"2018-11-26 19:29:33:INFO:20 seconds: waiting for job to complete\n",
"2018-11-26 19:29:38:INFO:25 seconds: waiting for job to complete\n",
"2018-11-26 19:29:43:INFO:30 seconds: waiting for job to complete\n",
"2018-11-26 19:29:48:INFO:35 seconds: waiting for job to complete\n",
"2018-11-26 19:29:53:INFO:40 seconds: waiting for job to complete\n",
"2018-11-26 19:29:58:INFO:45 seconds: waiting for job to complete\n",
"2018-11-26 19:30:03:INFO:50 seconds: waiting for job to complete\n",
"2018-11-26 19:30:08:INFO:55 seconds: waiting for job to complete\n",
"2018-11-26 19:30:13:INFO:60 seconds: waiting for job to complete\n",
"2018-11-26 19:30:18:INFO:65 seconds: waiting for job to complete\n",
"2018-11-26 19:30:23:INFO:70 seconds: waiting for job to complete\n",
"2018-11-26 19:30:28:INFO:75 seconds: waiting for job to complete\n",
"2018-11-26 19:30:33:INFO:80 seconds: waiting for job to complete\n",
"2018-11-26 19:30:38:INFO:85 seconds: waiting for job to complete\n",
"2018-11-26 19:30:43:INFO:90 seconds: waiting for job to complete\n",
"2018-11-26 19:30:48:INFO:95 seconds: waiting for job to complete\n",
"2018-11-26 19:30:53:INFO:100 seconds: waiting for job to complete\n",
"2018-11-26 19:30:58:INFO:105 seconds: waiting for job to complete\n",
"2018-11-26 19:31:03:INFO:110 seconds: waiting for job to complete\n",
"2018-11-26 19:31:08:INFO:115 seconds: waiting for job to complete\n",
"2018-11-26 19:31:13:INFO:120 seconds: waiting for job to complete\n",
"2018-11-26 19:31:18:INFO:125 seconds: waiting for job to complete\n",
"2018-11-26 19:31:24:INFO:130 seconds: waiting for job to complete\n",
"2018-11-26 19:31:29:INFO:135 seconds: waiting for job to complete\n",
"2018-11-26 19:31:34:INFO:140 seconds: waiting for job to complete\n",
"2018-11-26 19:31:39:INFO:145 seconds: waiting for job to complete\n",
"2018-11-26 19:31:39:INFO:Kaniko job complete.\n",
"2018-11-26 19:31:39:INFO:Build component complete.\n"
]
}
],
"source": [
"from kfp import compiler\n",
"\n",
"Now that we've tested the function locally, we want to build a component that can run as a step in the pipeline. First we need to build a base docker container image. We need TensorFlow and google-api-python-client packages."
"# The return value \"DeployerOp\" represents a step that can be used directly in a pipeline function\n",
"DeployerOp = compiler.build_python_component(\n",
" component_func=deploy_model,\n",
" staging_gcs_path=OUTPUT_DIR,\n",
" dependency=[kfp.compiler.VersionedDependency(name='google-api-python-client', version='1.7.0')],\n",
" target_image=TARGET_IMAGE)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Option Two: build a base docker container image with both tensorflow and google api client packages"
]
},
{
Expand Down Expand Up @@ -547,7 +679,7 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
Expand Down

0 comments on commit 29c2b4b

Please sign in to comment.