Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Samples - Simplified pipeline submission code in samples #2293

Merged
merged 1 commit into from
Oct 4, 2019
Merged

Samples - Simplified pipeline submission code in samples #2293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 9 additions & 29 deletions samples/contrib/arena-samples/standalonejob/standalone_pipeline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,29 +208,6 @@
"!pip3 install $KFP_ARENA_PACKAGE --upgrade"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import the KubeFlow Pipeline library and define the client and experiment "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"import kfp\n",
"\n",
"client = kfp.Client(KUBEFLOW_PIPELINE_LINK)\n",
"\n",
"try:\n",
" experiment_id = client.get_experiment(experiment_name=EXPERIMENT_NAME).id\n",
"except:\n",
" experiment_id = client.create_experiment(EXPERIMENT_NAME).id"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -246,7 +223,6 @@
"source": [
"import arena\n",
"import kfp.dsl as dsl\n",
"import kfp.compiler as compiler\n",
"\n",
"@dsl.pipeline(\n",
" name='pipeline to run jobs',\n",
Expand Down Expand Up @@ -328,11 +304,15 @@
"dropout = \"0.8\"\n",
"model_verison = \"1\"\n",
"\n",
"compiler.Compiler().compile(sample_pipeline, 'standalone.tar.gz')\n",
"arguments = {\n",
" 'learning_rate': learning_rate,\n",
" 'dropout': dropout,\n",
" 'model_version': model_version,\n",
"}\n",
"\n",
"run = client.run_pipeline(experiment_id, 'mnist', 'standalone.tar.gz', params={'learning_rate':learning_rate,\n",
" 'dropout':dropout,\n",
" 'model_version':model_version})\n",
"import kfp\n",
"client = kfp.Client(host=KUBEFLOW_PIPELINE_LINK)\n",
"run = client.create_run_from_pipeline_func(sample_pipeline, arguments=arguments).run_info\n",
"\n",
"print('The above run link is assuming you ran this cell on JupyterHub that is deployed on the same cluster. ' +\n",
" 'The actual run link is ' + KUBEFLOW_PIPELINE_LINK + '/#/runs/details/' + run.id)"
Expand Down Expand Up @@ -362,7 +342,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
"version": "3.6.8"
}
},
"nbformat": 4,
Expand Down
22 changes: 9 additions & 13 deletions samples/contrib/arena-samples/standalonejob/standalone_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,19 +77,15 @@ def sample_pipeline(learning_rate='0.01',
dropout = FLAGS.dropout
learning_rate = FLAGS.learning_rate
commit = FLAGS.commit

arguments = {
'learning_rate': learning_rate,
'dropout': dropout,
'model_version': model_version,
'commit': commit,
}

EXPERIMENT_NAME="mnist"
RUN_ID="run"
KFP_SERVICE="ml-pipeline.kubeflow.svc.cluster.local:8888"
import kfp.compiler as compiler
compiler.Compiler().compile(sample_pipeline, __file__ + '.tar.gz')
client = kfp.Client(host=KFP_SERVICE)
try:
experiment_id = client.get_experiment(experiment_name=EXPERIMENT_NAME).id
except:
experiment_id = client.create_experiment(EXPERIMENT_NAME).id
run = client.run_pipeline(experiment_id, RUN_ID, __file__ + '.tar.gz',
params={'learning_rate':learning_rate,
'dropout':dropout,
'model_version':model_version,
'commit':commit})

client.create_run_from_pipeline_func(sample_pipeline, arguments=arguments)
17 changes: 6 additions & 11 deletions samples/contrib/ibm-samples/ffdl-seldon/ffdl_pipeline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,14 @@
"# Train and deploy with FfDL and Seldon demo\n"
]
},

{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### A simple IBM OSS pipeline demonstrates how to train a model using Fabric for Deep Learning and then deploy it with Seldon.\n",
"\n"
]
},
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -33,7 +32,7 @@
"source": [
"### Define the necessary environment variables and install the KubeFlow Pipeline SDK\n",
"We assume this notebook kernel has access to Python's site-packages and is in Python3.\n",
"\n",
"\n"
]
},
{
Expand Down Expand Up @@ -94,9 +93,7 @@
"client = kfp.Client()\n",
"\n",
"\n",
"EXPERIMENT_NAME = 'FfDL-Seldon Experiments'\n",
"\n",
"exp = client.create_experiment(name=EXPERIMENT_NAME)\n"
"EXPERIMENT_NAME = 'FfDL-Seldon Experiments'"
]
},
{
Expand Down Expand Up @@ -182,10 +179,8 @@
" 'model-class-file': 'gender_classification.py'}\n",
"\n",
"\n",
"compiler.Compiler().compile(ffdlPipeline, 'ffdl-pipeline.tar.gz')\n",
"run = client.create_run_from_pipeline_func(ffdlPipeline, arguments=parameters, experiment_name=EXPERIMENT_NAME).run_info\n",
"\n",
"run = client.run_pipeline(exp.id, 'ffdl-seldon-pipeline', 'ffdl-pipeline.tar.gz', \n",
" params=parameters)\n",
"import IPython\n",
"html = ('<p id=\"link\"> </p> <script> document.getElementById(\"link\").innerHTML = \"Actual Run link <a href=//\" + location.hostname + \"%s/#/runs/details/%s target=_blank >here</a>\"; </script>'\n",
" % (client._get_url_prefix(), run.id))\n",
Expand All @@ -202,9 +197,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python on Kubernetes",
"display_name": "Python 3",
"language": "python",
"name": "python_kubernetes"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand Down
9 changes: 3 additions & 6 deletions samples/contrib/ibm-samples/openscale/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,10 @@ To access the credentials file, the user should provide a github access token an
```python
import kfp.dsl as dsl
import kfp.components as components
from kfp import compiler
import kfp
secret_name = 'aios-creds'
configuration_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/master/components/ibm-components/commons/config/component.yaml')
client = kfp.Client()
EXPERIMENT_NAME = 'create secret'
exp = client.create_experiment(name=EXPERIMENT_NAME)

@dsl.pipeline(
name='create secret',
description=''
Expand All @@ -67,8 +64,8 @@ def secret_pipeline(
url=CONFIG_FILE_URL,
name=secret_name
)
compiler.Compiler().compile(secret_pipeline, 'secret_pipeline.tar.gz')
run = client.run_pipeline(exp.id, 'secret_pipeline', 'secret_pipeline.tar.gz')

kfp.Client().create_run_from_pipeline_func(secret_pipeline, arguments={})
```

## Instructions
Expand Down
27 changes: 2 additions & 25 deletions samples/contrib/image-captioning-gcp/Image Captioning TF 2.0.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,28 +893,6 @@
" use_gcp_secret('user-gcp-sa'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipeline_filename = caption_pipeline.__name__ + '.pipeline.zip'\n",
"compiler.Compiler().compile(pipeline_func, pipeline_filename)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"client = kfp.Client()\n",
"experiment = client.create_experiment(EXPERIMENT_NAME)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -934,9 +912,8 @@
" 'num_examples': 100, # Small test to make sure pipeline functions properly\n",
" 'training_batch_size': 16, # has to be smaller since only training on 80/100 examples \n",
"}\n",
"run_name = caption_pipeline.__name__ + ' run'\n",
"run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename,\n",
" params=arguments)"
"\n",
"kfp.Client().create_run_from_pipeline_func(pipeline, arguments=arguments, experiment_name=EXPERIMENT_NAME)"
]
},
{
Expand Down
41 changes: 5 additions & 36 deletions samples/core/ai-platform/Chicago Crime Pipeline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
"import json\n",
"\n",
"import kfp\n",
"import kfp.compiler as compiler\n",
"import kfp.components as comp\n",
"import kfp.dsl as dsl\n",
"import kfp.gcp as gcp\n",
Expand Down Expand Up @@ -254,24 +253,6 @@
"pipeline_func = pipeline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Compile pipeline"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipeline_filename = PIPELINE_FILENAME_PREFIX + '.pipeline.zip'\n",
"\n",
"compiler.Compiler().compile(pipeline_func, pipeline_filename)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -285,19 +266,7 @@
"metadata": {},
"outputs": [],
"source": [
"# Specify pipeline argument values\n",
"arguments = {}\n",
"\n",
"# Get or create an experiment and submit a pipeline run\n",
"client = kfp.Client()\n",
"try:\n",
" experiment = client.get_experiment(experiment_name=EXPERIMENT_NAME)\n",
"except:\n",
" experiment = client.create_experiment(EXPERIMENT_NAME)\n",
"\n",
"# Submit a pipeline run\n",
"run_name = pipeline_func.__name__ + ' run'\n",
"run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, arguments)"
"kfp.Client().create_run_from_pipeline_func(pipeline, arguments={}, experiment_name=EXPERIMENT_NAME)"
]
}
],
Expand All @@ -317,18 +286,18 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
"version": "3.6.8"
},
"pycharm": {
"stem_cell": {
"cell_type": "raw",
"source": [],
"metadata": {
"collapsed": false
}
},
"source": []
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
}
32 changes: 2 additions & 30 deletions samples/core/dataflow/dataflow.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@
"metadata": {},
"outputs": [],
"source": [
"import kfp\n",
"import kfp.dsl as dsl\n",
"import kfp.gcp as gcp\n",
"import json\n",
Expand Down Expand Up @@ -329,25 +330,6 @@
" wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compile the pipeline"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"pipeline_func = pipeline\n",
"pipeline_filename = pipeline_func.__name__ + '.zip'\n",
"import kfp.compiler as compiler\n",
"compiler.Compiler().compile(pipeline_func, pipeline_filename)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -386,17 +368,7 @@
}
],
"source": [
"#Specify pipeline argument values\n",
"arguments = {}\n",
"\n",
"#Get or create an experiment and submit a pipeline run\n",
"import kfp\n",
"client = kfp.Client()\n",
"experiment = client.create_experiment(experiment_name)\n",
"\n",
"#Submit a pipeline run\n",
"run_name = pipeline_func.__name__ + ' run'\n",
"run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, arguments)"
"kfp.Client().create_run_from_pipeline_func(pipeline, arguments={}, experiment_name=experiment_name)"
]
},
{
Expand Down
Loading