From 5dc1c75b0d87cc3cbe50720952bad78fc7a5941e Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Fri, 14 Feb 2020 19:06:01 +0800 Subject: [PATCH 1/8] TFMA basic: rendering slicing metrics --- frontend/src/lib/OutputArtifactLoader.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 61d967ac22a..32d49933211 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -316,6 +316,20 @@ export class OutputArtifactLoader { return buildArtifactViewer(script); }), ); + const EvaluatorArtifactUris = filterArtifactUrisByType('ModelEvaluation', artifactTypes, artifacts); + viewers = viewers.concat( + EvaluatorArtifactUris.map(uri => { + const script = [ + 'import tensorflow_model_analysis as tfma', + 'from ipywidgets.embed import embed_minimal_html', + `eval_res = tfma.load_eval_result('${uri}')`, + 'slicing_metrics_view = tfma.view.render_slicing_metrics(eval_res)', + `embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics')`, + `with open('tfma_export.html', 'r') as view: print(view.read())`, + ]; + return buildArtifactViewer(script); + }), + ); return Promise.all(viewers); } From 0858039cca130ba340b5e2c628f2718bd8abc385 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Tue, 18 Feb 2020 17:25:33 +0800 Subject: [PATCH 2/8] tfma rendering --- frontend/src/lib/OutputArtifactLoader.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 32d49933211..bdc71f0e4dc 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -319,17 +319,29 @@ export class OutputArtifactLoader { const EvaluatorArtifactUris = filterArtifactUrisByType('ModelEvaluation', artifactTypes, artifacts); viewers = viewers.concat( EvaluatorArtifactUris.map(uri => { + const config_file_path = uri + '/eval_config.json'; const script = [ - 'import tensorflow_model_analysis as tfma', - 'from ipywidgets.embed import embed_minimal_html', - `eval_res = tfma.load_eval_result('${uri}')`, - 'slicing_metrics_view = tfma.view.render_slicing_metrics(eval_res)', - `embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics')`, - `with open('tfma_export.html', 'r') as view: print(view.read())`, + ` + import json + import tensorflow as tf + import tensorflow_model_analysis as tfma + from ipywidgets.embed import embed_minimal_html + from IPython.core.display import display, HTML + config_file=tf.io.gfile.GFile('${config_file_path}', 'r') + config=json.loads(config_file.read()) + featureKeys=list(filter(lambda x: 'featureKeys' in x, config['evalConfig']['slicingSpecs'])) + columns=[] if len(featureKeys) == 0 else featureKeys[0]['featureKeys'] + slicing_spec = tfma.slicer.SingleSliceSpec(columns=columns) + eval_result = tfma.load_eval_result('${uri}') + slicing_metrics_view = tfma.view.render_slicing_metrics(eval_result, slicing_spec=slicing_spec) + embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics') + with open('tfma_export.html', 'r') as view: display(HTML(view.read())) + ` ]; return buildArtifactViewer(script); }), ); + return Promise.all(viewers); } From bc7765b5c8206471e9c4529caf3ab3fa065383ac Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Tue, 18 Feb 2020 22:07:03 +0800 Subject: [PATCH 3/8] render --- frontend/src/lib/OutputArtifactLoader.ts | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index bdc71f0e4dc..055f97d9ad4 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -319,24 +319,24 @@ export class OutputArtifactLoader { const EvaluatorArtifactUris = filterArtifactUrisByType('ModelEvaluation', artifactTypes, artifacts); viewers = viewers.concat( EvaluatorArtifactUris.map(uri => { - const config_file_path = uri + '/eval_config.json'; + const configFilePath = uri + '/eval_config.json'; const script = [ - ` - import json - import tensorflow as tf - import tensorflow_model_analysis as tfma - from ipywidgets.embed import embed_minimal_html - from IPython.core.display import display, HTML - config_file=tf.io.gfile.GFile('${config_file_path}', 'r') - config=json.loads(config_file.read()) - featureKeys=list(filter(lambda x: 'featureKeys' in x, config['evalConfig']['slicingSpecs'])) - columns=[] if len(featureKeys) == 0 else featureKeys[0]['featureKeys'] - slicing_spec = tfma.slicer.SingleSliceSpec(columns=columns) - eval_result = tfma.load_eval_result('${uri}') - slicing_metrics_view = tfma.view.render_slicing_metrics(eval_result, slicing_spec=slicing_spec) - embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics') - with open('tfma_export.html', 'r') as view: display(HTML(view.read())) - ` + `import json`, + `import tensorflow as tf`, + `import tensorflow_model_analysis as tfma`, + `from ipywidgets.embed import embed_minimal_html`, + `from IPython.core.display import display, HTML`, + `config_file=tf.io.gfile.GFile('${configFilePath}', 'r')`, + `config=json.loads(config_file.read())`, + `featureKeys=list(filter(lambda x: 'featureKeys' in x, config['evalConfig']['slicingSpecs']))`, + `columns=[] if len(featureKeys) == 0 else featureKeys[0]['featureKeys']`, + `slicing_spec = tfma.slicer.SingleSliceSpec(columns=columns)`, + `eval_result = tfma.load_eval_result('${uri}')`, + `slicing_metrics_view = tfma.view.render_slicing_metrics(eval_result, slicing_spec=slicing_spec)`, + `embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics')`, + `with open('tfma_export.html', 'r') as view: view_html = view.read()`, + `res_html = view_html.replace('dist/embed-amd.js" crossorigin="anonymous">', 'dist/embed-amd.js" crossorigin="anonymous" data-jupyter-widgets-cdn="https://cdn.jsdelivr.net/gh/Bobgy/model-analysis@kfp/tensorflow_model_analysis/notebook/jupyter/js/dist/" crossorigin="anonymous">')`, + `display(HTML(res_html))` ]; return buildArtifactViewer(script); }), From 6b0e8ac9dabe1031b90f41298f666f63a77e8b7c Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 19 Feb 2020 08:37:07 +0800 Subject: [PATCH 4/8] remove out-of-date todos --- frontend/src/lib/OutputArtifactLoader.ts | 43 +----------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 055f97d9ad4..f30946ab1ed 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -334,6 +334,7 @@ export class OutputArtifactLoader { `eval_result = tfma.load_eval_result('${uri}')`, `slicing_metrics_view = tfma.view.render_slicing_metrics(eval_result, slicing_spec=slicing_spec)`, `embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics')`, + `view_html=None`, `with open('tfma_export.html', 'r') as view: view_html = view.read()`, `res_html = view_html.replace('dist/embed-amd.js" crossorigin="anonymous">', 'dist/embed-amd.js" crossorigin="anonymous" data-jupyter-widgets-cdn="https://cdn.jsdelivr.net/gh/Bobgy/model-analysis@kfp/tensorflow_model_analysis/notebook/jupyter/js/dist/" crossorigin="anonymous">')`, `display(HTML(res_html))` @@ -562,45 +563,3 @@ async function buildArtifactViewer(script: string[]): Promise type: PlotType.WEB_APP, }; } - -// TODO: add tfma back -// function filterTfmaArtifactsPaths( -// artifactTypes: ArtifactType[], -// artifacts: Artifact[], -// ): string[] { -// const tfmaArtifactTypeIds = artifactTypes -// .filter(artifactType => artifactType.getName() === 'ModelEvaluation') -// .map(artifactType => artifactType.getId()); -// const tfmaArtifacts = artifacts.filter(artifact => -// tfmaArtifactTypeIds.includes(artifact.getTypeId()), -// ); - -// const tfmaArtifactPaths = tfmaArtifacts.map(artifact => artifact.getUri()).filter(uri => uri); // uri not empty -// return tfmaArtifactPaths; -// } - -// async function getTfmaArtifactViewers( -// tfmaArtifactPaths: string[], -// ): Array> { -// return tfmaArtifactPaths.map(async artifactPath => { -// const script = [ -// 'import tensorflow_model_analysis as tfma', -// `tfma_result = tfma.load_eval_result('${artifactPath}')`, -// 'tfma.view.render_slicing_metrics(tfma_result)', -// ]; -// const visualizationData: ApiVisualization = { -// arguments: JSON.stringify({ code: script }), -// source: '', -// type: ApiVisualizationType.CUSTOM, -// }; -// const visualization = await Apis.buildPythonVisualizationConfig(visualizationData); -// if (!visualization.htmlContent) { -// // TODO: Improve error message with details. -// throw new Error('Failed to build TFMA artifact visualization'); -// } -// return { -// htmlContent: visualization.htmlContent, -// type: PlotType.WEB_APP, -// }; -// }); -// } From 03295b4308d5850ecef5168ceb2cacc4ce0fcaa6 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 19 Feb 2020 10:57:44 +0800 Subject: [PATCH 5/8] FE format --- frontend/src/lib/OutputArtifactLoader.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 368b5a276a9..fe313486490 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -311,7 +311,11 @@ export class OutputArtifactLoader { return buildArtifactViewer(script); }), ); - const EvaluatorArtifactUris = filterArtifactUrisByType('ModelEvaluation', artifactTypes, artifacts); + const EvaluatorArtifactUris = filterArtifactUrisByType( + 'ModelEvaluation', + artifactTypes, + artifacts, + ); viewers = viewers.concat( EvaluatorArtifactUris.map(uri => { const configFilePath = uri + '/eval_config.json'; @@ -332,7 +336,7 @@ export class OutputArtifactLoader { `view_html=None`, `with open('tfma_export.html', 'r') as view: view_html = view.read()`, `res_html = view_html.replace('dist/embed-amd.js" crossorigin="anonymous">', 'dist/embed-amd.js" crossorigin="anonymous" data-jupyter-widgets-cdn="https://cdn.jsdelivr.net/gh/Bobgy/model-analysis@kfp/tensorflow_model_analysis/notebook/jupyter/js/dist/" crossorigin="anonymous">')`, - `display(HTML(res_html))` + `display(HTML(res_html))`, ]; return buildArtifactViewer(script); }), From c9282724aa6d7b7dcdb80d4d605ecf6dc8edaec9 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 19 Feb 2020 11:23:51 +0800 Subject: [PATCH 6/8] Use io.StringIO instead of file --- frontend/src/lib/OutputArtifactLoader.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index fe313486490..75005574c13 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -320,6 +320,7 @@ export class OutputArtifactLoader { EvaluatorArtifactUris.map(uri => { const configFilePath = uri + '/eval_config.json'; const script = [ + `import io`, `import json`, `import tensorflow as tf`, `import tensorflow_model_analysis as tfma`, @@ -332,11 +333,10 @@ export class OutputArtifactLoader { `slicing_spec = tfma.slicer.SingleSliceSpec(columns=columns)`, `eval_result = tfma.load_eval_result('${uri}')`, `slicing_metrics_view = tfma.view.render_slicing_metrics(eval_result, slicing_spec=slicing_spec)`, - `embed_minimal_html('tfma_export.html', views=[slicing_metrics_view], title='Slicing Metrics')`, - `view_html=None`, - `with open('tfma_export.html', 'r') as view: view_html = view.read()`, - `res_html = view_html.replace('dist/embed-amd.js" crossorigin="anonymous">', 'dist/embed-amd.js" crossorigin="anonymous" data-jupyter-widgets-cdn="https://cdn.jsdelivr.net/gh/Bobgy/model-analysis@kfp/tensorflow_model_analysis/notebook/jupyter/js/dist/" crossorigin="anonymous">')`, - `display(HTML(res_html))`, + `view = io.StringIO()`, + `embed_minimal_html(view, views=[slicing_metrics_view], title='Slicing Metrics')`, + `html = view.getvalue().replace('dist/embed-amd.js" crossorigin="anonymous">', 'dist/embed-amd.js" crossorigin="anonymous" data-jupyter-widgets-cdn="https://cdn.jsdelivr.net/gh/Bobgy/model-analysis@kfp/tensorflow_model_analysis/notebook/jupyter/js/dist/" crossorigin="anonymous">')`, + `display(HTML(html))`, ]; return buildArtifactViewer(script); }), From e3d38ffea70a8dcd75c4299e7032d33f26c9408d Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 19 Feb 2020 11:34:01 +0800 Subject: [PATCH 7/8] Add comment for context and reference of TFMA widget hack --- frontend/src/lib/OutputArtifactLoader.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 75005574c13..29b3c664f53 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -319,6 +319,9 @@ export class OutputArtifactLoader { viewers = viewers.concat( EvaluatorArtifactUris.map(uri => { const configFilePath = uri + '/eval_config.json'; + // The visualization of TFMA inside KFP UI depends a hack of TFMA widget js + // For context and future improvement, please refer to + // https://github.com/tensorflow/model-analysis/issues/10#issuecomment-587422929 const script = [ `import io`, `import json`, From de1a73ff08325ba52a37824b9d51375f6c2f8b4e Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 19 Feb 2020 12:02:25 +0800 Subject: [PATCH 8/8] Add TODO --- frontend/src/lib/OutputArtifactLoader.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/lib/OutputArtifactLoader.ts b/frontend/src/lib/OutputArtifactLoader.ts index 29b3c664f53..687d04de123 100644 --- a/frontend/src/lib/OutputArtifactLoader.ts +++ b/frontend/src/lib/OutputArtifactLoader.ts @@ -344,6 +344,7 @@ export class OutputArtifactLoader { return buildArtifactViewer(script); }), ); + // TODO(jingzhang36): maybe move the above built-in scripts to visualization server. return Promise.all(viewers); }