Skip to content

Commit

Permalink
Add metric and ui-metadata samples (kubeflow#523)
Browse files Browse the repository at this point in the history 
* add metric sample code in roc component

* Change mlpipeline artifacts output path to tmp folder

* Revert /tmp path change and add output_dir in notebook sample

* Revert /tmp path kubeflow#2
  • Loading branch information
@hongye-sun @neuromage
hongye-sun authored and neuromage committed Dec 13, 2018
1 parent 8549d81 commit 164b85f
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 39 deletions.
11 changes: 10 additions & 1 deletion components/local/roc/src/roc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import json
import os
import pandas as pd
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_curve, roc_auc_score
from tensorflow.python.lib.io import file_io


Expand Down Expand Up @@ -58,6 +58,7 @@ def main(argv=None):
else:
df['target'] = df['target'].apply(lambda x: 1 if x == args.trueclass else 0)
fpr, tpr, thresholds = roc_curve(df['target'], df[args.trueclass])
roc_auc = roc_auc_score(df['target'], df[args.trueclass])
df_roc = pd.DataFrame({'fpr': fpr, 'tpr': tpr, 'thresholds': thresholds})
roc_file = os.path.join(args.output, 'roc.csv')
with file_io.FileIO(roc_file, 'w') as f:
Expand All @@ -79,6 +80,14 @@ def main(argv=None):
with file_io.FileIO('/mlpipeline-ui-metadata.json', 'w') as f:
json.dump(metadata, f)

metrics = {
'metrics': [{
'name': 'roc-auc-score',
'numberValue': roc_auc,
}]
}
with file_io.FileIO('/mlpipeline-metrics.json', 'w') as f:
json.dump(metrics, f)

if __name__== "__main__":
main()
78 changes: 40 additions & 38 deletions samples/notebooks/Lightweight Python components - basics.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -61,7 +61,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -80,7 +80,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -96,14 +96,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Advanced function\n",
"#Demonstrates imports, helper functions and multiple outputs\n",
"from typing import NamedTuple\n",
"def my_divmod(dividend: float, divisor:float) -> NamedTuple('MyDivmodOutput', [('quotient', float), ('remainder', float)]):\n",
"def my_divmod(dividend: float, divisor:float, output_dir:str = './') -> NamedTuple('MyDivmodOutput', [('quotient', float), ('remainder', float)]):\n",
" '''Divides two numbers and calculate the quotient and remainder'''\n",
" #Imports inside a component function:\n",
" import numpy as np\n",
Expand All @@ -114,6 +114,32 @@
"\n",
" (quotient, remainder) = divmod_helper(dividend, divisor)\n",
"\n",
" from tensorflow.python.lib.io import file_io\n",
" import json\n",
" \n",
" # Exports a sample tensorboard:\n",
" metadata = {\n",
" 'outputs' : [{\n",
" 'type': 'tensorboard',\n",
" 'source': 'gs://ml-pipeline-dataset/tensorboard-train',\n",
" }]\n",
" }\n",
" with open(output_dir + 'mlpipeline-ui-metadata.json', 'w') as f:\n",
" json.dump(metadata, f)\n",
"\n",
" # Exports two sample metrics:\n",
" metrics = {\n",
" 'metrics': [{\n",
" 'name': 'quotient',\n",
" 'numberValue': float(quotient),\n",
" },{\n",
" 'name': 'remainder',\n",
" 'numberValue': float(remainder),\n",
" }]}\n",
"\n",
" with file_io.FileIO(output_dir + 'mlpipeline-metrics.json', 'w') as f:\n",
" json.dump(metrics, f)\n",
"\n",
" from collections import namedtuple\n",
" divmod_output = namedtuple('MyDivmodOutput', ['quotient', 'remainder'])\n",
" return divmod_output(quotient, remainder)"
Expand All @@ -128,20 +154,9 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"MyDivmodOutput(quotient=14, remainder=2)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"my_divmod(100, 7)"
]
Expand All @@ -157,7 +172,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -174,7 +189,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -193,7 +208,7 @@
" \n",
" #Passing a task output reference as operation arguments\n",
" #For an operation with a single return value, the output reference can be accessed using `task.output` or `task.outputs['output_name']` syntax\n",
" divmod_task = divmod_op(add_task.output, b)\n",
" divmod_task = divmod_op(add_task.output, b, '/')\n",
"\n",
" #For an operation with a multiple return values, the output references can be accessed using `task.outputs['output_name']` syntax\n",
" result_task = add_op(divmod_task.outputs['quotient'], c)"
Expand All @@ -208,7 +223,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -227,22 +242,9 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Run link <a href=\"/pipeline/#/runs/details/23daf747-e2d5-11e8-bcc5-42010a800195\" target=\"_blank\" >here</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"#Specify pipeline argument values\n",
"arguments = {'a': '7', 'b': '8'}\n",
Expand Down Expand Up @@ -276,7 +278,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
"version": "3.6.4"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 164b85f

Please sign in to comment.