Skip to content

Commit 30d73f0

Browse files
isabelizimmisabelizimm
authored
Merge pull request #203 from rstudio/monitoring-dashboard
2 parents 3608744 + 27dce4d commit 30d73f0

File tree

7 files changed

+331
-25
lines changed

7 files changed

+331
-25
lines changed

‎docs/_quarto.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ website:
2424
file: reference/index.qmd
2525
- text: "Advanced Usage"
2626
menu:
27-
- custom_elements.qmd
27+
- custom_code.qmd
2828
- text: "Changelog"
2929
file: changelog.md
3030
- text: "Learn more"
@@ -59,7 +59,7 @@ quartodoc:
5959
- VetiverModel
6060
- vetiver_pin_write
6161
- vetiver_create_prototype
62-
- model_card.model_card
62+
- templates.model_card
6363

6464
- title: Deploy
6565
desc: ""
@@ -72,7 +72,7 @@ quartodoc:
7272
- predict
7373
- write_app
7474
- prepare_docker
75-
- write_docker.write_docker
75+
- write_docker
7676
- deploy_rsconnect
7777

7878
- title: Monitor
@@ -81,6 +81,7 @@ quartodoc:
8181
- compute_metrics
8282
- pin_metrics
8383
- plot_metrics
84+
- templates.monitoring_dashboard
8485

8586
- title: Model Handlers
8687
desc: ""

‎vetiver/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from .helpers import api_data_to_frame # noqa
2525
from .rsconnect import deploy_rsconnect # noqa
2626
from .monitor import compute_metrics, pin_metrics, plot_metrics, _rolling_df # noqa
27-
from .model_card import model_card # noqa
27+
from .templates import model_card, monitoring_dashboard # noqa
2828
from .types import create_prototype, Prototype # noqa
2929

3030
__author__ = "Isabel Zimmerman <isabel.zimmerman@posit.co>"

‎vetiver/model_card.py

-21
This file was deleted.

‎vetiver/templates.py

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
from importlib_resources import files as _files
2+
import shutil
3+
from pathlib import Path
4+
5+
6+
def model_card(path="."):
7+
"""
8+
Create a model card for documentation
9+
10+
Parameters
11+
----------
12+
path : str
13+
Path to save model card
14+
15+
Notes
16+
-----
17+
This model card is generated as a Quarto document. For more info on
18+
Quarto, visit https://quarto.org/
19+
"""
20+
src_path = _files("vetiver") / "templates/model_card.qmd"
21+
22+
return shutil.copy(src=src_path, dst=path)
23+
24+
25+
def monitoring_dashboard(path: str = "."):
26+
"""
27+
Generate a monitoring dashboard template
28+
29+
Parameters
30+
----------
31+
path : str
32+
Path to save monitoring dashboard
33+
34+
Notes
35+
-----
36+
This model card is generated as a Quarto document. For more info on
37+
Quarto, visit https://quarto.org/
38+
"""
39+
p = Path(path)
40+
src_path = p / _files("vetiver") / "templates" / "monitoring_dashboard.qmd"
41+
42+
return shutil.copy(src=src_path, dst=path)

‎vetiver/templates/monitoring_dashboard.qmd

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
---
2+
title: "Monitoring dashboard"
3+
format:
4+
dashboard:
5+
orientation: columns
6+
logo: https://github.com/rstudio/vetiver-python/blob/main/docs/figures/logo.png?raw=true
7+
output: asis
8+
jupyter: python3
9+
---
10+
11+
```{python}
12+
#| include: false
13+
#| tags: [parameters]
14+
15+
# import model and metadata
16+
import pins
17+
from IPython.display import display, Markdown, IFrame
18+
from datetime import datetime, timedelta
19+
import pandas as pd
20+
import plotly.express as px
21+
from sklearn import metrics
22+
from vetiver import VetiverModel, compute_metrics, plot_metrics
23+
from sklearn.metrics import recall_score, accuracy_score
24+
25+
raw = "https://colorado.rstudio.com/rsc"
26+
paths = {"chicago-model-python": "chicago-model-python/"}
27+
board = pins.board_url(raw, paths, allow_pickle_read=True)
28+
v = VetiverModel.from_pin(board, "chicago-model-python")
29+
v_meta = board.pin_meta("chicago-model-python")
30+
days_old = datetime.today() - datetime.strptime(v_meta.created, "%Y%m%dT%H%M%SZ")
31+
```
32+
33+
```{python}
34+
## the next few lines are an example model, here is a place to
35+
## add any code you need to import new data and make predictions
36+
37+
# import new data to track performance over time
38+
raw = "https://colorado.rstudio.com/rsc"
39+
paths = {"new-data": "inspections-new-data/"}
40+
board = pins.board_url(raw, paths, allow_pickle_read=True)
41+
inspections_new = board.pin_read("new-data")
42+
43+
# make predictions
44+
inspections_new["preds"] = v.model.predict(
45+
inspections_new.drop(columns=["results", "aka_name", "inspection_date"])
46+
)
47+
48+
# map results
49+
inspections_new["preds"] = inspections_new["preds"].map({"PASS": 0, "FAIL": 1})
50+
inspections_new["results"] = inspections_new["results"].map({"PASS": 0, "FAIL": 1})
51+
```
52+
53+
# Model info
54+
55+
## Column
56+
### Row {height="33%"}
57+
::: {.valuebox}
58+
`{python} v.description`
59+
60+
`{python} v.model_name`
61+
:::
62+
63+
::: {.valuebox}
64+
Model age
65+
66+
`{python} days_old.days` days old
67+
:::
68+
69+
### Row
70+
71+
Model details
72+
73+
- This model has the prototype:
74+
75+
```
76+
`{python} v.prototype.construct().schema().get("properties")`
77+
```
78+
79+
- The model was created by ...
80+
81+
# Model metrics
82+
83+
## Column
84+
```{python}
85+
import itables
86+
87+
td = timedelta(weeks = 4)
88+
metric_set = [accuracy_score, recall_score]
89+
90+
metrics_df = compute_metrics(
91+
data = inspections_new,
92+
date_var = "inspection_date",
93+
period = td,
94+
metric_set = metric_set,
95+
truth = "results",
96+
estimate = "preds"
97+
)
98+
itables.show(metrics_df)
99+
```
100+
101+
```{python}
102+
plot_metrics(metrics_df).show()
103+
```
104+
105+
## Column {.sidebar}
106+
107+
This tab is used to see model performance over time. In this context, _performance_ is the statistical properties of the model, eg, accuracy and recall.
108+
109+
You can add custom information and metrics here.
110+
111+
# Explore validation data
112+
113+
```{python}
114+
fig = px.histogram(inspections_new, x = "facility_type")
115+
fig.show()
116+
```
117+
118+
## Column {.sidebar}
119+
120+
Write your own code to make visualizations or tables with the new validation data, and/or the new predictions.
121+
122+
123+
# API visual documentation
124+
125+
## Column
126+
127+
```{python}
128+
from IPython.display import IFrame
129+
IFrame('https://colorado.posit.co/rsc/chicago-inspections-python', width=750, height=350)
130+
```
131+
---
132+
133+
134+
## Column {.sidebar}
135+
136+
Interact directly with your model via its visual documentation, and get `curl` examples.

‎vetiver/tests/snapshots/monitoring_dashboard.qmd

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
---
2+
title: "Monitoring dashboard"
3+
format:
4+
dashboard:
5+
orientation: columns
6+
logo: https://github.com/rstudio/vetiver-python/blob/main/docs/figures/logo.png?raw=true
7+
output: asis
8+
jupyter: python3
9+
---
10+
11+
```{python}
12+
#| include: false
13+
#| tags: [parameters]
14+
15+
# import model and metadata
16+
import pins
17+
from IPython.display import display, Markdown, IFrame
18+
from datetime import datetime, timedelta
19+
import pandas as pd
20+
import plotly.express as px
21+
from sklearn import metrics
22+
from vetiver import VetiverModel, compute_metrics, plot_metrics
23+
from sklearn.metrics import recall_score, accuracy_score
24+
25+
raw = "https://colorado.rstudio.com/rsc"
26+
paths = {"chicago-model-python": "chicago-model-python/"}
27+
board = pins.board_url(raw, paths, allow_pickle_read=True)
28+
v = VetiverModel.from_pin(board, "chicago-model-python")
29+
v_meta = board.pin_meta("chicago-model-python")
30+
days_old = datetime.today() - datetime.strptime(v_meta.created, "%Y%m%dT%H%M%SZ")
31+
```
32+
33+
```{python}
34+
## the next few lines are an example model, here is a place to
35+
## add any code you need to import new data and make predictions
36+
37+
# import new data to track performance over time
38+
raw = "https://colorado.rstudio.com/rsc"
39+
paths = {"new-data": "inspections-new-data/"}
40+
board = pins.board_url(raw, paths, allow_pickle_read=True)
41+
inspections_new = board.pin_read("new-data")
42+
43+
# make predictions
44+
inspections_new["preds"] = v.model.predict(
45+
inspections_new.drop(columns=["results", "aka_name", "inspection_date"])
46+
)
47+
48+
# map results
49+
inspections_new["preds"] = inspections_new["preds"].map({"PASS": 0, "FAIL": 1})
50+
inspections_new["results"] = inspections_new["results"].map({"PASS": 0, "FAIL": 1})
51+
```
52+
53+
# Model info
54+
55+
## Column
56+
### Row {height="33%"}
57+
::: {.valuebox}
58+
`{python} v.description`
59+
60+
`{python} v.model_name`
61+
:::
62+
63+
::: {.valuebox}
64+
Model age
65+
66+
`{python} days_old.days` days old
67+
:::
68+
69+
### Row
70+
71+
Model details
72+
73+
- This model has the prototype:
74+
75+
```
76+
`{python} v.prototype.construct().schema().get("properties")`
77+
```
78+
79+
- The model was created by ...
80+
81+
# Model metrics
82+
83+
## Column
84+
```{python}
85+
import itables
86+
87+
td = timedelta(weeks = 4)
88+
metric_set = [accuracy_score, recall_score]
89+
90+
metrics_df = compute_metrics(
91+
data = inspections_new,
92+
date_var = "inspection_date",
93+
period = td,
94+
metric_set = metric_set,
95+
truth = "results",
96+
estimate = "preds"
97+
)
98+
itables.show(metrics_df)
99+
```
100+
101+
```{python}
102+
plot_metrics(metrics_df).show()
103+
```
104+
105+
## Column {.sidebar}
106+
107+
This tab is used to see model performance over time. In this context, _performance_ is the statistical properties of the model, eg, accuracy and recall.
108+
109+
You can add custom information and metrics here.
110+
111+
# Explore validation data
112+
113+
```{python}
114+
fig = px.histogram(inspections_new, x = "facility_type")
115+
fig.show()
116+
```
117+
118+
## Column {.sidebar}
119+
120+
Write your own code to make visualizations or tables with the new validation data, and/or the new predictions.
121+
122+
123+
# API visual documentation
124+
125+
## Column
126+
127+
```{python}
128+
from IPython.display import IFrame
129+
IFrame('https://colorado.posit.co/rsc/chicago-inspections-python', width=750, height=350)
130+
```
131+
---
132+
133+
134+
## Column {.sidebar}
135+
136+
Interact directly with your model via its visual documentation, and get `curl` examples.

‎vetiver/tests/test_templates.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import vetiver
2+
from tempfile import TemporaryDirectory
3+
from pathlib import Path
4+
5+
6+
def test_monitoring_dashboard(snapshot):
7+
with TemporaryDirectory() as tempdir:
8+
file_path = Path(tempdir, "monitoring_dashboard.qmd")
9+
vetiver.monitoring_dashboard(path=file_path)
10+
snapshot.snapshot_dir = "./vetiver/tests/snapshots"
11+
contents = open(file_path).read()
12+
snapshot.assert_match(contents, "monitoring_dashboard.qmd")

0 commit comments

Comments
 (0)