Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DOC Autogeneration of getting started code #132

Merged
merged 7 commits into from
Sep 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions examples/plot_model_card.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,9 @@
" max_leaf_nodes and max_depth."
)
model_card_authors = "skops_user"
get_started_code = (
"import pickle\nwith open(pkl_filename, 'rb') as file:\n clf = pickle.load(file)"
)
citation_bibtex = "bibtex\n@inproceedings{...,year={2020}}"
model_card.add(
citation_bibtex=citation_bibtex,
get_started_code=get_started_code,
model_card_authors=model_card_authors,
limitations=limitations,
model_description=model_description,
Expand Down
12 changes: 11 additions & 1 deletion skops/card/_model_card.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def metadata_from_config(config_path: Union[str, Path]) -> CardData:
task = config.get("sklearn", {}).get("task", None)
if task:
card_data.tags += [task]

card_data.model_file = config.get("sklearn", {}).get("model", {}).get("file")
example_input = config.get("sklearn", {}).get("example_input", None)
# Documentation on what the widget expects:
# https://huggingface.co/docs/hub/models-widgets-examples
Expand Down Expand Up @@ -380,6 +380,16 @@ def _generate_card(self) -> ModelCard:
# add evaluation results

template_sections = copy.deepcopy(self._template_sections)
if self.metadata:
model_file = self.metadata.to_dict().get("model_file")
if model_file:
template_sections["get_started_code"] = (
"import joblib\nimport json\nimport pandas as pd\nclf ="
f' joblib.load({model_file})\nwith open("config.json") as f:\n '
" config ="
" json.load(f)\n"
'clf.predict(pd.DataFrame.from_dict(config["sklearn"]["example_input"]))'
)
template_sections["eval_results"] = tabulate(
list(self._eval_results.items()),
headers=["Metric", "Value"],
Expand Down
7 changes: 0 additions & 7 deletions skops/card/default_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,10 @@ You can find the details about evaluation process and the evaluation results.

Use the code below to get started with the model.

<details>
<summary> Click to expand </summary>

```python
{{ get_started_code | default("[More Information Needed]", true)}}
```

</details>




# Model Card Authors

Expand Down
50 changes: 33 additions & 17 deletions skops/card/tests/test_card.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,26 @@ def model_card(model_diagram=True):
yield card


@pytest.fixture
def model_card_metadata_from_config(destination_path):
X, y = load_iris(return_X_y=True, as_frame=True)
est = LogisticRegression(solver="liblinear").fit(X, y)
pkl_file = tempfile.mkstemp(suffix=".pkl", prefix="skops-test")[1]
with open(pkl_file, "wb") as f:
pickle.dump(est, f)
hub_utils.init(
model=pkl_file,
requirements=[f"scikit-learn=={sklearn.__version__}"],
dst=destination_path,
task="tabular-classification",
data=X,
)
card = Card(
est, model_diagram=True, metadata=metadata_from_config(destination_path)
)
yield card


@pytest.fixture
def destination_path():
with tempfile.TemporaryDirectory(prefix="skops-test") as dir_path:
Expand Down Expand Up @@ -109,24 +129,20 @@ def test_add_metrics(destination_path, model_card):
assert ("acc" in card) and ("f1" in card) and ("0.1" in card)


def test_metadata_from_config_tabular_data(destination_path):
def test_code_autogeneration(destination_path, model_card_metadata_from_config):
# test if getting started code is automatically generated
model_card_metadata_from_config.save(Path(destination_path) / "README.md")
metadata = metadata_load(local_path=Path(destination_path) / "README.md")
filename = metadata["model_file"]
with open(Path(destination_path) / "README.md") as f:
assert f"joblib.load({filename})" in f.read()


def test_metadata_from_config_tabular_data(
model_card_metadata_from_config, destination_path
):
# test if widget data is correctly set in the README
X, y = load_iris(return_X_y=True, as_frame=True)
est = LogisticRegression(solver="liblinear").fit(X, y)
pkl_file = tempfile.mkstemp(suffix=".pkl", prefix="skops-test")[1]
with open(pkl_file, "wb") as f:
pickle.dump(est, f)
hub_utils.init(
model=pkl_file,
requirements=[f"scikit-learn=={sklearn.__version__}"],
dst=destination_path,
task="tabular-classification",
data=X,
)
card = Card(
est, model_diagram=True, metadata=metadata_from_config(destination_path)
)
card.save(Path(destination_path) / "README.md")
model_card_metadata_from_config.save(Path(destination_path) / "README.md")
metadata = metadata_load(local_path=Path(destination_path) / "README.md")
assert "widget" in metadata

Expand Down