Skip to content

Commit

Permalink
add mean/std to timeit decorator (#284)
Browse files Browse the repository at this point in the history 
* add mean/std to timeit decorator

* fix tests

* update docs

* update docstrings
  • Loading branch information
@PythonFZ
PythonFZ authored May 2, 2022
1 parent fb20d34 commit 5bb55e5
Show file tree
Hide file tree
Showing 4 changed files with 177 additions and 58 deletions.
106 changes: 80 additions & 26 deletions examples/docs/05_metadata.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
"cell_type": "markdown",
"id": "7be10adc-8677-4a69-87f6-489b10d7a989",
"metadata": {
"tags": []
"tags": [],
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"# Metadata collection with ZnTrack\n",
Expand All @@ -18,7 +21,11 @@
"cell_type": "code",
"execution_count": 1,
"id": "f0676d7e-7803-488c-9891-d500d55aa47f",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"from zntrack import Node, zn, config\n",
Expand All @@ -34,7 +41,10 @@
"id": "355f3eb2-b42b-43d6-aa18-29dfa579b742",
"metadata": {
"nbsphinx": "hidden",
"tags": []
"tags": [],
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
Expand All @@ -47,13 +57,17 @@
"cell_type": "code",
"execution_count": 3,
"id": "c2ea08fe-7113-421e-b6b3-71b4e048c724",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Initialized empty Git repository in /tmp/tmpiprbxls8/.git/\r\n",
"Initialized empty Git repository in /tmp/tmp4us1bcfh/.git/\r\n",
"Initialized DVC repository.\r\n",
"\r\n",
"You can now commit the changes to git.\r\n",
Expand Down Expand Up @@ -84,7 +98,11 @@
"cell_type": "code",
"execution_count": 4,
"id": "bf980834-b0d5-4be4-ac26-d653736645f6",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"class SleepNode(Node):\n",
Expand All @@ -107,15 +125,19 @@
"cell_type": "code",
"execution_count": 5,
"id": "e4e13211-1462-44d4-939a-9a96a057969e",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2022-02-22 13:49:02,925 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n",
"2022-04-29 15:25:27,153 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n",
"Submit issues to https://github.com/zincware/ZnTrack.\n",
"2022-02-22 13:49:06,330 (WARNING): Running DVC command: 'dvc run -n SleepNode ...'\n"
"2022-04-29 15:25:31,612 (WARNING): Running DVC command: 'dvc run -n SleepNode ...'\n"
]
}
],
Expand All @@ -127,14 +149,18 @@
"cell_type": "code",
"execution_count": 6,
"id": "d87965c0-b9c9-40cc-9b63-dc9047c42915",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" \rore\u001B[39m>Path metadata.run:timeit metadata.sleep_1s:timeit\r\n",
"nodes/SleepNode/metrics_no_cache.json 3.0023 1.00115\r\n",
"nodes/SleepNode/metrics_no_cache.json 3.00312 1.00053\r\n",
"\u001B[0m"
]
}
Expand All @@ -146,7 +172,11 @@
{
"cell_type": "markdown",
"id": "8d5fdb4c-ae21-47d9-8c2f-e1c89d48a1da",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"We can also time a sinlge function multiple times, using the following example:"
]
Expand All @@ -155,16 +185,21 @@
"cell_type": "code",
"execution_count": 7,
"id": "a689306e-4b24-44c7-a933-2482483090f9",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"class SleepNodeMulti(Node):\n",
" metadata = zn.metadata()\n",
"\n",
" @TimeIt\n",
" def run(self):\n",
" self.sleep(1)\n",
" self.sleep(2)\n",
" self.sleep(1.0)\n",
" self.sleep(0.8)\n",
" self.sleep(1.2)\n",
"\n",
" @TimeIt\n",
" def sleep(self, time):\n",
Expand All @@ -175,13 +210,17 @@
"cell_type": "code",
"execution_count": 8,
"id": "cbc74582-c997-4050-8076-199ef2f63df6",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2022-02-22 13:49:19,112 (WARNING): Running DVC command: 'dvc run -n SleepNodeMulti ...'\n"
"2022-04-29 15:25:43,644 (WARNING): Running DVC command: 'dvc run -n SleepNodeMulti ...'\n"
]
}
],
Expand All @@ -193,15 +232,19 @@
"cell_type": "code",
"execution_count": 9,
"id": "f8336ac2-bb26-45c7-9094-262f4de57126",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" \rore\u001B[39m>Path metadata.run:timeit metadata.sleep:timeit metadata.sleep_1:timeit metadata.sleep_1s:timeit\r\n",
"nodes/SleepNode/metrics_no_cache.json 3.0023 - - 1.00115\r\n",
"nodes/SleepNodeMulti/metrics_no_cache.json 3.00354 1.00103 2.00188 -\r\n",
" \rore\u001B[39m>Path metadata.run:timeit metadata.sleep:timeit.mean metadata.sleep:timeit.std metadata.sleep_1s:timeit\r\n",
"nodes/SleepNode/metrics_no_cache.json 3.00312 - - 1.00053\r\n",
"nodes/SleepNodeMulti/metrics_no_cache.json 3.00455 1.00099 0.20019 -\r\n",
"\u001B[0m"
]
}
Expand All @@ -213,20 +256,28 @@
{
"cell_type": "markdown",
"id": "aca25029-74ec-457c-9759-8b80ec06bce9",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"One can also access the metrics directly within Python. This is possible, because they are just another `zn.metrics` which is automatically added when using one of the given metadata decorators."
"One can also access the metrics directly within Python. This is possible, because they are just another `zn.metrics` which is automatically added when using one of the given metadata decorators. This also gives access to the individual run times which are hidden when using `dvc metrics show`."
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "6b6d4e62-f2db-4550-b89c-9012c3387cbf",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"data": {
"text/plain": "{'sleep:timeit': 1.0010313987731934,\n 'sleep_1:timeit': 2.0018763542175293,\n 'run:timeit': 3.00354266166687}"
"text/plain": "{'sleep:timeit': {'values': [1.0008997917175293,\n 0.8008444309234619,\n 1.2012319564819336],\n 'mean': 1.0009920597076416,\n 'std': 0.20019377872637664},\n 'run:timeit': 3.0045523643493652}"
},
"execution_count": 10,
"metadata": {},
Expand All @@ -243,7 +294,10 @@
"id": "35ed65a1-9590-43f8-8aa9-244fd187f823",
"metadata": {
"nbsphinx": "hidden",
"tags": []
"tags": [],
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
Expand Down
27 changes: 26 additions & 1 deletion tests/integration_tests/test_timeit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,36 @@ def run(self):
time.sleep(self.sleep_for)


class SleepClassLoop(Node):
sleep_for = zn.params(0.1)
timeit_metrics = zn.metadata()
iterations = zn.params(30)

def run(self):
for _ in range(30):
self.sleep()

@TimeIt
def sleep(self):
time.sleep(self.sleep_for)


def test_timeit_no_metadata_err(proj_path):
with pytest.raises(DescriptorMissing):
SleepClassNoMetadata().run_and_save()


def test_timeit(proj_path):
SleepClass().run_and_save()
assert SleepClass.load().timeit_metrics["run:timeit"] - 1.0 < 0.1
assert pytest.approx(SleepClass.load().timeit_metrics["run:timeit"], 0.1) == 1.0


def test_timeit_loop(proj_path):
sleep_class = SleepClassLoop()
sleep_class.run_and_save()
assert (
pytest.approx(SleepClassLoop.load().timeit_metrics["sleep:timeit"]["mean"], 0.01)
== 0.1
)
assert SleepClassLoop.load().timeit_metrics["sleep:timeit"]["std"] < 1e-3
assert len(SleepClassLoop.load().timeit_metrics["sleep:timeit"]["values"]) == 30
62 changes: 33 additions & 29 deletions zntrack/metadata/base.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import re
from abc import ABC, abstractmethod
from functools import partial
from typing import Callable
Expand Down Expand Up @@ -38,6 +37,37 @@ def __call__(self, cls, *args, **kwargs):
"""Actual decorator"""
raise NotImplementedError

@property
def name(self) -> str:
"""Get the name of the metric this decorator will use"""
return f"{self.func_name}:{self.name_of_metric}"

@staticmethod
def get_history(cls) -> (dict, str):
"""Get the values of the zn.metadata descriptor
Get a full metadata dict containing all previously collected metadata.
"""
try:
metadata_attr, metadata = next(
iter(
filter_ZnTrackOption(
data=cls._descriptor_list,
cls=cls,
zn_type=utils.ZnTypes.METADATA,
allow_none=True,
).items()
)
)
except StopIteration as error:
raise utils.exceptions.DescriptorMissing(
"Could not find a metadata descriptor. Please add zn.metadata()!"
) from error
if metadata is None:
metadata = {}

return metadata, metadata_attr

def __get__(self, instance, owner):
"""Converting decorator into descriptor
Expand Down Expand Up @@ -71,33 +101,7 @@ def save_metadata(self, cls, value):
If the Node does not contain a <zn.metadata> descriptor
"""

try:
metadata_attr, metadata = next(
iter(
filter_ZnTrackOption(
data=cls._descriptor_list,
cls=cls,
zn_type=utils.ZnTypes.METADATA,
allow_none=True,
).items()
)
)
except StopIteration as error:
raise utils.exceptions.DescriptorMissing(
"Could not find a metadata descriptor. Please add zn.metadata()!"
) from error
if metadata is None:
metadata = {}

pattern = re.compile(rf"{self.func_name}(_[1-9]+)?:{self.name_of_metric}")
metadata, metadata_attr = self.get_history(cls)
metadata[self.name] = value

number_already_collected = len(list(filter(pattern.match, metadata)))

if number_already_collected == 0:
metadata_name = f"{self.func_name}:{self.name_of_metric}"
else:
metadata_name = (
f"{self.func_name}_{number_already_collected}:{self.name_of_metric}"
)
metadata[metadata_name] = value
setattr(cls, metadata_attr, metadata)
Loading

0 comments on commit 5bb55e5

Please sign in to comment.