You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
importasyncioimporttimefrompydantic_evalsimportCase, Dataset# Create a dataset with multiple test casesdataset=Dataset(
cases=[
Case(
name=f'case_{i}',
inputs=i,
expected_output=i*2,
)
foriinrange(5)
]
)
asyncdefdouble_number(input_value: int) ->int:
"""Function that simulates work by sleeping for a second before returning double the input."""awaitasyncio.sleep(0.1) # Simulate workreturninput_value*2asyncdefmain():
# Run evaluation with unlimited concurrencyt0=time.time()
report_default=awaitdataset.evaluate(double_number)
print(f'Evaluation took less than 0.2s: {time.time() -t0<0.2}')
#> Evaluation took less than 0.2s: Truereport_default.print()
""" Evaluation Summary: double_number┏━━━━━━━━━━┳━━━━━━━━━━┓┃ Case ID ┃ Duration ┃┡━━━━━━━━━━╇━━━━━━━━━━┩│ case_0 │ 101.0ms │├──────────┼──────────┤│ case_1 │ 101.0ms │├──────────┼──────────┤│ case_2 │ 101.0ms │├──────────┼──────────┤│ case_3 │ 101.0ms │├──────────┼──────────┤│ case_4 │ 101.0ms │├──────────┼──────────┤│ Averages │ 101.0ms │└──────────┴──────────┘"""# Run evaluation with limited concurrencyt0=time.time()
report_limited=awaitdataset.evaluate(double_number, max_concurrency=1)
print(f'Evaluation took more than 0.5s: {time.time() -t0>0.5}')
#> Evaluation took more than 0.5s: Truereport_limited.print()
""" Evaluation Summary: double_number┏━━━━━━━━━━┳━━━━━━━━━━┓┃ Case ID ┃ Duration ┃┡━━━━━━━━━━╇━━━━━━━━━━┩│ case_0 │ 101.0ms │├──────────┼──────────┤│ case_1 │ 101.0ms │├──────────┼──────────┤│ case_2 │ 101.0ms │├──────────┼──────────┤│ case_3 │ 101.0ms │├──────────┼──────────┤│ case_4 │ 101.0ms │├──────────┼──────────┤│ Averages │ 101.0ms │└──────────┴──────────┘"""
The text was updated successfully, but these errors were encountered:
The text was updated successfully, but these errors were encountered: