Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: streamlit import error #686

Merged
merged 1 commit into from
Aug 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion nemoguardrails/eval/ui/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
plot_bar_series,
plot_matrix_series,
)
from nemoguardrails.eval.ui.streamlit_utils import load_eval_data
from nemoguardrails.eval.ui.utils import (
EvalData,
collect_interaction_metrics,
collect_interaction_metrics_with_expected_latencies,
load_eval_data,
)

# Disable SettingWithCopyWarning
Expand Down
3 changes: 2 additions & 1 deletion nemoguardrails/eval/ui/pages/0_Config.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
import streamlit as st

from nemoguardrails.eval.models import EvalConfig
from nemoguardrails.eval.ui.utils import EvalData, load_eval_data
from nemoguardrails.eval.ui.streamlit_utils import load_eval_data
from nemoguardrails.eval.ui.utils import EvalData


def _render_policies(eval_config: EvalConfig):
Expand Down
3 changes: 2 additions & 1 deletion nemoguardrails/eval/ui/pages/1_Review.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
import streamlit as st

from nemoguardrails.eval.models import ComplianceCheckResult, InteractionOutput, Policy
from nemoguardrails.eval.ui.utils import EvalData, get_span_colors, load_eval_data
from nemoguardrails.eval.ui.streamlit_utils import get_span_colors, load_eval_data
from nemoguardrails.eval.ui.utils import EvalData
from nemoguardrails.utils import new_uuid


Expand Down
78 changes: 78 additions & 0 deletions nemoguardrails/eval/ui/streamlit_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import random

import streamlit as st

from nemoguardrails.eval.models import EvalConfig, EvalOutput
from nemoguardrails.eval.ui.utils import EvalData
from nemoguardrails.eval.utils import get_output_paths


@st.cache_resource
def get_span_colors(_eval_output: EvalOutput):
"""Helper to get colors for the spans."""
random.seed(4)
colors = {}
for log in _eval_output.logs:
for span in reversed(log.trace):
if span.name not in colors:
colors[span.name] = "#" + "".join(
[random.choice("0123456789ABCDEF") for _ in range(6)]
)
return colors


@st.cache_resource
def load_eval_data():
"""Loads the evaluation data"""
# Setup argparse
parser = argparse.ArgumentParser()
parser.add_argument("--eval-config-path", type=str, default="")
parser.add_argument("--output-path", nargs="+", type=str, default=[])

# Parse arguments
args = parser.parse_args()

# Load the evaluation configuration
eval_config_path = os.path.abspath(args.eval_config_path)
eval_config = EvalConfig.from_path(eval_config_path)

# If no explicit output paths are provided, load all the output
# dirs from the current folder
if not args.output_path:
args.output_path = get_output_paths()

eval_outputs = {}
for output_path in args.output_path:
# We use relative paths to CWD to have them shorter in the UI
output_path = os.path.relpath(output_path, os.getcwd())

if os.path.basename(output_path).startswith("."):
continue

# Load the output
eval_output = EvalOutput.from_path(output_path)
eval_outputs[output_path] = eval_output

return EvalData(
eval_config_path=eval_config_path,
output_paths=args.output_path,
eval_config=eval_config,
eval_outputs=eval_outputs,
)
64 changes: 1 addition & 63 deletions nemoguardrails/eval/ui/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import random
from time import time
from typing import Dict, List, Optional, Union

import streamlit as st
from pydantic import BaseModel

from nemoguardrails.eval.models import (
Expand All @@ -29,11 +25,7 @@
InteractionOutput,
Span,
)
from nemoguardrails.eval.utils import (
_collect_span_metrics,
get_output_paths,
update_dict_at_path,
)
from nemoguardrails.eval.utils import _collect_span_metrics, update_dict_at_path


class EvalData(BaseModel):
Expand Down Expand Up @@ -72,60 +64,6 @@ def update_config_latencies(self):
print(f"Updating expected latencies took {time() - t0:.2f} seconds.")


@st.cache_resource
def get_span_colors(_eval_output: EvalOutput):
"""Helper to get colors for the spans."""
random.seed(4)
colors = {}
for log in _eval_output.logs:
for span in reversed(log.trace):
if span.name not in colors:
colors[span.name] = "#" + "".join(
[random.choice("0123456789ABCDEF") for _ in range(6)]
)
return colors


@st.cache_resource
def load_eval_data():
"""Loads the evaluation data"""
# Setup argparse
parser = argparse.ArgumentParser()
parser.add_argument("--eval-config-path", type=str, default="")
parser.add_argument("--output-path", nargs="+", type=str, default=[])

# Parse arguments
args = parser.parse_args()

# Load the evaluation configuration
eval_config_path = os.path.abspath(args.eval_config_path)
eval_config = EvalConfig.from_path(eval_config_path)

# If no explicit output paths are provided, load all the output
# dirs from the current folder
if not args.output_path:
args.output_path = get_output_paths()

eval_outputs = {}
for output_path in args.output_path:
# We use relative paths to CWD to have them shorter in the UI
output_path = os.path.relpath(output_path, os.getcwd())

if os.path.basename(output_path).startswith("."):
continue

# Load the output
eval_output = EvalOutput.from_path(output_path)
eval_outputs[output_path] = eval_output

return EvalData(
eval_config_path=eval_config_path,
output_paths=args.output_path,
eval_config=eval_config,
eval_outputs=eval_outputs,
)


def collect_interaction_metrics(
interaction_outputs: List[InteractionOutput],
) -> Dict[str, Union[int, float]]:
Expand Down