AnalysisCard refactor

ax/analysis/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ax.analysis.analysis import Analysis, AnalysisCard, AnalysisCardLevel
+from ax.analysis.markdown import *  # noqa
+from ax.analysis.plotly import *  # noqa
+
+__all__ = ["Analysis", "AnalysisCard", "AnalysisCardLevel"]

ax/analysis/analysis.py

@@ -0,0 +1,87 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from enum import Enum
+from typing import Any, Optional, Protocol
+
+import pandas as pd
+from ax.core.experiment import Experiment
+from ax.modelbridge.generation_strategy import GenerationStrategy
+
+
+class AnalysisCardLevel(Enum):
+    DEBUG = 0
+    LOW = 1
+    MID = 2
+    HIGH = 3
+    CRITICAL = 4
+
+
+class AnalysisCard:
+    # Name of the analysis computed, usually the class name of the Analysis which
+    # produced the card. Useful for grouping by when querying a large collection of
+    # cards.
+    name: str
+
+    title: str
+    subtitle: str
+    level: AnalysisCardLevel
+
+    df: pd.DataFrame  # Raw data produced by the Analysis
+
+    # pyre-ignore[4] We explicitly want to allow any type here, blob is narrowed in
+    # AnalysisCard's subclasses
+    blob: Any  # Data processed and ready for end-user consumption
+
+    # How to interpret the blob (ex. "dataframe", "plotly", "markdown")
+    blob_annotation = "dataframe"
+
+    def __init__(
+        self,
+        name: str,
+        title: str,
+        subtitle: str,
+        level: AnalysisCardLevel,
+        df: pd.DataFrame,
+        # pyre-ignore[2] We explicitly want to allow any type here, blob is narrowed in
+        # AnalysisCard's subclasses
+        blob: Any,
+    ) -> None:
+        self.name = name
+        self.title = title
+        self.subtitle = subtitle
+        self.level = level
+        self.df = df
+        self.blob = blob
+
+
+class Analysis(Protocol):
+    """
+    An Analysis is a class that given either and Experiment, a GenerationStrategy, or
+    both can compute some data intended for end-user consumption. The data is returned
+    to the user in the form of an AnalysisCard which contains the raw data, a blob (the
+    data processed for end-user consumption), and miscellaneous metadata that can be
+    useful for rendering the card or a collection of cards.
+
+    The AnalysisCard is a thin wrapper around the raw data and the processed blob;
+    Analyses impose structure on their blob should subclass Analysis. See
+    PlotlyAnalysis for an example which produces cards where the blob is always a
+    Plotly Figure object.
+
+    A good pattern to follow when implementing your own Analyses is to configure
+    "settings" (like which parameter or metrics to operate on, or whether to use
+    observed or modeled effects) in your Analyses' __init__ methods, then to consume
+    these settings in the compute method.
+    """
+
+    def compute(
+        self,
+        experiment: Optional[Experiment] = None,
+        generation_strategy: Optional[GenerationStrategy] = None,
+    ) -> AnalysisCard:
+        # Note: when implementing compute always prefer experiment.lookup_data() to
+        # experiment.fetch_data() to avoid unintential data fetching within the report
+        # generation.
+        ...

ax/analysis/markdown/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ax.analysis.markdown.markdown_analysis import (
+    MarkdownAnalysis,
+    MarkdownAnalysisCard,
+)
+
+__all__ = ["MarkdownAnalysis", "MarkdownAnalysisCard"]

ax/analysis/markdown/markdown_analysis.py

@@ -0,0 +1,38 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Optional
+
+import pandas as pd
+from ax.analysis.analysis import Analysis, AnalysisCard, AnalysisCardLevel
+from ax.core.experiment import Experiment
+from ax.modelbridge.generation_strategy import GenerationStrategy
+
+
+class MarkdownAnalysisCard(AnalysisCard):
+    name: str
+
+    title: str
+    subtitle: str
+    level: AnalysisCardLevel
+
+    df: pd.DataFrame
+    blob: str
+    blob_annotation = "markdown"
+
+    def get_markdown(self) -> str:
+        return self.blob
+
+
+class MarkdownAnalysis(Analysis):
+    """
+    An Analysis that computes a paragraph of Markdown formatted text.
+    """
+
+    def compute(
+        self,
+        experiment: Optional[Experiment] = None,
+        generation_strategy: Optional[GenerationStrategy] = None,
+    ) -> MarkdownAnalysisCard: ...

ax/analysis/analysis_report.py → ax/analysis/old/analysis_report.py

@@ -10,8 +10,8 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from ax.analysis.base_analysis import BaseAnalysis
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_analysis import BaseAnalysis
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
 from ax.core.experiment import Experiment
 

ax/analysis/base_plotly_visualization.py → ax/analysis/old/base_plotly_visualization.py

@@ -11,7 +11,7 @@
 
 import plotly.graph_objects as go
 
-from ax.analysis.base_analysis import BaseAnalysis
+from ax.analysis.old.base_analysis import BaseAnalysis
 from ax.core.experiment import Experiment
 
 

ax/analysis/cross_validation_plot.py → ax/analysis/old/cross_validation_plot.py

@@ -10,17 +10,17 @@
 
 import pandas as pd
 
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
-from ax.analysis.helpers.cross_validation_helpers import (
+from ax.analysis.old.helpers.cross_validation_helpers import (
     cv_results_to_df,
     diagonal_trace,
     get_plotting_limit_ignore_outliers,
 )
 
-from ax.analysis.helpers.layout_helpers import layout_format, updatemenus_format
+from ax.analysis.old.helpers.layout_helpers import layout_format, updatemenus_format
 
-from ax.analysis.helpers.scatter_helpers import (
+from ax.analysis.old.helpers.scatter_helpers import (
     error_scatter_trace_from_df,
     extract_mean_and_error_from_df,
 )

ax/analysis/helpers/cross_validation_helpers.py → ax/analysis/old/helpers/cross_validation_helpers.py

@@ -12,9 +12,9 @@
 import pandas as pd
 import plotly.graph_objs as go
 
-from ax.analysis.helpers.constants import Z
+from ax.analysis.old.helpers.constants import Z
 
-from ax.analysis.helpers.plot_helpers import arm_name_to_sort_key
+from ax.analysis.old.helpers.plot_helpers import arm_name_to_sort_key
 
 from ax.modelbridge.cross_validation import CVResult
 

ax/analysis/helpers/plot_helpers.py → ax/analysis/old/helpers/plot_helpers.py

@@ -10,7 +10,7 @@
 from logging import Logger
 from typing import Dict, List, Optional, Tuple, Union
 
-from ax.analysis.helpers.constants import DECIMALS, Z
+from ax.analysis.old.helpers.constants import DECIMALS, Z
 
 from ax.core.generator_run import GeneratorRun
 

ax/analysis/helpers/scatter_helpers.py → ax/analysis/old/helpers/scatter_helpers.py

@@ -14,11 +14,11 @@
 import pandas as pd
 
 import plotly.graph_objs as go
-from ax.analysis.helpers.color_helpers import rgba
+from ax.analysis.old.helpers.color_helpers import rgba
 
-from ax.analysis.helpers.constants import CI_OPACITY, COLORS, DECIMALS, Z
+from ax.analysis.old.helpers.constants import CI_OPACITY, COLORS, DECIMALS, Z
 
-from ax.analysis.helpers.plot_helpers import _format_CI, _format_dict
+from ax.analysis.old.helpers.plot_helpers import _format_CI, _format_dict
 
 from ax.core.types import TParameterization
 

ax/analysis/helpers/tests/test_cross_validation_helpers.py → ax/analysis/old/helpers/tests/test_cross_validation_helpers.py

@@ -10,9 +10,9 @@
 
 import plotly.graph_objects as go
 import plotly.io as pio
-from ax.analysis.cross_validation_plot import CrossValidationPlot
-from ax.analysis.helpers.constants import Z
-from ax.analysis.helpers.cross_validation_helpers import get_min_max_with_errors
+from ax.analysis.old.cross_validation_plot import CrossValidationPlot
+from ax.analysis.old.helpers.constants import Z
+from ax.analysis.old.helpers.cross_validation_helpers import get_min_max_with_errors
 from ax.modelbridge.registry import Models
 from ax.utils.common.testutils import TestCase
 from ax.utils.testing.core_stubs import get_branin_experiment

ax/analysis/helpers/tests/test_cv_consistency_checks.py → ax/analysis/old/helpers/tests/test_cv_consistency_checks.py

@@ -9,11 +9,11 @@
 import copy
 
 import plotly.graph_objects as go
-from ax.analysis.cross_validation_plot import CrossValidationPlot
-from ax.analysis.helpers.cross_validation_helpers import (
+from ax.analysis.old.cross_validation_plot import CrossValidationPlot
+from ax.analysis.old.helpers.cross_validation_helpers import (
     error_scatter_data_from_cv_results,
 )
-from ax.analysis.helpers.scatter_helpers import error_scatter_trace_from_df
+from ax.analysis.old.helpers.scatter_helpers import error_scatter_trace_from_df
 from ax.modelbridge.cross_validation import cross_validate
 from ax.modelbridge.registry import Models
 from ax.plot.base import PlotMetric

ax/analysis/parallel_coordinates_plot.py → ax/analysis/old/parallel_coordinates_plot.py

@@ -9,7 +9,7 @@
 
 import pandas as pd
 
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
 from ax.core.arm import Arm
 from ax.core.base_trial import BaseTrial

ax/analysis/predicted_outcomes_dot_plot.py → ax/analysis/old/predicted_outcomes_dot_plot.py

@@ -11,14 +11,14 @@
 
 import pandas as pd
 
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
-from ax.analysis.helpers.layout_helpers import updatemenus_format
+from ax.analysis.old.helpers.layout_helpers import updatemenus_format
 
-from ax.analysis.helpers.plot_data_df_helpers import get_plot_data_in_sample_arms_df
-from ax.analysis.helpers.plot_helpers import arm_name_to_sort_key, resize_subtitles
+from ax.analysis.old.helpers.plot_data_df_helpers import get_plot_data_in_sample_arms_df
+from ax.analysis.old.helpers.plot_helpers import arm_name_to_sort_key, resize_subtitles
 
-from ax.analysis.helpers.scatter_helpers import (
+from ax.analysis.old.helpers.scatter_helpers import (
     error_dot_plot_trace_from_df,
     relativize_dataframe,
 )

ax/analysis/tests/test_analysis_report.py → ax/analysis/old/tests/test_analysis_report.py

@@ -8,10 +8,10 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from ax.analysis.analysis_report import AnalysisReport
+from ax.analysis.old.analysis_report import AnalysisReport
 
-from ax.analysis.base_analysis import BaseAnalysis
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_analysis import BaseAnalysis
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
 from ax.modelbridge.registry import Models
 from ax.utils.common.testutils import TestCase

ax/analysis/tests/test_base_classes.py → ax/analysis/old/tests/test_base_classes.py

@@ -11,8 +11,8 @@
 
 import plotly.graph_objects as go
 
-from ax.analysis.base_analysis import BaseAnalysis
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_analysis import BaseAnalysis
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
 from ax.modelbridge.registry import Models
 

ax/analysis/tests/test_cross_validation_plot.py → ax/analysis/old/tests/test_cross_validation_plot.py

@@ -6,7 +6,7 @@
 # pyre-strict
 
 import plotly.graph_objects as go
-from ax.analysis.cross_validation_plot import CrossValidationPlot
+from ax.analysis.old.cross_validation_plot import CrossValidationPlot
 from ax.modelbridge.registry import Models
 from ax.utils.common.testutils import TestCase
 from ax.utils.testing.core_stubs import get_branin_experiment

ax/analysis/tests/test_parallel_coordinates_plot.py → ax/analysis/old/tests/test_parallel_coordinates_plot.py

@@ -6,7 +6,7 @@
 # pyre-strict
 
 
-from ax.analysis.parallel_coordinates_plot import ParallelCoordinatesPlot
+from ax.analysis.old.parallel_coordinates_plot import ParallelCoordinatesPlot
 from ax.core.batch_trial import BatchTrial
 
 from ax.utils.common.testutils import TestCase

ax/analysis/tests/test_predicted_outcomes_dot_plot.py → ax/analysis/old/tests/test_predicted_outcomes_dot_plot.py

@@ -9,7 +9,7 @@
 import unittest
 
 import plotly.graph_objects as go
-from ax.analysis.predicted_outcomes_dot_plot import PredictedOutcomesDotPlot
+from ax.analysis.old.predicted_outcomes_dot_plot import PredictedOutcomesDotPlot
 from ax.exceptions.core import UnsupportedPlotError
 from ax.modelbridge.registry import Models
 from ax.utils.testing.core_stubs import get_branin_experiment

ax/analysis/plotly/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ax.analysis.plotly.plotly_analysis import PlotlyAnalysis, PlotlyAnalysisCard
+
+__all__ = ["PlotlyAnalysis", "PlotlyAnalysisCard"]

ax/analysis/plotly/plotly_analysis.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Optional
+
+import pandas as pd
+from ax.analysis.analysis import Analysis, AnalysisCard, AnalysisCardLevel
+from ax.core.experiment import Experiment
+from ax.modelbridge.generation_strategy import GenerationStrategy
+from plotly import graph_objects as go
+
+
+class PlotlyAnalysisCard(AnalysisCard):
+    name: str
+
+    title: str
+    subtitle: str
+    level: AnalysisCardLevel
+
+    df: pd.DataFrame
+    blob: go.Figure
+    blob_annotation = "plotly"
+
+    def get_figure(self) -> go.Figure:
+        return self.blob
+
+
+class PlotlyAnalysis(Analysis):
+    """
+    An Analysis that computes a Plotly figure.
+    """
+
+    def compute(
+        self,
+        experiment: Optional[Experiment] = None,
+        generation_strategy: Optional[GenerationStrategy] = None,
+    ) -> PlotlyAnalysisCard: ...

ax/storage/sqa_store/decoder.py

@@ -16,8 +16,8 @@
 import pandas as pd
 import plotly.io as pio
 
-from ax.analysis.base_analysis import BaseAnalysis
-from ax.analysis.base_plotly_visualization import BasePlotlyVisualization
+from ax.analysis.old.base_analysis import BaseAnalysis
+from ax.analysis.old.base_plotly_visualization import BasePlotlyVisualization
 
 from ax.core.arm import Arm
 from ax.core.base_trial import BaseTrial, TrialStatus