Changes needed for Deepsparse.Analyze

src/sparsezoo/analyze/analysis.py

@@ -268,6 +268,11 @@ class BenchmarkResult(YAMLSerializableBaseModel):
         description="Node level inference results",
     )
 
+    supported_graph_percentage: Optional[float] = Field(
+        default=None,
+        description="Percentage of model graph supported by the runtime engine"
+    )
+
 
 class NodeAnalysis(YAMLSerializableBaseModel):
     """
@@ -625,19 +630,21 @@ def quant_count(self):
         return sum(item.quantized for item in self.items)
 
     @property
-    def sparsity(self):
+    def sparsity_percent(self):
         return round(self.sparse_count * 100.0 / self.total, self._precision)
 
     @property
-    def quantized(self):
+    def quantized_percent(self):
         return round(self.quant_count * 100.0 / self.total, self._precision)
 
     @property
     def size(self):
         """
         :return: size in bits ignoring zeros
         """
-        return self.quant_count * 8 + self.dense_count * 32
+        return (self.quant_count * 8 + self.dense_count * 32) * (
+            1 - self.sparsity_percent / 100.0
+        )
 
     def __add__(self, other):
         new_items = self.items + other.items
@@ -710,8 +717,8 @@ def from_model_analysis(
                     name=node.name,
                     total=node_count_summary.total,
                     size=node_count_summary.size,
-                    sparsity=node_count_summary.sparsity,
-                    quantized=node_count_summary.quantized,
+                    sparsity=node_count_summary.sparsity_percent,
+                    quantized=node_count_summary.quantized_percent,
                 )
                 by_layers_entries.append(entry)
             if by_layers_entries:
@@ -731,8 +738,8 @@ def from_model_analysis(
                     model=analysis.model_name,
                     count=param_count_summary.total,
                     size=param_count_summary.size,
-                    sparsity=param_count_summary.sparsity,
-                    quantized=param_count_summary.quantized,
+                    sparsity=param_count_summary.sparsity_percent,
+                    quantized=param_count_summary.quantized_percent,
                 ),
             ],
         )
@@ -746,8 +753,8 @@ def from_model_analysis(
                     model=analysis.model_name,
                     count=ops_count_summary.total,
                     size=ops_count_summary.size,
-                    sparsity=ops_count_summary.sparsity,
-                    quantized=ops_count_summary.quantized,
+                    sparsity=ops_count_summary.sparsity_percent,
+                    quantized=ops_count_summary.quantized_percent,
                 ),
             ],
         )
@@ -760,17 +767,17 @@ def from_model_analysis(
                 entry = TypedEntry(
                     type=item.name,
                     size=item_count_summary.size,
-                    sparsity=item_count_summary.sparsity,
-                    quantized=item_count_summary.quantized,
+                    sparsity=item_count_summary.sparsity_percent,
+                    quantized=item_count_summary.quantized_percent,
                 )
                 entries.append(entry)
 
             entries.append(
                 TypedEntry(
                     type="Total",
                     size=param_count_summary.size,
-                    sparsity=param_count_summary.sparsity,
-                    quantized=param_count_summary.quantized,
+                    sparsity=param_count_summary.sparsity_percent,
+                    quantized=param_count_summary.quantized_percent,
                 )
             )
 
@@ -786,17 +793,17 @@ def from_model_analysis(
                 entry = TypedEntry(
                     type=item.name,
                     size=item_count_summary.size,
-                    sparsity=item_count_summary.sparsity,
-                    quantized=item_count_summary.quantized,
+                    sparsity=item_count_summary.sparsity_percent,
+                    quantized=item_count_summary.quantized_percent,
                 )
                 entries.append(entry)
 
             entries.append(
                 TypedEntry(
                     type="Total",
                     size=ops_count_summary.size,
-                    sparsity=ops_count_summary.sparsity,
-                    quantized=ops_count_summary.quantized,
+                    sparsity=ops_count_summary.sparsity_percent,
+                    quantized=ops_count_summary.quantized_percent,
                 )
             )
             type_ops_section = Section(section_name="Ops by types", entries=entries)
@@ -812,8 +819,8 @@ def from_model_analysis(
                 entries=[
                     ModelEntry(
                         model=analysis.model_name,
-                        sparsity=overall_count_summary.sparsity,
-                        quantized=overall_count_summary.quantized,
+                        sparsity=overall_count_summary.sparsity_percent,
+                        quantized=overall_count_summary.quantized_percent,
                     )
                 ],
             )
@@ -823,12 +830,15 @@ def from_model_analysis(
                 section_name="Overall",
                 entries=[
                     PerformanceEntry(
-                        model=idx,
-                        sparsity=overall_count_summary.sparsity,
-                        quantized=overall_count_summary.quantized,
+                        model=analysis.model_name,
+                        sparsity=overall_count_summary.sparsity_percent,
+                        quantized=overall_count_summary.quantized_percent,
                         latency=benchmark_result.average_latency,
                         throughput=benchmark_result.items_per_second,
-                        supported_graph=0.0,  # TODO: fill in correct value
+                        supported_graph=(
+                                benchmark_result.supported_graph_percentage
+                                or 0.0
+                        ),
                     )
                     for idx, benchmark_result in enumerate(analysis.benchmark_results)
                 ],

src/sparsezoo/analyze/utils/models.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Dict, List, Optional, Union
+import textwrap
+from typing import Dict, List, Optional, Tuple, Union
 
 from pydantic import BaseModel, Field
 
@@ -237,6 +238,11 @@ class Entry(BaseModel):
     _print_order: List[str] = []
 
     def __sub__(self, other):
+        """
+        Allows base functionality for all inheriting classes to be subtract-able,
+        subtracts the fields of self with other while providing some additional
+        support for string and unrolling list type fields
+        """
         my_fields = self.__fields__
         other_fields = other.__fields__
 
@@ -253,7 +259,11 @@ def __sub__(self, other):
             if field == "section_name":
                 new_fields[field] = my_value
             elif isinstance(my_value, str):
-                new_fields[field] = f"{my_value} - {other_value}"
+                new_fields[field] = (
+                    my_value
+                    if my_value == other_value
+                    else f"{my_value} - {other_value}"
+                )
             elif isinstance(my_value, list):
                 new_fields[field] = [
                     item_a - item_b for item_a, item_b in zip(my_value, other_value)
@@ -263,27 +273,26 @@ def __sub__(self, other):
 
         return self.__class__(**new_fields)
 
-    def pretty_print(self, headers: bool = False):
-        column_width = 15
+    def pretty_print(self, headers: bool = False, column_width=30):
+        """
+        pretty print current Entry object with all it's fields
+        """
         field_names = self._print_order
         field_values = []
         for field_name in field_names:
             field_value = getattr(self, field_name)
             if isinstance(field_value, float):
                 field_value = f"{field_value:.2f}"
-            if field_name == "model":
-                field_value = field_value[-40:]
             field_values.append(field_value)
 
-        column_fmt = "{{:>{0}}} ".format(column_width)
-        fmt_string = "{:>40}" + (column_fmt * (len(field_names) - 1))
-
         if headers:
             print(
-                fmt_string.format(*(field_name.upper() for field_name in field_names))
+                multiline_pretty_print(
+                    row=[field_name.upper() for field_name in field_names],
+                    column_width=column_width,
+                )
             )
-
-        print(fmt_string.format(*field_values))
+        print(multiline_pretty_print(row=field_values, column_width=column_width))
 
 
 class BaseEntry(Entry):
@@ -365,17 +374,125 @@ class Section(Entry):
     Represents a list of Entries with an optional name
     """
 
-    entries: List[Union[NamedEntry, TypedEntry, SizedModelEntry, ModelEntry, BaseEntry]]
+    entries: List[
+        Union[
+            PerformanceEntry,
+            NamedEntry,
+            TypedEntry,
+            SizedModelEntry,
+            ModelEntry,
+            BaseEntry,
+        ]
+    ]
 
     section_name: str = ""
 
     def pretty_print(self):
+        """
+        pretty print current section, with its entries
+        """
         if self.section_name:
-            print(f"{self.section_name}:")
+            if not self.entries:
+                print(f"No entries found in: {self.section_name}")
+            else:
+                print(f"{self.section_name}:")
 
         for idx, entry in enumerate(self.entries):
             if idx == 0:
                 entry.pretty_print(headers=True)
             else:
                 entry.pretty_print(headers=False)
         print()
+
+    def __sub__(self, other: "Section"):
+        """
+        A method that allows us to subtract two Section objects,
+        If the section includes `NamedEntry` or `TypedEntry` then we only compare
+        the entries which have the same name or type (and others will be ignored),
+        Subtraction of other Entry types is delegated to their own implementation
+        This function also assumes that a Section has entries of the same type
+        """
+
+        if not isinstance(other, Section):
+            raise TypeError(
+                f"unsupported operand type(s) for -: {type(self)} and {type(other)}"
+            )
+
+        section_name = self.section_name or ""
+        self_entries, other_entries = self.get_comparable_entries(other)
+
+        compared_entries = [
+            self_entry - other_entry
+            for self_entry, other_entry in zip(self_entries, other_entries)
+        ]
+
+        return Section(
+            section_name=section_name,
+            entries=compared_entries,
+        )
+
+    def get_comparable_entries(self, other: "Section") -> Tuple[List[Entry], ...]:
+        """
+        Get comparable entries by same name or type if they belong to
+        `NamedEntry` or `TypedEntry`, else return all entries
+
+        :return: A tuple composed of two lists, containing comparable entries
+            in correct order from current and other Section objects
+        """
+        assert self.entries
+        entry_type_to_extractor = {
+            "NamedEntry": lambda entry: entry.name,
+            "TypedEntry": lambda entry: entry.type,
+        }
+        entry_type = self.entries[0].__class__.__name__
+
+        if entry_type not in entry_type_to_extractor:
+            return self.entries, other.entries
+
+        key_extractor = entry_type_to_extractor[entry_type]
+        self_entry_dict = {key_extractor(entry): entry for entry in self.entries}
+        other_entry_dict = {key_extractor(entry): entry for entry in other.entries}
+
+        self_comparable_entries = []
+        other_comparable_entries = []
+
+        for key, value in self_entry_dict.items():
+            if key in other_entry_dict:
+                self_comparable_entries.append(value)
+                other_comparable_entries.append(other_entry_dict[key])
+
+        if len(self_comparable_entries) != len(self_entry_dict):
+            _LOGGER.info(
+                "Found mismatching entries, these will be ignored during "
+                f"comparison in Section: {self.section_name}"
+            )
+        return self_comparable_entries, other_comparable_entries
+
+
+def multiline_pretty_print(row: List[str], column_width=20) -> str:
+    """
+    Formats the contents of the specified row into a multiline string which
+    each column is wrapped into a multiline string if its length is greater
+    than the specified column_width
+
+    :param row: A list of strings to be formatted into a multiline row
+    :param column_width: The max width of each column for formatting, default is 20
+    :returns: A multiline formatted string representing the row,
+    """
+    row = [str(column) for column in row]
+    result_string = ""
+    col_delim = " "
+    wrapped_row = [textwrap.wrap(col, column_width) for col in row]
+    max_lines_needed = max(len(col) for col in wrapped_row)
+
+    for line_idx in range(max_lines_needed):
+        result_string += col_delim
+        for column in wrapped_row:
+            if line_idx < len(column):
+                result_string += column[line_idx].ljust(column_width)
+            else:
+                result_string += " " * column_width
+            result_string += col_delim
+        if line_idx < max_lines_needed - 1:
+            result_string += "\n"
+    return result_string

src/sparsezoo/analyze_cli.py

@@ -72,7 +72,7 @@
 __all__ = ["main"]
 
 
-LOGGER = logging.getLogger()
+_LOGGER = logging.getLogger(__name__)
 
 
 @click.command(context_settings=CONTEXT_SETTINGS)
@@ -106,9 +106,9 @@ def main(
                 f"--{unimplemented_feat} has not been implemented yet"
             )
 
-    LOGGER.info("Starting Analysis ...")
+    _LOGGER.info("Starting Analysis ...")
     analysis = ModelAnalysis.create(model_path)
-    LOGGER.info("Analysis complete, collating results...")
+    _LOGGER.info("Analysis complete, collating results...")
 
     by_types: bool = convert_to_bool(by_types)
     by_layers: bool = convert_to_bool(by_layers)
@@ -125,18 +125,20 @@ def main(
         else:
             compare = [compare]
 
-        print("Comparision Results")
+        print("Comparison Analysis!!!")
         for model_to_compare in compare:
             compare_model_analysis = ModelAnalysis.create(model_to_compare)
             summary_comparison_model = compare_model_analysis.summary(
                 by_types=by_types,
                 by_layers=by_layers,
             )
+            print(f"Comparing {model_path} with {model_to_compare}")
+            print("Note: comparison analysis displays differences b/w models")
             comparison = summary - summary_comparison_model
             comparison.pretty_print()
 
     if save:
-        LOGGER.info(f"Writing results to {save}")
+        _LOGGER.info(f"Writing results to {save}")
         analysis.yaml(file_path=save)