Use snapshot based tests for printers

scripts/ci_test_printers.sh

@@ -1,34 +1,9 @@
 #!/usr/bin/env bash
 
-### Test printer
+### Test the EVM printer
 
-cd tests/e2e/solc_parsing/test_data/compile/ || exit
-
-# Do not test the evm printer,as it needs a refactoring
-ALL_PRINTERS="cfg,constructor-calls,contract-summary,data-dependency,echidna,function-id,function-summary,modifiers,call-graph,halstead,human-summary,inheritance,inheritance-graph,loc,martin,slithir,slithir-ssa,vars-and-auth,require,variable-order,declaration,ck"
-
-# Only test 0.5.17 to limit test time
-for file in *0.5.17-compact.zip; do
-  if ! slither "$file" --print "$ALL_PRINTERS" ; then
-    echo "Printer failed"
-    echo "$file"
-    exit 1
-  fi
-done
-
-# Only test 0.8.12 to limit test time
-for file in *0.8.12-compact.zip; do
-  if ! slither "$file" --print "declaration" ; then
-    echo "Printer failed"
-    echo "$file"
-    exit 1
-  fi
-done
-
-cd ../../.. || exit
-# Needed for evm printer
 pip install evm-cfg-builder
 solc-select use "0.5.1"
 if ! slither examples/scripts/test_evm_api.sol --print evm; then
   echo "EVM printer failed"
-fi
+fi

slither/printers/call/call_graph.py

@@ -74,7 +74,7 @@ def _process_internal_call(
 
 
 def _render_external_calls(external_calls: Set[str]) -> str:
-    return "\n".join(external_calls)
+    return "\n".join(sorted(external_calls))
 
 
 def _render_internal_calls(
@@ -87,8 +87,8 @@ def _render_internal_calls(
     lines.append(f"subgraph {_contract_subgraph(contract)} {{")
     lines.append(f'label = "{contract.name}"')
 
-    lines.extend(contract_functions[contract])
-    lines.extend(contract_calls[contract])
+    lines.extend(sorted(contract_functions[contract]))
+    lines.extend(sorted(contract_calls[contract]))
 
     lines.append("}")
 
@@ -101,8 +101,8 @@ def _render_solidity_calls(solidity_functions: Set[str], solidity_calls: Set[str
     lines.append("subgraph cluster_solidity {")
     lines.append('label = "[Solidity]"')
 
-    lines.extend(solidity_functions)
-    lines.extend(solidity_calls)
+    lines.extend(sorted(solidity_functions))
+    lines.extend(sorted(solidity_calls))
 
     lines.append("}")
 
@@ -205,7 +205,7 @@ def _process_functions(functions: Sequence[Function]) -> str:
             )
 
     render_internal_calls = ""
-    for contract in all_contracts:
+    for contract in sorted(all_contracts, key=lambda x: x.name):
         render_internal_calls += _render_internal_calls(
             contract, contract_functions, contract_calls
         )
@@ -230,16 +230,12 @@ def output(self, filename: str) -> Output:
             filename(string)
         """
 
-        all_contracts_filename = ""
-        if not filename.endswith(".dot"):
-            if filename in ("", "."):
-                filename = ""
-            else:
-                filename += "."
-            all_contracts_filename = f"{filename}all_contracts.call-graph.dot"
-
-        if filename == ".dot":
-            all_contracts_filename = "all_contracts.dot"
+        if filename in ("", "."):
+            all_contracts_filename = "all_contracts.call-graph.dot"
+        elif not filename.endswith(".dot"):
+            all_contracts_filename = f"{filename}.all_contracts.call-graph.dot"
+        else:
+            all_contracts_filename = filename
 
         info = ""
         results = []
@@ -256,7 +252,7 @@ def output(self, filename: str) -> Output:
             }
             content = "\n".join(
                 ["strict digraph {"]
-                + [_process_functions(list(all_functions_as_dict.values()))]
+                + [_process_functions(sorted(all_functions_as_dict.values(), key=lambda x: x.name))]
                 + ["}"]
             )
             f.write(content)

slither/printers/guidance/echidna.py

@@ -156,15 +156,8 @@ def _extract_assert(contracts: List[Contract]) -> Dict[str, Dict[str, List[Dict]
 
 # Create a named tuple that is serialization in json
 def json_serializable(cls):
-    # pylint: disable=unnecessary-comprehension
-    # TODO: the next line is a quick workaround to prevent pylint from crashing
-    # It can be removed once https://github.com/PyCQA/pylint/pull/3810 is merged
-    my_super = super
-
     def as_dict(self):
-        yield {
-            name: value for name, value in zip(self._fields, iter(my_super(cls, self).__iter__()))
-        }
+        yield dict(zip(self._fields, super(cls, self).__iter__()))
 
     cls.__iter__ = as_dict
     return cls
@@ -263,13 +256,15 @@ def _extract_constants(
                 context_explored,
             )
 
-            # Note: use list(set()) instead of set
-            # As this is meant to be serialized in JSON, and JSON does not support set
+            # Note: use sorted(set()) instead of set
+            # As this is meant to be serialized in JSON, and JSON does not support set. Moreover,
+            # we want stable results so the printer output is always the same for the same file
             if all_cst_used:
-                ret_cst_used[contract.name][_get_name(function)] = list(set(all_cst_used))
+                ret_cst_used[contract.name][_get_name(function)] = sorted(set(all_cst_used))
             if all_cst_used_in_binary:
                 ret_cst_used_in_binary[contract.name][_get_name(function)] = {
-                    k: list(set(v)) for k, v in all_cst_used_in_binary.items()
+                    k: sorted(set(all_cst_used_in_binary[k]))
+                    for k in sorted(all_cst_used_in_binary)
                 }
     return ret_cst_used, ret_cst_used_in_binary
 
@@ -315,7 +310,7 @@ def _have_external_calls(contracts: List[Contract]) -> Dict[str, List[str]]:
             if function.all_high_level_calls() or function.all_low_level_calls():
                 ret[contract.name].append(_get_name(function))
         if contract.name in ret:
-            ret[contract.name] = list(set(ret[contract.name]))
+            ret[contract.name] = sorted(set(ret[contract.name]))
     return ret
 
 
@@ -334,7 +329,7 @@ def _use_balance(contracts: List[Contract]) -> Dict[str, List[str]]:
                 ):
                     ret[contract.name].append(_get_name(function))
         if contract.name in ret:
-            ret[contract.name] = list(set(ret[contract.name]))
+            ret[contract.name] = sorted(set(ret[contract.name]))
     return ret
 
 
@@ -441,9 +436,9 @@ def output(self, filename: str) -> Output:  # pylint: disable=too-many-locals
 
         use_balance = _use_balance(contracts)
 
-        with_fallback = list(_with_fallback(contracts))
+        with_fallback = sorted(_with_fallback(contracts))
 
-        with_receive = list(_with_receive(contracts))
+        with_receive = sorted(_with_receive(contracts))
 
         d = {
             "payable": payable,

slither/slither.py

@@ -11,7 +11,6 @@
 from slither.printers.abstract_printer import AbstractPrinter
 from slither.solc_parsing.slither_compilation_unit_solc import SlitherCompilationUnitSolc
 from slither.vyper_parsing.vyper_compilation_unit import VyperCompilationUnit
-from slither.utils.output import Output
 from slither.vyper_parsing.ast.ast import parse
 
 logger = logging.getLogger("Slither")
@@ -294,7 +293,7 @@ def run_detectors(self) -> List[Dict]:
         self.write_results_to_hide()
         return results
 
-    def run_printers(self) -> List[Output]:
+    def run_printers(self) -> List[Dict]:
         """
         :return: List of registered printers outputs.
         """

tests/e2e/printers/README.md

@@ -0,0 +1,21 @@
+# Testing printers
+
+The printers are tested with files from the test data of `solc_parsing/compile` directory.
+
+They are automatically detected if added to the `all_detectors` element in `Slither`.
+
+## Snapshots
+
+The snapshots are generated using [pytest-insta](https://github.com/vberlier/pytest-insta).
+
+To update snapshots, run:
+
+```shell
+pytest tests/e2e/printers/test_printers.py --insta update
+```
+
+This will create text files in `tests/e2e/printers/snapshots/` containing the expected output of printers.
+
+
+
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_call_graph__0.txt

@@ -0,0 +1,21 @@
+Call Graph: tmp.zip.all_contracts.call-graph.dot
+Call Graph: tmp.zipC.call-graph.dot
+
+strict digraph {
+subgraph cluster_29_C {
+label = "C"
+"29_f" [label="f"]
+"29_f" -> "29_a"
+}subgraph cluster_solidity {
+label = "[Solidity]"
+}
+}
+strict digraph {
+subgraph cluster_29_C {
+label = "C"
+"29_f" [label="f"]
+"29_f" -> "29_a"
+}subgraph cluster_solidity {
+label = "[Solidity]"
+}
+}

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_cfg__0.txt

@@ -0,0 +1,42 @@
+Export tmp.zip-C-f().dot
+Export tmp.zip-C-a().dot
+
+digraph{
+0[label="Node Type: ENTRY_POINT 0
+"];
+0->5;
+1[label="Node Type: INLINE ASM 1
+"];
+1->2;
+2[label="Node Type: END INLINE ASM 2
+"];
+2->3;
+3[label="Node Type: INLINE ASM 3
+"];
+3->4;
+4[label="Node Type: END INLINE ASM 4
+"];
+5[label="Node Type: EXPRESSION 5
+
+EXPRESSION:
+a()
+
+IRs:
+MODIFIER_CALL, C.a()()"];
+5->1;
+}
+
+digraph{
+0[label="Node Type: ENTRY_POINT 0
+"];
+0->1;
+1[label="Node Type: INLINE ASM 1
+"];
+1->2;
+2[label="Node Type: END INLINE ASM 2
+"];
+2->3;
+3[label="Node Type: _ 3
+"];
+}
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_ck__0.txt

@@ -0,0 +1,49 @@
+
+
+CK complexity metrics (Variables):
++----------+-----------------+-----------+------------+
+| Contract | State variables | Constants | Immutables |
++----------+-----------------+-----------+------------+
+| C        |               0 |         0 |          0 |
+| TOTAL    |               0 |         0 |          0 |
++----------+-----------------+-----------+------------+
+
+
+CK complexity metrics (Function visibility):
++----------+--------+----------+----------+---------+
+| Contract | Public | External | Internal | Private |
++----------+--------+----------+----------+---------+
+| C        |      1 |        0 |        0 |       0 |
+| TOTAL    |      1 |        0 |        0 |       0 |
++----------+--------+----------+----------+---------+
+
+
+CK complexity metrics (State mutability):
++----------+----------+------+------+
+| Contract | Mutating | View | Pure |
++----------+----------+------+------+
+| C        |        1 |    0 |    0 |
+| TOTAL    |        1 |    0 |    0 |
++----------+----------+------+------+
+
+
+CK complexity metrics (External mutating functions):
++----------+-------------------+----------------------+--------------+
+| Contract | External mutating | No auth or onlyOwner | No modifiers |
++----------+-------------------+----------------------+--------------+
+| C        |                 1 |                    1 |            0 |
+| TOTAL    |                 1 |                    1 |            0 |
++----------+-------------------+----------------------+--------------+
+
+
+CK complexity metrics (Core):
+RFC: Response For a Class
+NOC: Number of Children
+DIT: Depth of Inheritance Tree
+CBO: Coupling Between Object Classes
++----------+-----------+-----+-----+-----+-----+
+| Contract | Ext calls | RFC | NOC | DIT | CBO |
++----------+-----------+-----+-----+-----+-----+
+| C        |         0 |   1 |   0 |   0 |   0 |
++----------+-----------+-----+-----+-----+-----+
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_constructor_calls__0.txt

@@ -0,0 +1 @@
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_contract_summary__0.txt

@@ -0,0 +1,5 @@
+
++ Contract C (Most derived contract)
+  - From C
+    - f() (public)
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_data_dependency__0.txt

@@ -0,0 +1,17 @@
+
+Contract C
++----------+--------------+
+| Variable | Dependencies |
++----------+--------------+
++----------+--------------+
+
+Function f()
++----------+--------------+
+| Variable | Dependencies |
++----------+--------------+
++----------+--------------+
+Function a()
++----------+--------------+
+| Variable | Dependencies |
++----------+--------------+
++----------+--------------+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_declaration__0.txt

@@ -0,0 +1,17 @@
+
+# Contracts
+# C
+	- Declaration: tests/e2e/solc_parsing/test_data/assembly-all.sol#1 (10 - 12)
+	- Implementation(s): []
+	- References: []
+
+	## Function
+		- C.f()
+			- Declaration: tests/e2e/solc_parsing/test_data/assembly-all.sol#9 (14 - 16)
+			- Implementation(s): ['tests/e2e/solc_parsing/test_data/assembly-all.sol#9-17 (5 - 6)']
+			- References: []
+
+	## State variables
+
+	## Structures
+

tests/e2e/printers/snapshots/printers__printer_assembly_all_sol_0_5_17_compact_zip_dominator__0.txt

@@ -0,0 +1,5 @@
+Export tmp.zip-C-f().dot
+Export tmp.zip-C-a().dot
+
+None
+None