Add bench command

.gitignore

@@ -9,3 +9,5 @@ __pycache/
 /build/
 /dist/
 /venv/
+*/__pycache__
+*/*/__pycache__

README.md

@@ -183,6 +183,20 @@ Positive is advantage white, negative is advantage black
 {"type":"mate", "value":-3}
 ```
 
+### Run benchmark 
+```python 
+stockfish.benchmark()
+```
+This will run the bench command with default settings. kwargs can be supplied as options.
+```text
+ttSize: int -> Transposition Table size in MB (max 2048)
+threads: int -> Number of search threads that should be used (max 512)
+limit: int -> Limit value of limitType spent for each position (max 10000)
+fenFile: str -> Path to a FEN format file containing positions to bench (path/to/file.fen)
+limitType: str -> Type of the limit used with limit value (depth, perft, nodes, movetime)
+evalType: str -> Evaluation type used (mixed, classical, NNUE)
+```
+
 ### Get current major version of stockfish engine
 ```python 
 stockfish.get_stockfish_major_version()

stockfish/models.py

@@ -6,8 +6,9 @@
 """
 
 import subprocess
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Dict
 import copy
+from os import path
 
 
 class Stockfish:
@@ -33,7 +34,11 @@ def __init__(
             "UCI_Elo": 1350,
         }
         self.stockfish = subprocess.Popen(
-            path, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+            path,
+            universal_newlines=True,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
         )
 
         self._stockfish_major_version: int = int(
@@ -91,6 +96,7 @@ def _read_line(self) -> str:
 
     def _set_option(self, name: str, value: Any) -> None:
         self._put(f"setoption name {name} value {value}")
+        self._parameters.update({name: value})
         self._is_ready()
 
     def _is_ready(self) -> None:
@@ -187,6 +193,7 @@ def set_skill_level(self, skill_level: int = 20) -> None:
             None
         """
         self._set_option("UCI_LimitStrength", "false")
+        self._parameters.update({"UCI_LimitStrength": "false"})
         self._set_option("Skill Level", skill_level)
         self._parameters.update({"Skill Level": skill_level})
 
@@ -200,6 +207,7 @@ def set_elo_rating(self, elo_rating: int = 1350) -> None:
             None
         """
         self._set_option("UCI_LimitStrength", "true")
+        self._parameters.update({"UCI_LimitStrength": "true"})
         self._set_option("UCI_Elo", elo_rating)
         self._parameters.update({"UCI_Elo": elo_rating})
 
@@ -380,6 +388,59 @@ def get_top_moves(self, num_top_moves: int = 5) -> List[dict]:
             self._parameters.update({"MultiPV": old_MultiPV_value})
         return top_moves
 
+    def benchmark(self, **kwargs: Any) -> str:
+        """Benchmark will run the bench command with kwargs as options or with the Defaults provided.
+        It is an Additional custom non-UCI command, mainly for debugging.
+        Do not use this command during a search!
+
+        Kwargs:
+            ttSize: int -> Transposition Table size in MB (max 2048)
+            threads: int -> Number of search threads that should be used (max 512)
+            limit: int -> Limit value of limitType spent for each position (max 10000)
+            fenFile: str -> Path to a FEN format file containing positions to bench (path/to/file.fen)
+            limitType: str -> Type of the limit used with limit value (depth, perft, nodes, movetime)
+            evalType: str -> Evaluation type used (mixed, classical, NNUE)
+        """
+        defaults: Dict[str, Any] = {
+            "ttSize": {"option": range(1, 2049), "default": 16},
+            "threads": {"option": range(1, 513), "default": 1},
+            "limit": {"option": range(1, 10001), "default": 13},
+            "fenFile": {"default": "default"},
+            "limitType": {
+                "option": ["depth", "perft", "nodes", "movetime"],
+                "default": "depth",
+            },
+            "evalType": {"option": ["mixed", "classical", "NNUE"], "default": "mixed"},
+        }
+        options: str = ""
+
+        for key in defaults:
+            try:
+                # Handle case for path to a FEN format file provided
+                if key == "fenFile":
+                    if kwargs[key].endswith(".fen") and path.isfile(kwargs["fenFile"]):
+                        options += str(kwargs[key]) + " "
+                        continue
+                value = kwargs[key]
+                option = (
+                    value
+                    if value in defaults[key]["option"]
+                    else defaults[key]["default"]
+                )
+                options += str(option) + " "
+            except KeyError:
+                options += str(defaults[key]["default"]) + " "
+
+        self._put(f"bench {options}")
+        last_text: str = ""
+        while True:
+            text = self._read_line()
+            splitted_text = text.split(" ")
+            if splitted_text[0] == "Nodes/second":
+                last_text = text
+                return last_text
+            last_text = text
+
     def set_depth(self, depth_value: int = 2) -> None:
         """Sets current depth of stockfish engine.
 

tests/stockfish/test_models.py

@@ -21,7 +21,7 @@ def test_get_best_move_first_move(self, stockfish):
 
     def test_get_best_move_time_first_move(self, stockfish):
         best_move = stockfish.get_best_move_time(1000)
-        assert best_move in ("e2e3", "e2e4", "g1f3", "b1c3")
+        assert best_move in ("e2e3", "e2e4", "g1f3", "b1c3", "d2d4")
 
     def test_set_position_resets_info(self, stockfish):
         stockfish.set_position(["e2e4", "e7e6"])
@@ -87,8 +87,8 @@ def test_set_fen_position_starts_new_game(self, stockfish):
         stockfish.set_fen_position("3kn3/p5rp/1p3p2/3B4/3P1P2/2P5/1P3K2/8 w - - 0 53")
         assert stockfish.info == ""
 
-    def test_set_fen_position_second_argument(self):
-        stockfish = Stockfish(depth=16)
+    def test_set_fen_position_second_argument(self, stockfish):
+        stockfish.set_depth(16)
         stockfish.set_fen_position(
             "rnbqk2r/pppp1ppp/3bpn2/8/3PP3/2N5/PPP2PPP/R1BQKBNR w KQkq - 0 1", True
         )
@@ -182,6 +182,7 @@ def test_set_elo_rating(self, stockfish):
             "d1e2",
             "g2g3",
             "c2c4",
+            "f1e2",
         )
         assert stockfish.get_parameters()["UCI_Elo"] == 2000
 
@@ -191,11 +192,13 @@ def test_set_elo_rating(self, stockfish):
             "b1c3",
             "d2d3",
             "d2d4",
+            "c2c4",
+            "f1e2",
         )
         assert stockfish.get_parameters()["UCI_Elo"] == 1350
 
-    def test_stockfish_constructor_with_custom_params(self):
-        stockfish = Stockfish(parameters={"Skill Level": 1})
+    def test_stockfish_constructor_with_custom_params(self, stockfish):
+        stockfish.set_skill_level(1)
         assert stockfish.get_parameters() == {
             "Write Debug Log": "false",
             "Contempt": 0,
@@ -296,28 +299,22 @@ def test_set_depth(self, stockfish):
         stockfish.get_best_move()
         assert "depth 12" in stockfish.info
 
-    def test_get_best_move_wrong_position(self):
+    def test_get_best_move_wrong_position(self, stockfish):
         wrong_fen = "3kk3/8/8/8/8/8/8/3KK3 w - - 0 0"
-        s = Stockfish()
-        s.set_fen_position(wrong_fen)
-        assert s.get_best_move() in (
+        stockfish.set_fen_position(wrong_fen)
+        assert stockfish.get_best_move() in (
             "d1e2",
             "d1c1",
         )
 
-    def test_get_parameters(self):
-        s1 = Stockfish()
-        s2 = Stockfish()
-        arg1 = s1.get_parameters()
-        arg2 = s2.get_parameters()
-        assert arg1 == arg2
-        s1.set_skill_level(1)
-        arg1 = s1.get_parameters()
-        arg2 = s2.get_parameters()
-        assert arg1 != arg2
-
-    def test_get_top_moves(self):
-        stockfish = Stockfish(depth=15, parameters={"MultiPV": 4})
+    def test_get_parameters(self, stockfish):
+        stockfish._set_option("Minimum Thinking Time", 10)
+        parameters = stockfish.get_parameters()
+        assert parameters["Minimum Thinking Time"] == 10
+
+    def test_get_top_moves(self, stockfish):
+        stockfish.set_depth(15)
+        stockfish._set_option("MultiPV", 4)
         stockfish.set_fen_position("1rQ1r1k1/5ppp/8/8/1R6/8/2r2PPP/4R1K1 w - - 0 1")
         assert stockfish.get_top_moves(2) == [
             {"Move": "e1e8", "Centipawn": None, "Mate": 1},
@@ -329,14 +326,14 @@ def test_get_top_moves(self):
             {"Move": "g1h1", "Centipawn": None, "Mate": -1},
         ]
 
-    def test_get_top_moves_mate(self):
-        stockfish = Stockfish(depth=10, parameters={"MultiPV": 3})
+    def test_get_top_moves_mate(self, stockfish):
+        stockfish.set_depth(10)
+        stockfish._set_option("MultiPV", 3)
         stockfish.set_fen_position("8/8/8/8/8/6k1/8/3r2K1 w - - 0 1")
         assert stockfish.get_top_moves() == []
         assert stockfish.get_parameters()["MultiPV"] == 3
 
-    def test_get_top_moves_raising_error(self):
-        stockfish = Stockfish()
+    def test_get_top_moves_raising_error(self, stockfish):
         stockfish.set_fen_position(
             "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"
         )
@@ -374,7 +371,7 @@ def test_make_moves_from_current_position(self, stockfish):
             == "r1b1kb1r/ppp1n1pp/2p5/4Pp2/8/2N2N1P/PPP2PP1/R1BR2K1 w - f6 0 9"
         )
 
-    def test_make_moves_transposition_table_speed(self):
+    def test_make_moves_transposition_table_speed(self, stockfish):
         """
         make_moves_from_current_position won't send the "ucinewgame" token to Stockfish, since it
         will reach a new position similar to the current one. Meanwhile, set_fen_position will send this
@@ -386,7 +383,7 @@ def test_make_moves_transposition_table_speed(self):
         evaluating a consecutive set of positions when the make_moves_from_current_position function is used.
         """
 
-        stockfish = Stockfish(depth=16)
+        stockfish.set_depth(16)
         positions_considered = []
         stockfish.set_fen_position(
             "rnbqkbnr/ppp1pppp/8/3p4/2PP4/8/PP2PPPP/RNBQKBNR b KQkq - 0 2"
@@ -408,3 +405,28 @@ def test_make_moves_transposition_table_speed(self):
             total_time_calculating_second += default_timer() - start
 
         assert total_time_calculating_first < total_time_calculating_second
+
+    def test_benchmark_result_with_defaults(self, stockfish):
+        defaults = stockfish.benchmark()
+        assert defaults.split(" ")[0] == "Nodes/second"
+
+    def test_benchmark_result_with_valid_options(self, stockfish):
+        valid_options = stockfish.benchmark(
+            ttSize=64,
+            threads=2,
+            limit=1000,
+            limitType="movetime",
+            evalType="classical",
+        )
+        assert valid_options.split(" ")[0] == "Nodes/second"
+
+    def test_benchmark_result_with_invalid_options(self, stockfish):
+        invalid_options = stockfish.benchmark(
+            ttSize=2049,
+            threads=0,
+            limit=0,
+            fenFile="./fakefile.fen",
+            limitType="fghthtr",
+            evalType="",
+        )
+        assert invalid_options.split(" ")[0] == "Nodes/second"