init commit avalon

configs/assignments/test_avalon.yaml

@@ -0,0 +1,15 @@
+import: definition.yaml
+
+concurrency:
+  task:
+    avalon-dev-single: 3
+  agent:
+    gpt-3.5-turbo-0613: 3
+
+assignments: # List[Assignment] | Assignment
+  - agent: # "task": List[str] | str ,  "agent": List[str] | str
+      - gpt-3.5-turbo-0613
+    task:
+      - avalon-dev-single
+
+output: "outputs/{TIMESTAMP}"

configs/tasks/avalon.yaml

@@ -0,0 +1,18 @@
+default:
+  module: "src.server.tasks.avalon.AvalonBench"
+  parameters:
+    concurrency: 5
+    num_players: 5
+    discussion: False
+
+avalon-dev-naive:
+  parameters:
+    name: "AvalonBench-dev-naive"
+    data_file: "data/avalon/dev.json"
+    agent_list: ["naive", "naive", "naive", "naive", "naive"]
+
+avalon-dev-single:
+  parameters:
+    name: "AvalonBench-dev-single"
+    data_file: "data/avalon/dev.json"
+    agent_list: ["llm", "naive", "naive", "naive", "naive"]

configs/tasks/task_assembly.yaml

@@ -11,3 +11,4 @@ import:
   - os.yaml
   - ltp.yaml
   - alfworld.yaml
+  - avalon.yaml

data/avalon/dev.json

@@ -0,0 +1,102 @@
+[
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Merlin", "Servant", "Assassin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Assassin", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Assassin", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Servant", "Assassin", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Servant", "Minion", "Assassin", "Merlin", "Servant"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Servant", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Servant", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Servant", "Servant", "Minion", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Merlin", "Servant", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 0,
+        "role_names": ["Assassin", "Minion", "Servant", "Servant", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Merlin", "Servant", "Assassin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Assassin", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Assassin", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Servant", "Assassin", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Servant", "Minion", "Assassin", "Merlin", "Servant"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Merlin", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Servant", "Merlin", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Servant", "Servant", "Minion", "Merlin"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Merlin", "Servant", "Servant", "Minion"]
+    },
+    {
+        "num_players": 5,
+        "quest_leader": 1,
+        "role_names": ["Assassin", "Minion", "Servant", "Servant", "Merlin"]
+    }
+]

src/server/tasks/avalon/README.md

@@ -0,0 +1,64 @@
+# AvalonBench
+
+## Quick Start 
+
+### Start the task server and the assigner
+
+Start the game (3 is the number of workers)
+```bash
+python -m src.start_task -a --start avalon-dev-single 3
+```
+Start the assigner
+```bash
+python -m src.assigner --config ./configs/assignments/test_avalon.yaml
+```
+
+### Customize configurations and data
+
+1. You can modify the file `configs/tasks/avalon.yaml` to configure the agent list. A config file looks like this:
+```yaml
+default:
+  module: "src.server.tasks.avalon.AvalonBench"
+  parameters:
+    num_players: 5
+    discussion: False
+
+avalon-dev-naive:
+  parameters:
+    name: "AvalonBench-dev-naive"
+    data_file: "data/avalon/dev.json"
+    agent_list: ["naive", "naive", "naive", "naive", "naive"]
+
+avalon-dev-single:
+  parameters:
+    name: "AvalonBench-dev-single"
+    data_file: "data/avalon/dev.json"
+    agent_list: ["llm", "naive", "naive", "naive", "naive"]
+```
+where `naive` stands for the naive bots. Agents will play the roles with the same index in the data file (see following).
+```plaintext
+Note: There should only be one "llm" in the `agent_list`
+```
+
+2. You can also add data in `data/avalon/dev.json` (Note: Currently we only support the 5-player game setting, which includes 1 Merlin, 2 Servants, 1 Minion and 1 Assassin). A data item looks like this:
+
+```json
+ {
+     "num_players": 5,
+     "quest_leader": 0,
+     "role_names": ["Assassin", "Servant", "Servant", "Merlin", "Minion"]
+ }
+```
+where `quest_leader` is the id of the initial quest leader in this game. You can change the game setup by altering `quest_leader` with number from 0 to 4, and by permuting `role_names`.
+
+### Naive experiment
+
+You can also start a naive experiment using:
+```bash
+python -m src.start_task -a --start avalon-dev-naive 3
+```
+where all the agents are naive bots. For details of the naive strategies, please refer to the [paper](https://arxiv.org/pdf/2310.05036.pdf).
+
+## Prompts
+
+All the prompts are maintained in `src/server/tasks/avalon/prompt.py`. You can find the respective prompts used in `src/server/tasks/avalon/agents/llm_with_discussion.py` and `src/server/tasks/avalon/wrapper.py`.

src/server/tasks/avalon/__init__.py

@@ -0,0 +1 @@
+from .task import AvalonBench

src/server/tasks/avalon/agents/agent.py

@@ -0,0 +1,82 @@
+from typing import List
+from ..engine import AvalonBasicConfig
+class Agent:
+    r"""The base class for all agents.
+
+    Args:
+        id (int): The Player id of the agent.
+        role (int): The role (id) of the agent.
+        config (AvalonBasicConfig): The config of the agent.
+
+    To implement your own agent, subclass this class and implement the following methods:
+        - :method:`Agent.propose_team`
+        - :method:`Agent.vote_on_team`
+        - :method:`Agent.vote_on_mission`
+    """
+    def __init__(self, id: int, role: int, config: AvalonBasicConfig) -> None:
+        self.id = id
+        self.name = f"Player {id}"
+        self.role = role
+        self.role_name = config.ROLES[role]
+        self.config = config
+
+
+    def propose_team(self, mission_id: int) -> frozenset[int]:
+        r"""Propose a team of given size.
+
+        Args:
+            mission_id (int): The id of the mission. team_size = config.num_players_for_quest[mission_id]
+
+        Returns:
+            frozenset[int]: The list of player ids to be included in the team.
+        """
+        raise NotImplementedError
+
+
+    def vote_on_team(self, mission_id: int, team: frozenset[int]) -> bool:
+        r"""Vote on a given team.
+
+        Args:
+            mission_id (int): The id of the mission. num_fails = self.config.num_fails_for_quest[mission_id]
+            team (frozenset[int]): The list of player ids included in the team.
+
+        Returns:
+            bool: The vote result.
+        """
+        raise NotImplementedError
+
+    def vote_on_mission(self, mission_id: int, quest_team: frozenset[int]) -> bool:
+        r"""Vote on a quest (team).
+
+        Args:
+            mission_id (int): The id of the mission. num_fails = self.config.num_fails_for_quest[mission_id]
+            quest_team (frozenset[int]): The list of player ids included in the quest.
+
+        Returns:
+            bool: The vote result.
+        """
+        raise NotImplementedError
+
+
+    def assassinate(self, num_players: int) -> int:
+        r"""Assassinate a player.
+
+        Args:
+            num_players (int): The number of players in the game.
+
+        Returns:
+            int: The id of the player to assassinate. The id is in the range [0, num_players).
+        """
+        raise NotImplementedError
+
+
+    def get_believed_sides(self, num_players: int) -> List[float]:
+        r"""Get the believed sides of all players.
+
+        Args:
+            num_players (int): The number of players in the game.
+
+        Returns:
+            List[float]: The list of believed sides (probability) of all players.
+        """
+        raise NotImplementedError