Skip to content

Evaluation Space

ah2ac2.evaluation.evaluation_space.EvaluationSpace

Manages the interaction with the AH2AC2 evaluation server.

This class allows users to request new evaluation environments, retrieve information about the ongoing evaluation, and manage the lifecycle of these environments.

Attributes:

Name Type Description
submission_key str

The unique key identifying the submission.
evaluation_environment Union[EvaluationEnvironment, None]

The currently active evaluation environment, if any.
Source code in ah2ac2/evaluation/evaluation_space.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
class EvaluationSpace:
    """
    Manages the interaction with the AH2AC2 evaluation server.

    This class allows users to request new evaluation environments,
    retrieve information about the ongoing evaluation, and manage
    the lifecycle of these environments.

    Attributes:
        submission_key: The unique key identifying the submission.
        evaluation_environment: The currently active evaluation environment, if any.
    """
    __DEFAULT_BASE_URL = "prod-proxy.ah2ac2.com"
    __BASE_URL_ENV_VAR = "AH2AC2_EVALUATION_BASE_URL"

    def __init__(self, submission_key: str) -> None:
        """
        Initializes the EvaluationSpace with a submission key.

        Args:
            submission_key: The unique key for the submission.
        """
        self.submission_key: str = submission_key
        self.evaluation_environment: Union[EvaluationEnvironment, None] = None

        self.__base_url: str = os.getenv(self.__BASE_URL_ENV_VAR, self.__DEFAULT_BASE_URL)

    @property
    def info(self) -> EvaluationInfo:
        """
        Retrieves information about the overall evaluation process.

        This includes the status of all environments associated with the submission key.

        Returns:
            An `EvaluationInfo` object containing details about the evaluation.
        """
        url = f"https://{self.__base_url}/evaluation-info/{self.submission_key}"
        return EvaluationInfo(**requests.get(url).json())

    def next_environment(self) -> EvaluationEnvironment:
        """
        Requests the next official evaluation environment from the server.

        Raises:
            Exception: If a previous environment is still active and not done.

        Returns:
            An `EvaluationEnvironment` instance for the next environment.
        """
        if self.evaluation_environment is not None and not self.evaluation_environment.done:
            raise Exception("Previous environment is still active.")

        url = f"{self.__base_url}/play-next?submission_key={self.submission_key}"
        self.evaluation_environment: EvaluationEnvironment = EvaluationEnvironment(url)
        return self.evaluation_environment

    def new_test_environment(self, num_players: int, candidate_position: list[int]) -> EvaluationEnvironment:
        """
        Requests a new test environment with a random agent.

        This is useful for testing and debugging the agent\'s interaction
        with the environment.

        Args:
            num_players: The number of players in the test environment.
            candidate_position: A list of positions (0-indexed) that the candidate agent will control.

        Returns:
            An `EvaluationEnvironment` instance for the test environment.
        """
        url = f"{self.__base_url}/play-random-agent?num_players={num_players}"
        for candidate_pos in candidate_position:
            url += f"&candidate_positions={candidate_pos}"
        url += f"&test_submission_key={self.submission_key}"

        self.evaluation_environment = EvaluationEnvironment(url)
        return self.evaluation_environment

__init__(submission_key: str) -> None

Initializes the EvaluationSpace with a submission key.

Parameters:

Name Type Description Default
submission_key str

The unique key for the submission.

 required
Source code in ah2ac2/evaluation/evaluation_space.py 
74
75
76
77
78
79
80
81
82
83
84
def __init__(self, submission_key: str) -> None:
    """
    Initializes the EvaluationSpace with a submission key.

    Args:
        submission_key: The unique key for the submission.
    """
    self.submission_key: str = submission_key
    self.evaluation_environment: Union[EvaluationEnvironment, None] = None

    self.__base_url: str = os.getenv(self.__BASE_URL_ENV_VAR, self.__DEFAULT_BASE_URL)

info: EvaluationInfo property

Retrieves information about the overall evaluation process.

This includes the status of all environments associated with the submission key.

Returns:

Type Description
EvaluationInfo

An EvaluationInfo object containing details about the evaluation.

next_environment() -> EvaluationEnvironment

Requests the next official evaluation environment from the server.

Raises:

Type Description
Exception

If a previous environment is still active and not done.

Returns:

Type Description
EvaluationEnvironment

An EvaluationEnvironment instance for the next environment.
Source code in ah2ac2/evaluation/evaluation_space.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
def next_environment(self) -> EvaluationEnvironment:
    """
    Requests the next official evaluation environment from the server.

    Raises:
        Exception: If a previous environment is still active and not done.

    Returns:
        An `EvaluationEnvironment` instance for the next environment.
    """
    if self.evaluation_environment is not None and not self.evaluation_environment.done:
        raise Exception("Previous environment is still active.")

    url = f"{self.__base_url}/play-next?submission_key={self.submission_key}"
    self.evaluation_environment: EvaluationEnvironment = EvaluationEnvironment(url)
    return self.evaluation_environment

new_test_environment(num_players: int, candidate_position: list[int]) -> EvaluationEnvironment

Requests a new test environment with a random agent.

This is useful for testing and debugging the agent's interaction with the environment.

Parameters:

Name Type Description Default
num_players int

The number of players in the test environment.

 required
candidate_position list[int]

A list of positions (0-indexed) that the candidate agent will control.

 required

Returns:

Type Description
EvaluationEnvironment

An EvaluationEnvironment instance for the test environment.
Source code in ah2ac2/evaluation/evaluation_space.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def new_test_environment(self, num_players: int, candidate_position: list[int]) -> EvaluationEnvironment:
    """
    Requests a new test environment with a random agent.

    This is useful for testing and debugging the agent\'s interaction
    with the environment.

    Args:
        num_players: The number of players in the test environment.
        candidate_position: A list of positions (0-indexed) that the candidate agent will control.

    Returns:
        An `EvaluationEnvironment` instance for the test environment.
    """
    url = f"{self.__base_url}/play-random-agent?num_players={num_players}"
    for candidate_pos in candidate_position:
        url += f"&candidate_positions={candidate_pos}"
    url += f"&test_submission_key={self.submission_key}"

    self.evaluation_environment = EvaluationEnvironment(url)
    return self.evaluation_environment

ah2ac2.evaluation.evaluation_space.EvaluationInfo

Overall information about the evaluation process for a submission.

Attributes:

Name Type Description
current_env Union[EvaluationEnvironmentInfo, None]

Information about the currently active environment, if any.
all_envs list[EvaluationEnvironmentInfo]

A list containing information for all environments associated with the submission key.
human_ai_eval_done bool | None

A flag indicating whether the human-AI evaluation phase is complete for this submission.
Source code in ah2ac2/evaluation/evaluation_space.py 
45
46
47
48
49
50
51
52
53
54
55
56
class EvaluationInfo(BaseModel):
    """
    Overall information about the evaluation process for a submission.

    Attributes:
        current_env: Information about the currently active environment, if any.
        all_envs: A list containing information for all environments associated with the submission key.
        human_ai_eval_done: A flag indicating whether the human-AI evaluation phase is complete for this submission.
    """
    current_env: Union[EvaluationEnvironmentInfo, None] = None
    all_envs: list[EvaluationEnvironmentInfo] = []
    human_ai_eval_done: bool | None = False

ah2ac2.evaluation.evaluation_space.EvaluationEnvironmentInfo

Information about a single evaluation environment.

Attributes:

Name Type Description
status EnvironmentStatus

The current status of this environment.
score Union[int, None]

The final score achieved in this environment, if DONE; otherwise None.
num_players Union[int, None]

The number of players in this environment.
candidate_controlling list[str]

A list of agent IDs that the candidate (submission) controls in this environment.
Source code in ah2ac2/evaluation/evaluation_space.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class EvaluationEnvironmentInfo(BaseModel):
    """
    Information about a single evaluation environment.

    Attributes:
        status: The current status of this environment.
        score: The final score achieved in this environment, if DONE; otherwise None.
        num_players: The number of players in this environment.
        candidate_controlling: A list of agent IDs that the candidate (submission) controls in this environment.
    """
    status: EnvironmentStatus
    score: Union[int, None]
    num_players: Union[int, None]
    candidate_controlling: list[str]

ah2ac2.evaluation.evaluation_space.EnvironmentStatus

Status of an evaluation environment.

Source code in ah2ac2/evaluation/evaluation_space.py 
22
23
24
25
26
class EnvironmentStatus(str, Enum):
    """Status of an evaluation environment."""
    TODO = "TODO"  #: The environment is scheduled but not yet active.
    ACTIVE = "ACTIVE"  #: The environment is currently active and being played.
    DONE = "DONE"  #: The environment has finished, and results may be available.

TODO = 'TODO' class-attribute instance-attribute

ACTIVE = 'ACTIVE' class-attribute instance-attribute

DONE = 'DONE' class-attribute instance-attribute