Utilities

evalica.BootstrapResult dataclass

The result of a bootstrap operation.

Attributes:

Name	Type	Description
result	Result	The original point estimates (from the full dataset).
low	Series[float]	Lower bounds of the confidence interval.
high	Series[float]	Upper bounds of the confidence interval.
stderr	Series[float]	Standard errors of the scores.
distribution	DataFrame	The full bootstrap distribution (resamples x elements).
index	Index	The index of elements.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class BootstrapResult:
    """
    The result of a bootstrap operation.

    Attributes:
        result: The original point estimates (from the full dataset).
        low: Lower bounds of the confidence interval.
        high: Upper bounds of the confidence interval.
        stderr: Standard errors of the scores.
        distribution: The full bootstrap distribution (resamples x elements).
        index: The index of elements.

    """

    result: Result
    low: pd.Series[float]
    high: pd.Series[float]
    stderr: pd.Series[float]
    distribution: pd.DataFrame = field(repr=False)
    index: pd.Index

evalica.Winner

Bases: IntEnum

The outcome of the pairwise comparison.

Source code in evalica/__init__.py

class Winner(IntEnum):
    """The outcome of the pairwise comparison."""

    Draw = 0
    """There is a tie."""

    X = 1
    """The first element won."""

    Y = 2
    """The second element won."""

Draw = 0 class-attribute instance-attribute

There is a tie.

X = 1 class-attribute instance-attribute

The first element won.

Y = 2 class-attribute instance-attribute

The second element won.

evalica.WINNERS = list(Winner) module-attribute

Known values of Winner.

evalica.PYO3_AVAILABLE = True module-attribute

The Rust extension is available and can be used for performance-critical operations.

Please set the environment variable EVALICA_NIJE_BRZO to disable it.

evalica.SOLVER = 'pyo3' if PYO3_AVAILABLE else 'naive' module-attribute

The default solver.

evalica.bootstrap(method, xs, ys, winners, weights=None, index=None, win_weight=1.0, tie_weight=0.5, solver=SOLVER, *, n_resamples=1000, confidence_level=0.95, bootstrap_method='BCa', random_state=None, **kwargs)

Compute weighted bootstrap confidence intervals for the given pairwise comparison.

Parameters:

Name	Type	Description	Default
xs	Collection[T_contra]	The left-hand side elements.	required
ys	Collection[T_contra]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
weights	Collection[float] | None	The example weights.	None
method	RankingMethod[T_contra]	The ranking method to use.	required
index	Index | None	The index.	None
win_weight	float	The win weight.	1.0
tie_weight	float	The tie weight.	0.5
solver	SolverName	The solver.	SOLVER
n_resamples	int	The number of resamples.	1000
confidence_level	float	The confidence level.	0.95
bootstrap_method	Literal['percentile', 'basic', 'BCa']	The bootstrap method (percentile, basic, or BCa).	'BCa'
random_state	int | Generator | None	The random state.	None
**kwargs	Any	The additional arguments for the ranking method.	{}

Returns:

Type	Description
BootstrapResult	The bootstrap result.

Source code in evalica/__init__.py

def bootstrap(
    method: RankingMethod[T_contra],
    xs: Collection[T_contra],
    ys: Collection[T_contra],
    winners: Collection[Winner],
    weights: Collection[float] | None = None,
    index: pd.Index | None = None,
    win_weight: float = 1.0,
    tie_weight: float = 0.5,
    solver: SolverName = SOLVER,
    *,
    n_resamples: int = 1000,
    confidence_level: float = 0.95,
    bootstrap_method: Literal["percentile", "basic", "BCa"] = "BCa",
    random_state: int | np.random.Generator | None = None,
    **kwargs: Any,  # noqa: ANN401
) -> BootstrapResult:
    """
    Compute weighted bootstrap confidence intervals for the given pairwise comparison.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        weights: The example weights.
        method: The ranking method to use.
        index: The index.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.
        n_resamples: The number of resamples.
        confidence_level: The confidence level.
        bootstrap_method: The bootstrap method (percentile, basic, or BCa).
        random_state: The random state.
        **kwargs: The additional arguments for the ranking method.

    Returns:
        The bootstrap result.

    """
    _, _, index = indexing(xs, ys, index)

    result = method(
        xs=xs,
        ys=ys,
        winners=winners,
        weights=weights,
        index=index,
        win_weight=win_weight,
        tie_weight=tie_weight,
        solver=solver,
        **kwargs,
    )

    weights_array = np.array(_wrap_weights(weights, len(xs)), dtype=np.float64)

    samples = (np.array(xs, dtype=object), np.array(ys, dtype=object), np.array(winners, dtype=np.uint8), weights_array)

    def statistic(*data: tuple[Any, ...]) -> npt.NDArray[np.float64]:
        """Compute scores for a single bootstrap resample."""
        xs_sample, ys_sample, winners_sample, weights_sample = data

        result_sample = method(
            xs=xs_sample,
            ys=ys_sample,
            winners=list(winners_sample),  # TODO: ensure no copying needed
            index=index,
            weights=weights_sample,
            win_weight=win_weight,
            tie_weight=tie_weight,
            solver=solver,
            **kwargs,
        )

        return cast("npt.NDArray[np.float64]", result_sample.scores.to_numpy(dtype=np.float64))

    bootstrap_result = scipy_bootstrap(
        data=samples,
        statistic=statistic,
        paired=True,
        n_resamples=n_resamples,
        confidence_level=confidence_level,
        method=bootstrap_method,
        random_state=random_state,
        vectorized=False,
    )

    return BootstrapResult(
        result=result,
        low=pd.Series(bootstrap_result.confidence_interval.low, index=index, name="low"),
        high=pd.Series(bootstrap_result.confidence_interval.high, index=index, name="high"),
        stderr=pd.Series(bootstrap_result.standard_error, index=index, name="stderr"),
        distribution=pd.DataFrame(bootstrap_result.bootstrap_distribution.T, columns=index),
        index=index,
    )

evalica.indexing(xs, ys, index=None)

Map the input elements into their numerical representations.

Parameters:

Name	Type	Description	Default
xs	Collection[T_contra]	The left-hand side elements.	required
ys	Collection[T_contra]	The right-hand side elements.	required
index	Index | None	The index; if provided, all elements in xs and ys must be present in it.	None

Returns:

Type	Description
tuple[list[int], list[int], Index]	The tuple containing the numerical representations of the input elements and the corresponding index.

Source code in evalica/__init__.py

def indexing(
    xs: Collection[T_contra],
    ys: Collection[T_contra],
    index: pd.Index | None = None,
) -> tuple[list[int], list[int], pd.Index]:
    """
    Map the input elements into their numerical representations.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        index: The index; if provided, all elements in xs and ys must be present in it.

    Returns:
        The tuple containing the numerical representations of the input elements and the corresponding index.

    """
    if index is None:
        labels = list(dict.fromkeys([*xs, *ys]))
        index = pd.Index(labels)

    xi = index.get_indexer(cast("pd.Index", xs))
    yi = index.get_indexer(cast("pd.Index", ys))

    if (xi < 0).any() or (yi < 0).any():
        msg = "Unknown element in reindexing"
        raise TypeError(msg)

    return xi.tolist(), yi.tolist(), index

evalica.matrices(xs_indexed, ys_indexed, winners, index, weights=None, solver=SOLVER)

Build win and tie matrices from the given elements.

Parameters:

Name	Type	Description	Default
xs_indexed	Collection[int]	The left-hand side elements.	required
ys_indexed	Collection[int]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	Index	The index.	required
weights	Collection[float] | None	The example weights.	None
solver	SolverName	The solver.	SOLVER

Returns:

Type	Description
MatricesResult	The win and tie matrices.

Source code in evalica/__init__.py

def matrices(
    xs_indexed: Collection[int],
    ys_indexed: Collection[int],
    winners: Collection[Winner],
    index: pd.Index,
    weights: Collection[float] | None = None,
    solver: SolverName = SOLVER,
) -> MatricesResult:
    """
    Build win and tie matrices from the given elements.

    Args:
        xs_indexed: The left-hand side elements.
        ys_indexed: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        weights: The example weights.
        solver: The solver.

    Returns:
        The win and tie matrices.

    """
    weights = _wrap_weights(weights, len(xs_indexed))

    if solver == "pyo3":
        if not PYO3_AVAILABLE:
            raise SolverError(solver)

        win_matrix, tie_matrix = _brzo.matrices(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
        )
    else:
        win_matrix, tie_matrix = matrices_naive(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
        )

    return MatricesResult(
        win_matrix=win_matrix,
        tie_matrix=tie_matrix,
        index=index,
    )

evalica.MatricesResult dataclass

The win and tie matrices.

Attributes:

Name	Type	Description
win_matrix	NDArray[float64]	The matrix representing wins between the elements.
tie_matrix	NDArray[float64]	The matrix representing ties between the elements; it is always symmetric.
index	Index	The index.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class MatricesResult:
    """
    The win and tie matrices.

    Attributes:
        win_matrix: The matrix representing wins between the elements.
        tie_matrix: The matrix representing ties between the elements; it is always symmetric.
        index: The index.

    """

    win_matrix: npt.NDArray[np.float64]
    tie_matrix: npt.NDArray[np.float64]
    index: pd.Index

evalica.RankingMethod

Bases: Protocol[T_contra]

The ranking method protocol.

Source code in evalica/__init__.py

@runtime_checkable
class RankingMethod(Protocol[T_contra]):
    """The ranking method protocol."""

    def __call__(
        self,
        xs: Collection[T_contra],
        ys: Collection[T_contra],
        winners: Collection[Winner],
        index: pd.Index | None = None,
        weights: Collection[float] | None = None,
        **kwargs: Any,  # noqa: ANN401
    ) -> Result:
        """
        Compute the scores for the given pairwise comparison.

        Args:
            xs: The left-hand side elements.
            ys: The right-hand side elements.
            winners: The winner elements.
            index: The index.
            weights: The example weights.
            **kwargs: The additional keyword arguments.

        Returns:
            The ranking result.

        """
        ...

__call__(xs, ys, winners, index=None, weights=None, **kwargs)

Compute the scores for the given pairwise comparison.

Parameters:

Name	Type	Description	Default
xs	Collection[T_contra]	The left-hand side elements.	required
ys	Collection[T_contra]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	Index | None	The index.	None
weights	Collection[float] | None	The example weights.	None
**kwargs	Any	The additional keyword arguments.	{}

Returns:

Type	Description
Result	The ranking result.

Source code in evalica/__init__.py

def __call__(
    self,
    xs: Collection[T_contra],
    ys: Collection[T_contra],
    winners: Collection[Winner],
    index: pd.Index | None = None,
    weights: Collection[float] | None = None,
    **kwargs: Any,  # noqa: ANN401
) -> Result:
    """
    Compute the scores for the given pairwise comparison.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        weights: The example weights.
        **kwargs: The additional keyword arguments.

    Returns:
        The ranking result.

    """
    ...

evalica.Result

Bases: Protocol

The result protocol.

Attributes:

Name	Type	Description
scores	Series[float]	The element scores.
index	Index	The index.

Source code in evalica/__init__.py

@runtime_checkable
class Result(Protocol):
    """
    The result protocol.

    Attributes:
        scores: The element scores.
        index: The index.

    """

    scores: pd.Series[float]
    index: pd.Index

evalica.SolverName = Literal['naive', 'pyo3'] module-attribute

evalica.pairwise_frame(scores)

Create a data frame out of the estimated pairwise scores.

Parameters:

Name	Type	Description	Default
scores	Series[float]	The element scores.	required

Returns:

Type	Description
DataFrame	The data frame representing pairwise scores between the elements.

Source code in evalica/__init__.py

def pairwise_frame(scores: pd.Series[float]) -> pd.DataFrame:
    """
    Create a data frame out of the estimated pairwise scores.

    Args:
        scores: The element scores.

    Returns:
        The data frame representing pairwise scores between the elements.

    """
    arr = np.asarray(scores.array, dtype=np.float64)
    return pd.DataFrame(pairwise_scores(arr), index=scores.index, columns=scores.index)

evalica.pairwise_scores(scores, solver=SOLVER)

Estimate the pairwise scores.

Parameters:

Name	Type	Description	Default
scores	NDArray[float64]	The element scores.	required
solver	SolverName	The solver.	SOLVER

Returns:

Type	Description
NDArray[float64]	The matrix representing pairwise scores between the elements.

Source code in evalica/__init__.py

def pairwise_scores(
    scores: npt.NDArray[np.float64],
    solver: SolverName = SOLVER,
) -> npt.NDArray[np.float64]:
    """
    Estimate the pairwise scores.

    Args:
        scores: The element scores.
        solver: The solver.

    Returns:
        The matrix representing pairwise scores between the elements.

    """
    if scores.ndim != 1:
        raise ScoreDimensionError(scores.ndim)

    if solver == "pyo3":
        if not PYO3_AVAILABLE:
            raise SolverError(solver)

        return _brzo.pairwise_scores(scores)

    return pairwise_scores_naive(scores)

evalica.__version__ = '0.4.1' module-attribute

The version of Evalica.