Bradley-Terry

evalica.bradley_terry(xs, ys, winners, index=None, weights=None, win_weight=1.0, tie_weight=0.5, solver='pyo3', tolerance=1e-06, limit=100)

Compute the Bradley-Terry scores for the given pairwise comparison.

Quote

Bradley, R.A., Terry, M.E.: Rank Analysis of Incomplete Block Designs: I. The Method of Paired Comparisons. Biometrika. 39, 324–345 (1952). https://doi.org/10.2307/2334029.

Quote

Newman, M.E.J.: Efficient Computation of Rankings from Pairwise Comparisons. Journal of Machine Learning Research. 24, 1–25 (2023). https://www.jmlr.org/papers/v24/22-1086.html.

Parameters:

Name	Type	Description	Default
xs	Collection[T]	The left-hand side elements.	required
ys	Collection[T]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	dict[T, int] | None	The index.	None
weights	Collection[float] | None	The example weights.	None
win_weight	float	The win weight.	1.0
tie_weight	float	The tie weight.	0.5
solver	Literal['naive', 'pyo3']	The solver.	'pyo3'
tolerance	float	The convergence tolerance.	1e-06
limit	int	The maximum number of iterations.	100

Returns:

Type	Description
BradleyTerryResult[T]	The Bradley-Terry result.

Source code in evalica/__init__.py

def bradley_terry(
        xs: Collection[T],
        ys: Collection[T],
        winners: Collection[Winner],
        index: dict[T, int] | None = None,
        weights: Collection[float] | None = None,
        win_weight: float = 1.,
        tie_weight: float = .5,
        solver: Literal["naive", "pyo3"] = "pyo3",
        tolerance: float = 1e-6,
        limit: int = 100,
) -> BradleyTerryResult[T]:
    """
    Compute the Bradley-Terry scores for the given pairwise comparison.

    Quote:
        Bradley, R.A., Terry, M.E.: Rank Analysis of Incomplete Block Designs: I.
        The Method of Paired Comparisons. Biometrika. 39, 324–345 (1952).
        <https://doi.org/10.2307/2334029>.

    Quote:
        Newman, M.E.J.: Efficient Computation of Rankings from Pairwise Comparisons.
        Journal of Machine Learning Research. 24, 1&ndash;25 (2023).
        <https://www.jmlr.org/papers/v24/22-1086.html>.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        weights: The example weights.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.
        tolerance: The convergence tolerance.
        limit: The maximum number of iterations.

    Returns:
        The Bradley-Terry result.

    """
    assert np.isfinite(win_weight), "win_weight must be finite"
    assert np.isfinite(tie_weight), "tie_weight must be finite"

    xs_indexed, ys_indexed, index = indexing(xs, ys, index)

    assert index is not None, "index is None"

    weights = _wrap_weights(weights, len(xs_indexed))

    if solver == "pyo3":
        scores, iterations = bradley_terry_pyo3(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
            win_weight=win_weight,
            tie_weight=tie_weight,
            tolerance=tolerance,
            limit=limit,
        )
    else:
        _matrices = matrices(
            xs_indexed=xs_indexed,
            ys_indexed=ys_indexed,
            winners=winners,
            index=index,
            weights=weights,
        )

        matrix = _make_matrix(_matrices.win_matrix, _matrices.tie_matrix, win_weight, tie_weight, tolerance)

        scores, iterations = bradley_terry_naive(
            matrix=matrix,
            tolerance=tolerance,
            limit=limit,
        )

    return BradleyTerryResult(
        scores=pd.Series(scores, index=index, name=bradley_terry.__name__).sort_values(ascending=False, kind="stable"),
        index=index,
        win_weight=win_weight,
        tie_weight=tie_weight,
        solver=solver,
        tolerance=tolerance,
        iterations=iterations,
        limit=limit,
    )

evalica.BradleyTerryResult dataclass

Bases: Generic[T]

The Bradley-Terry result.

Attributes:

Name	Type	Description
scores	Series[float]	The element scores.
index	dict[T, int]	The index.
win_weight	float	The win weight.
tie_weight	float	The tie weight.
solver	str	The solver.
tolerance	float	The convergence tolerance.
iterations	int	The actual number of iterations.
limit	int	The maximum number of iterations.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class BradleyTerryResult(Generic[T]):
    """
    The Bradley-Terry result.

    Attributes:
        scores: The element scores.
        index: The index.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.
        tolerance: The convergence tolerance.
        iterations: The actual number of iterations.
        limit: The maximum number of iterations.

    """

    scores: pd.Series[float]
    index: dict[T, int]
    win_weight: float
    tie_weight: float
    solver: str
    tolerance: float
    iterations: int
    limit: int

evalica.newman(xs, ys, winners, index=None, v_init=0.5, weights=None, win_weight=1.0, tie_weight=1.0, solver='pyo3', tolerance=1e-06, limit=100)

Compute the scores for the given pairwise comparison using the Newman's algorithm.

Quote

Newman, M.E.J.: Efficient Computation of Rankings from Pairwise Comparisons. Journal of Machine Learning Research. 24, 1–25 (2023). https://www.jmlr.org/papers/v24/22-1086.html.

Parameters:

Name	Type	Description	Default
xs	Collection[T]	The left-hand side elements.	required
ys	Collection[T]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	dict[T, int] | None	The index.	None
v_init	float	The initial tie parameter.	0.5
weights	Collection[float] | None	The example weights.	None
win_weight	float	The win weight.	1.0
tie_weight	float	The tie weight.	1.0
solver	Literal['naive', 'pyo3']	The solver.	'pyo3'
tolerance	float	The convergence tolerance.	1e-06
limit	int	The maximum number of iterations.	100

Returns:

Type	Description
NewmanResult[T]	The Newman's result.

Source code in evalica/__init__.py

def newman(
        xs: Collection[T],
        ys: Collection[T],
        winners: Collection[Winner],
        index: dict[T, int] | None = None,
        v_init: float = .5,
        weights: Collection[float] | None = None,
        win_weight: float = 1.,
        tie_weight: float = 1.,
        solver: Literal["naive", "pyo3"] = "pyo3",
        tolerance: float = 1e-6,
        limit: int = 100,
) -> NewmanResult[T]:
    """
    Compute the scores for the given pairwise comparison using the Newman's algorithm.

    Quote:
        Newman, M.E.J.: Efficient Computation of Rankings from Pairwise Comparisons.
        Journal of Machine Learning Research. 24, 1–25 (2023).
        <https://www.jmlr.org/papers/v24/22-1086.html>.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        v_init: The initial tie parameter.
        weights: The example weights.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.
        tolerance: The convergence tolerance.
        limit: The maximum number of iterations.

    Returns:
        The Newman's result.

    """
    assert np.isfinite(win_weight), "win_weight must be finite"
    assert np.isfinite(tie_weight), "tie_weight must be finite"

    xs_indexed, ys_indexed, index = indexing(xs, ys, index)

    assert index is not None, "index is None"

    weights = _wrap_weights(weights, len(xs_indexed))

    if solver == "pyo3":
        scores, v, iterations = newman_pyo3(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
            v_init=v_init,
            win_weight=win_weight,
            tie_weight=tie_weight,
            tolerance=tolerance,
            limit=limit,
        )

    else:
        _matrices = matrices(
            xs_indexed=xs_indexed,
            ys_indexed=ys_indexed,
            winners=winners,
            index=index,
            weights=weights,
        )

        win_matrix = np.nan_to_num(win_weight * np.nan_to_num(_matrices.win_matrix, nan=tolerance), nan=tolerance)
        tie_matrix = np.nan_to_num(tie_weight * np.nan_to_num(_matrices.tie_matrix, nan=tolerance), nan=tolerance)

        scores, v, iterations = newman_naive(
            win_matrix=win_matrix,
            tie_matrix=tie_matrix,
            v=v_init,
            tolerance=tolerance,
            limit=limit,
        )

    return NewmanResult(
        scores=pd.Series(scores, index=index, name=newman.__name__).sort_values(ascending=False, kind="stable"),
        index=index,
        v=v,
        v_init=v_init,
        win_weight=win_weight,
        tie_weight=tie_weight,
        solver=solver,
        tolerance=tolerance,
        iterations=iterations,
        limit=limit,
    )

evalica.NewmanResult dataclass

Bases: Generic[T]

The Newman's algorithm result.

Attributes:

Name	Type	Description
scores	Series[float]	The element scores.
index	dict[T, int]	The index.
v	float	The tie parameter.
v_init	float	The initial tie parameter.
win_weight	float	The win weight.
tie_weight	float	The tie weight.
solver	str	The solver.
tolerance	float	The convergence tolerance.
iterations	int	The actual number of iterations.
limit	int	The maximum number of iterations.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class NewmanResult(Generic[T]):
    """
    The Newman's algorithm result.

    Attributes:
        scores: The element scores.
        index: The index.
        v: The tie parameter.
        v_init: The initial tie parameter.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.
        tolerance: The convergence tolerance.
        iterations: The actual number of iterations.
        limit: The maximum number of iterations.

    """

    scores: pd.Series[float]
    index: dict[T, int]
    v: float
    v_init: float
    win_weight: float
    tie_weight: float
    solver: str
    tolerance: float
    iterations: int
    limit: int