Counting

evalica.counting(xs, ys, winners, index=None, weights=None, win_weight=1.0, tie_weight=0.5, solver='pyo3')

Count individual elements.

Parameters:

Name	Type	Description	Default
xs	Collection[T]	The left-hand side elements.	required
ys	Collection[T]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	dict[T, int] | None	The index.	None
weights	Collection[float] | None	The example weights.	None
win_weight	float	The win weight.	1.0
tie_weight	float	The tie weight.	0.5
solver	Literal['naive', 'pyo3']	The solver.	'pyo3'

Returns:

Type	Description
CountingResult[T]	The counting result.

Source code in evalica/__init__.py

def counting(
        xs: Collection[T],
        ys: Collection[T],
        winners: Collection[Winner],
        index: dict[T, int] | None = None,
        weights: Collection[float] | None = None,
        win_weight: float = 1.,
        tie_weight: float = .5,
        solver: Literal["naive", "pyo3"] = "pyo3",
) -> CountingResult[T]:
    """
    Count individual elements.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        weights: The example weights.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.

    Returns:
        The counting result.

    """
    assert np.isfinite(win_weight), "win_weight must be finite"
    assert np.isfinite(tie_weight), "tie_weight must be finite"

    xs_indexed, ys_indexed, index = indexing(xs, ys, index)

    assert index is not None, "index is None"

    weights = _wrap_weights(weights, len(xs_indexed))

    if solver == "pyo3":
        scores = counting_pyo3(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
            win_weight=win_weight,
            tie_weight=tie_weight,
        )
    else:
        scores = counting_naive(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
            win_weight=win_weight,
            tie_weight=tie_weight,
        )

    return CountingResult(
        scores=pd.Series(scores, index=index, name=counting.__name__).sort_values(ascending=False, kind="stable"),
        index=index,
        win_weight=win_weight,
        tie_weight=tie_weight,
        solver=solver,
    )

evalica.CountingResult dataclass

Bases: Generic[T]

The counting result.

Attributes:

Name	Type	Description
scores	Series[float]	The element scores.
index	dict[T, int]	The index.
win_weight	float	The win weight.
tie_weight	float	The tie weight.
solver	str	The solver.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class CountingResult(Generic[T]):
    """
    The counting result.

    Attributes:
        scores: The element scores.
        index: The index.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.

    """

    scores: pd.Series[float]
    index: dict[T, int]
    win_weight: float
    tie_weight: float
    solver: str

evalica.average_win_rate(xs, ys, winners, index=None, weights=None, win_weight=1.0, tie_weight=0.5, solver='pyo3')

Count pairwise win rates between the elements and average per element.

Parameters:

Name	Type	Description	Default
xs	Collection[T]	The left-hand side elements.	required
ys	Collection[T]	The right-hand side elements.	required
winners	Collection[Winner]	The winner elements.	required
index	dict[T, int] | None	The index.	None
weights	Collection[float] | None	The example weights.	None
win_weight	float	The win weight.	1.0
tie_weight	float	The tie weight.	0.5
solver	Literal['naive', 'pyo3']	The solver.	'pyo3'

Returns:

Type	Description
AverageWinRateResult[T]	The average win rate result.

Source code in evalica/__init__.py

def average_win_rate(
        xs: Collection[T],
        ys: Collection[T],
        winners: Collection[Winner],
        index: dict[T, int] | None = None,
        weights: Collection[float] | None = None,
        win_weight: float = 1.,
        tie_weight: float = .5,
        solver: Literal["naive", "pyo3"] = "pyo3",
) -> AverageWinRateResult[T]:
    """
    Count pairwise win rates between the elements and average per element.

    Args:
        xs: The left-hand side elements.
        ys: The right-hand side elements.
        winners: The winner elements.
        index: The index.
        weights: The example weights.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.

    Returns:
        The average win rate result.

    """
    assert np.isfinite(win_weight), "win_weight must be finite"
    assert np.isfinite(tie_weight), "tie_weight must be finite"

    xs_indexed, ys_indexed, index = indexing(xs, ys, index)

    assert index is not None, "index is None"

    weights = _wrap_weights(weights, len(xs_indexed))

    if solver == "pyo3":
        scores = average_win_rate_pyo3(
            xs=xs_indexed,
            ys=ys_indexed,
            winners=winners,
            weights=weights,
            total=len(index),
            win_weight=win_weight,
            tie_weight=tie_weight,
        )

    else:
        _matrices = matrices(
            xs_indexed=xs_indexed,
            ys_indexed=ys_indexed,
            winners=winners,
            index=index,
            weights=weights,
        )

        matrix = _make_matrix(_matrices.win_matrix, _matrices.tie_matrix, win_weight, tie_weight)

        with np.errstate(all="ignore"):
            denominator = np.nan_to_num(matrix + matrix.T)

            matrix /= denominator

        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", "Mean of empty slice")

            scores = np.nan_to_num(np.nanmean(matrix, axis=1), copy=False)

    return AverageWinRateResult(
        scores=pd.Series(scores, index=index, name=average_win_rate.__name__).sort_values(
            ascending=False, kind="stable"),
        index=index,
        win_weight=win_weight,
        tie_weight=tie_weight,
        solver=solver,
    )

evalica.AverageWinRateResult dataclass

Bases: Generic[T]

The average win rate result.

Attributes:

Name	Type	Description
scores	Series[float]	The element scores.
index	dict[T, int]	The index.
win_weight	float	The win weight.
tie_weight	float	The tie weight.
solver	str	The solver.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class AverageWinRateResult(Generic[T]):
    """
    The average win rate result.

    Attributes:
        scores: The element scores.
        index: The index.
        win_weight: The win weight.
        tie_weight: The tie weight.
        solver: The solver.

    """

    scores: pd.Series[float]
    index: dict[T, int]
    win_weight: float
    tie_weight: float
    solver: str