Alpha

evalica.alpha(data, distance='nominal', solver=SOLVER)

Compute Krippendorff's alpha.

Quote

Krippendorff, K.: Content Analysis: An Introduction to Its Methodology. Sage Publications, Thousand Oaks, CA (2018).

Parameters:

Name	Type	Description	Default
data	DataFrame	Ratings by observer (rows) and unit (columns).	required
distance	DistanceFunc[T_distance_contra] | DistanceName	Distance metric (nominal, ordinal, interval, ratio) or a custom function.	'nominal'
solver	SolverName	The solver to use (naive or pyo3).	SOLVER

Returns:

Type	Description
AlphaResult	The alpha result.

Source code in evalica/__init__.py

def alpha(
    data: pd.DataFrame,
    distance: DistanceFunc[T_distance_contra] | DistanceName = "nominal",
    solver: SolverName = SOLVER,
) -> AlphaResult:
    """
    Compute Krippendorff's alpha.

    Quote:
        Krippendorff, K.: Content Analysis: An Introduction to Its Methodology.
        Sage Publications, Thousand Oaks, CA (2018).

    Args:
        data: Ratings by observer (rows) and unit (columns).
        distance: Distance metric (nominal, ordinal, interval, ratio) or a custom function.
        solver: The solver to use (naive or pyo3).

    Returns:
        The alpha result.

    """
    matrix = _as_unit_matrix(data)
    codes, unique_values = _factorize_matrix(matrix)

    if solver == "pyo3":
        if not PYO3_AVAILABLE:
            raise SolverError(solver)

        numeric_values = np.asarray(unique_values, dtype=np.float64)

        if callable(distance):
            distance_matrix = _custom_distance(distance, unique_values)
            _alpha, observed, expected = _brzo.alpha(codes, numeric_values, distance_matrix)
        else:
            _alpha, observed, expected = _brzo.alpha(codes, numeric_values, distance)

        if expected == 0.0:
            _alpha = 1.0 if observed == 0.0 else 0.0
    else:
        _alpha, observed, expected = _alpha_naive(codes, unique_values, distance)

    return AlphaResult(
        alpha=_alpha,
        observed=observed,
        expected=expected,
        solver=solver,
    )

evalica.AlphaResult dataclass

The result of Krippendorff's alpha.

Attributes:

Name	Type	Description
alpha	float	The alpha value.
observed	float	The observed disagreement.
expected	float	The expected disagreement.
solver	SolverName	The solver used.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class AlphaResult:
    """
    The result of Krippendorff's alpha.

    Attributes:
        alpha: The alpha value.
        observed: The observed disagreement.
        expected: The expected disagreement.
        solver: The solver used.

    """

    alpha: float
    observed: float
    expected: float
    solver: SolverName

evalica.alpha_bootstrap(data, distance='nominal', solver=SOLVER, *, n_resamples=5000, confidence_level=0.95, random_state=None)

Compute confidence intervals for Krippendorff's alpha using KALPHA-style bootstrap.

Quote

Krippendorff, K.: Bootstrapping Distributions for Krippendorff's Alpha. (2006). https://www.asc.upenn.edu/sites/default/files/2021-03/Algorithm%20for%20Bootstrapping%20a%20Distribution%20of%20Alpha.pdf.

Quote

Hayes, A.F.: Statistical Methods and Macros for SPSS, SAS, and R. https://afhayes.com/spss-sas-and-r-macros-and-code.html.

Parameters:

Name	Type	Description	Default
data	DataFrame	Ratings by observer (rows) and unit (columns).	required
distance	DistanceFunc[T_distance_contra] | DistanceName	Distance metric (nominal, ordinal, interval, ratio) or a custom function.	'nominal'
solver	SolverName	The solver to use (naive or pyo3).	SOLVER
n_resamples	int	Number of bootstrap samples.	5000
confidence_level	float	The confidence level.	0.95
random_state	int | None	The random seed (non-negative integer or None).	None

Returns:

Type	Description
AlphaBootstrapResult	The alpha bootstrap result.

Source code in evalica/__init__.py

def alpha_bootstrap(
    data: pd.DataFrame,
    distance: DistanceFunc[T_distance_contra] | DistanceName = "nominal",
    solver: SolverName = SOLVER,
    *,
    n_resamples: int = 5000,
    confidence_level: float = 0.95,
    random_state: int | None = None,
) -> AlphaBootstrapResult:
    """
    Compute confidence intervals for Krippendorff's alpha using KALPHA-style bootstrap.

    Quote:
        Krippendorff, K.: Bootstrapping Distributions for Krippendorff's Alpha. (2006).
        <https://www.asc.upenn.edu/sites/default/files/2021-03/Algorithm%20for%20Bootstrapping%20a%20Distribution%20of%20Alpha.pdf>.

    Quote:
        Hayes, A.F.: Statistical Methods and Macros for SPSS, SAS, and R.
        <https://afhayes.com/spss-sas-and-r-macros-and-code.html>.

    Args:
        data: Ratings by observer (rows) and unit (columns).
        distance: Distance metric (nominal, ordinal, interval, ratio) or a custom function.
        solver: The solver to use (naive or pyo3).
        n_resamples: Number of bootstrap samples.
        confidence_level: The confidence level.
        random_state: The random seed (non-negative integer or None).

    Returns:
        The alpha bootstrap result.

    """
    if n_resamples <= 0:
        msg = "n_resamples must be a positive integer"
        raise ValueError(msg)
    if not 0.0 < confidence_level < 1.0:
        msg = "confidence_level must be in (0, 1)"
        raise ValueError(msg)
    if random_state is not None and random_state < 0:
        msg = "random_state must be a non-negative integer or None"
        raise ValueError(msg)

    random_seed = random_state

    matrix = _as_unit_matrix(data)
    codes, unique_values = _factorize_matrix(matrix)

    if solver == "pyo3":
        if not PYO3_AVAILABLE:
            raise SolverError(solver)

        numeric_values = np.asarray(unique_values, dtype=np.float64)

        if callable(distance):
            distance_matrix = _custom_distance(distance, unique_values)
            _alpha, observed, expected, distribution = _brzo.alpha_bootstrap(
                codes,
                numeric_values,
                distance_matrix,
                n_resamples,
                random_seed,
            )
        else:
            _alpha, observed, expected, distribution = _brzo.alpha_bootstrap(
                codes,
                numeric_values,
                distance,
                n_resamples,
                random_seed,
            )
    else:
        _alpha, observed, expected = _alpha_naive(codes, unique_values, distance)
        distribution = _alpha_bootstrap_naive(
            codes,
            unique_values,
            distance,
            n_resamples,
            random_seed,
        )

    distribution = np.asarray(distribution, dtype=np.float64)
    alpha_tail = (1.0 - confidence_level) / 2.0
    low_quantile, high_quantile = np.quantile(distribution, [alpha_tail, 1.0 - alpha_tail])
    low = float(low_quantile)
    high = float(high_quantile)

    return AlphaBootstrapResult(
        alpha=float(_alpha),
        observed=float(observed),
        expected=float(expected),
        low=low,
        high=high,
        distribution=pd.Series(distribution, name="alpha"),
        n_resamples=len(distribution),
        confidence_level=confidence_level,
        solver=solver,
    )

evalica.AlphaBootstrapResult dataclass

Bases: AlphaResult

The bootstrap result of Krippendorff's alpha.

Attributes:

Name	Type	Description
alpha	float	The alpha value.
observed	float	The observed disagreement.
expected	float	The expected disagreement.
low	float	The lower bound of the confidence interval.
high	float	The upper bound of the confidence interval.
distribution	Series	The bootstrap alpha distribution.
n_resamples	int	The number of bootstrap samples used.
confidence_level	float	The confidence level.
solver	SolverName	The solver used.

Source code in evalica/__init__.py

@dataclass(frozen=True)
class AlphaBootstrapResult(AlphaResult):
    """
    The bootstrap result of Krippendorff's alpha.

    Attributes:
        alpha: The alpha value.
        observed: The observed disagreement.
        expected: The expected disagreement.
        low: The lower bound of the confidence interval.
        high: The upper bound of the confidence interval.
        distribution: The bootstrap alpha distribution.
        n_resamples: The number of bootstrap samples used.
        confidence_level: The confidence level.
        solver: The solver used.

    """

    low: float
    high: float
    distribution: pd.Series
    n_resamples: int
    confidence_level: float

evalica.DistanceName = Literal['interval', 'nominal', 'ordinal', 'ratio'] module-attribute

evalica.DistanceFunc

Bases: Protocol[T_distance_contra]

The distance function protocol.

Source code in evalica/__init__.py

class DistanceFunc(Protocol[T_distance_contra]):
    """The distance function protocol."""

    def __call__(self, left: T_distance_contra, right: T_distance_contra, /) -> float:
        """
        Compute the distance between the values.

        Args:
            left: The left-hand side value.
            right: The right-hand side value.

        Returns:
            The non-negative distance between the values.

        """
        ...

__call__(left, right)

Compute the distance between the values.

Parameters:

Name	Type	Description	Default
left	T_distance_contra	The left-hand side value.	required
right	T_distance_contra	The right-hand side value.	required

Returns:

Type	Description
float	The non-negative distance between the values.

Source code in evalica/__init__.py

def __call__(self, left: T_distance_contra, right: T_distance_contra, /) -> float:
    """
    Compute the distance between the values.

    Args:
        left: The left-hand side value.
        right: The right-hand side value.

    Returns:
        The non-negative distance between the values.

    """
    ...