Tutorial¶

In [1]:

import evalica
import pandas as pd
import plotly.express as px
from evalica import Winner, bootstrap, bradley_terry

%config InlineBackend.figure_formats = ['svg']

import evalica import pandas as pd import plotly.express as px from evalica import Winner, bootstrap, bradley_terry %config InlineBackend.figure_formats = ['svg']

In [2]:

evalica.__version__

evalica.__version__

Out[2]:

'0.4.0'

Pairwise Comparisons¶

In [3]:

df_food = pd.read_csv(
    "https://raw.githubusercontent.com/dustalov/evalica/0893fd0f1e8107b2d62fd6c5816b55b417c1a050/food.csv",
    dtype=str,
)

df_food["winner"] = df_food["winner"].map(
    {
        "left": Winner.X,
        "right": Winner.Y,
        "tie": Winner.Draw,
    },
)

df_food.head(5)

df_food = pd.read_csv( "https://raw.githubusercontent.com/dustalov/evalica/0893fd0f1e8107b2d62fd6c5816b55b417c1a050/food.csv", dtype=str, ) df_food["winner"] = df_food["winner"].map( { "left": Winner.X, "right": Winner.Y, "tie": Winner.Draw, }, ) df_food.head(5)

Out[3]:

	left	right	winner
0	Pizza	Sushi	1
1	Burger	Pasta	2
2	Tacos	Pizza	1
3	Sushi	Tacos	2
4	Burger	Pizza	1

In [4]:

df_food["left_id"], df_food["right_id"], index = evalica.indexing(df_food["left"], df_food["right"])

df_food["left_id"], df_food["right_id"], index = evalica.indexing(df_food["left"], df_food["right"])

In [5]:

matrices = evalica.matrices(df_food["left_id"], df_food["right_id"], df_food["winner"], index)

matrices = evalica.matrices(df_food["left_id"], df_food["right_id"], df_food["winner"], index)

In [6]:

pd.DataFrame(matrices.win_matrix, index=index, columns=index)  # win matrix

pd.DataFrame(matrices.win_matrix, index=index, columns=index) # win matrix

Out[6]:

	Pizza	Burger	Tacos	Sushi	Pasta
Pizza	0.0	0.0	0.0	2.0	1.0
Burger	3.0	0.0	1.0	2.0	0.0
Tacos	4.0	1.0	0.0	1.0	4.0
Sushi	0.0	4.0	1.0	0.0	1.0
Pasta	0.0	2.0	1.0	0.0	0.0

In [7]:

pd.DataFrame(matrices.tie_matrix, index=index, columns=index)  # tie matrix

pd.DataFrame(matrices.tie_matrix, index=index, columns=index) # tie matrix

Out[7]:

	Pizza	Burger	Tacos	Sushi	Pasta
Pizza	0.0	0.0	0.0	0.0	1.0
Burger	0.0	0.0	0.0	0.0	0.0
Tacos	0.0	0.0	0.0	0.0	0.0
Sushi	0.0	0.0	0.0	0.0	1.0
Pasta	1.0	0.0	0.0	1.0	0.0

In [8]:

count_result = evalica.counting(df_food["left"], df_food["right"], df_food["winner"])
count_result.scores.to_frame()

count_result = evalica.counting(df_food["left"], df_food["right"], df_food["winner"]) count_result.scores.to_frame()

Out[8]:

	counting
Tacos	10.0
Sushi	6.5
Burger	6.0
Pasta	4.0
Pizza	3.5

In [9]:

avr_result = evalica.average_win_rate(df_food["left"], df_food["right"], df_food["winner"])
avr_result.scores.to_frame()

avr_result = evalica.average_win_rate(df_food["left"], df_food["right"], df_food["winner"]) avr_result.scores.to_frame()

Out[9]:

	average_win_rate
Tacos	0.700000
Sushi	0.479167
Burger	0.458333
Pizza	0.437500
Pasta	0.425000

In [10]:

bt_result = evalica.bradley_terry(df_food["left"], df_food["right"], df_food["winner"])
bt_result.scores.to_frame()

bt_result = evalica.bradley_terry(df_food["left"], df_food["right"], df_food["winner"]) bt_result.scores.to_frame()

Out[10]:

	bradley_terry
Tacos	2.509025
Sushi	1.101156
Burger	0.854906
Pasta	0.740381
Pizza	0.571837

In [11]:

newman_result = evalica.newman(df_food["left"], df_food["right"], df_food["winner"])
newman_result.scores.to_frame()

newman_result = evalica.newman(df_food["left"], df_food["right"], df_food["winner"]) newman_result.scores.to_frame()

Out[11]:

	newman
Tacos	2.665211
Sushi	1.090627
Burger	0.829660
Pasta	0.710154
Pizza	0.536813

In [12]:

eigen_result = evalica.eigen(df_food["left"], df_food["right"], df_food["winner"])
eigen_result.scores.to_frame()

eigen_result = evalica.eigen(df_food["left"], df_food["right"], df_food["winner"]) eigen_result.scores.to_frame()

Out[12]:

	eigen
Burger	0.528534
Pizza	0.505309
Sushi	0.454870
Pasta	0.439948
Tacos	0.254665

In [13]:

elo_result = evalica.elo(df_food["left"], df_food["right"], df_food["winner"])
elo_result.scores.to_frame()

elo_result = evalica.elo(df_food["left"], df_food["right"], df_food["winner"]) elo_result.scores.to_frame()

Out[13]:

	elo
Tacos	1013.358777
Sushi	1002.098059
Burger	998.026093
Pasta	994.191306
Pizza	992.325765

In [14]:

df_bt_pairwise = evalica.pairwise_frame(bt_result.scores)

df_bt_pairwise

df_bt_pairwise = evalica.pairwise_frame(bt_result.scores) df_bt_pairwise

Out[14]:

	Tacos	Sushi	Burger	Pasta	Pizza
Tacos	0.500000	0.694986	0.745861	0.772149	0.814391
Sushi	0.305014	0.500000	0.562945	0.597955	0.658195
Burger	0.254139	0.437055	0.500000	0.535895	0.599201
Pasta	0.227851	0.402045	0.464105	0.500000	0.564221
Pizza	0.185609	0.341805	0.400799	0.435779	0.500000

In [15]:

fig = px.imshow(df_bt_pairwise, color_continuous_scale="RdBu", text_auto=".2f")
fig.update_layout(xaxis_title="Loser", yaxis_title="Winner", xaxis_side="top")
fig.update_traces(hovertemplate="Winner: %{y}<br>Loser: %{x}<br>Fraction of Wins: %{z}")
fig.show()

fig = px.imshow(df_bt_pairwise, color_continuous_scale="RdBu", text_auto=".2f") fig.update_layout(xaxis_title="Loser", yaxis_title="Winner", xaxis_side="top") fig.update_traces(hovertemplate="Winner: %{y}
Loser: %{x}
Fraction of Wins: %{z}") fig.show()

In [16]:

bootstrap_result = bootstrap(
    bradley_terry,
    df_food["left"],
    df_food["right"],
    df_food["winner"],
    n_resamples=10,
    random_state=42,
)

df_melted = bootstrap_result.distribution.melt(var_name="Item", value_name="Score")

df_melted.head(5)

bootstrap_result = bootstrap( bradley_terry, df_food["left"], df_food["right"], df_food["winner"], n_resamples=10, random_state=42, ) df_melted = bootstrap_result.distribution.melt(var_name="Item", value_name="Score") df_melted.head(5)

Out[16]:

	Item	Score
0	Pizza	3.965574
1	Pizza	3.484208
2	Pizza	3.370370
3	Pizza	1.540967
4	Pizza	3.407976

In [17]:

fig = px.box(df_melted, x="Score", y="Item", color="Item", title="Bradley–Terry Bootstrap Scores")
fig.update_traces(hovertemplate="<b>%{y}</b><br>Score: %{x:.3f}<extra></extra>")
fig.show()

fig = px.box(df_melted, x="Score", y="Item", color="Item", title="Bradley–Terry Bootstrap Scores") fig.update_traces(hovertemplate="%{y}
Score: %{x:.3f}") fig.show()

Inter-Rater Reliability¶

In [18]:

df_codings = pd.read_csv(
    "https://raw.githubusercontent.com/dustalov/evalica/d356c3988fdf9c1db249767413a7a8a1f49d64c0/codings.csv",
    header=None,
    dtype=str,
)

df_codings

df_codings = pd.read_csv( "https://raw.githubusercontent.com/dustalov/evalica/d356c3988fdf9c1db249767413a7a8a1f49d64c0/codings.csv", header=None, dtype=str, ) df_codings

Out[18]:

	0	1	2	3	4	5	6	7	8	9	10	11
0	1	2	3	3	2	1	4	1	2	NaN	NaN	NaN
1	1	2	3	3	2	2	4	1	2	5	NaN	3
2	NaN	3	3	3	2	3	4	2	2	5	1	NaN
3	1	2	3	3	2	4	4	1	2	5	1	NaN

In [19]:

distances = ["nominal", "ordinal", "interval", "ratio"]
alpha_values = {dist: evalica.alpha(df_codings, distance=dist).alpha for dist in distances}  # type: ignore[arg-type]

pd.Series(alpha_values, name="alpha").to_frame()

distances = ["nominal", "ordinal", "interval", "ratio"] alpha_values = {dist: evalica.alpha(df_codings, distance=dist).alpha for dist in distances} # type: ignore[arg-type] pd.Series(alpha_values, name="alpha").to_frame()

Out[19]:

	alpha
nominal	0.743421
ordinal	0.815388
interval	0.849107
ratio	0.797403