"""mcms-benchs: A toolbox that provides a common framework for benchmarks of multi-component signal processing methods.
Copyright (C) 2024 Juan Manuel Miramont
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import pandas as pd
import seaborn as sns
from mcsm_benchs.Benchmark import Benchmark
import numpy as np
from matplotlib import pyplot as plt, ticker as mticker
import string
import os
import plotly.express as px
import scipy.stats as spst
[docs]
class ResultsInterpreter:
"""This class takes a Benchmark-class object to produce a series of plots and tables
summarizing the obtained results:
Methods
-------
def get_benchmark_as_data_frame(self):
"""
def __init__(self, a_benchmark):
if not isinstance(a_benchmark, Benchmark):
# self.results = a_benchmark.get_results_as_df()
# else:
raise ValueError("Input should be a Benchmark.\n")
self.benchmark = a_benchmark
self.task = a_benchmark.task
self.methods_ids = a_benchmark.methods_ids
self.N = a_benchmark.N
self.repetitions = a_benchmark.repetitions
self.snr_values = a_benchmark.SNRin
self.signal_ids = a_benchmark.signal_ids
self.methods_and_params_dic = a_benchmark.methods_and_params_dic
# Number of elements to display in the plots.
self.ncomparisons = sum(
[len(self.methods_and_params_dic[m]) for m in self.methods_ids]
)
self.path_results = os.path.join("results")
self.path_results_figures = os.path.join("results", "figures")
# --------------------------------------------------------------------------------------
[docs]
def get_benchmark_as_data_frame(self, idx=0):
"""Returns a DataFrame with the raw data produced by the benchmark with the
following format:
-------------------------------------------------------------------------
| Method | Parameter | Signal_id | Repetition | SNRin_1 | ... | SNRin_n |
-------------------------------------------------------------------------
Returns:
DataFrame: Raw data of the comparisons.
"""
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
return df
# --------------------------------------------------------------------------------------
[docs]
def rearrange_data_frame(self, results=None, idx=0):
"""Rearrange DataFrame table for using Seaborn library.
Args:
results (DataFrame, optional): If not None, must receive the DataFrame
produced by a Benchmark-class object using get_results_as_df(). If None,
uses the Benchmark object given to the constructor of the Interpreter.
Defaults to None.
Returns:
DataFrame: Rearranged DataFrame
"""
if results is None:
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
else:
df = results
aux_dic = dict()
new_columns = df.columns.values[0:5].copy()
new_columns[-1] = "QRF"
for i in range(4, df.shape[1]):
idx = [j for j in range(4)] + [i]
df_aux = df.iloc[:, idx]
df_aux.columns = new_columns
aux_dic[df.columns[i]] = df_aux
df3 = pd.concat(aux_dic, axis=0)
df3 = df3.reset_index()
df3 = df3.drop(labels="level_1", axis=1)
df3.columns.values[0] = "SNRin"
return df3
# --------------------------------------------------------------------------------------
def _get_df_means(self, df=None, idx=0):
"""Generates a DataFrame of mean results to .md file.
Returns:
str: String containing the table.
"""
# if filename is None:
# filename = 'results'
if df is None:
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
# Check if matrix values are bools:
aux = df.iloc[:, 4::].to_numpy()
if type(aux[0, 0]) is bool:
df.iloc[:, 4::] = aux.astype(int)
column_names = ["Method + Param"] + [col for col in df.columns.values[4::]]
df_means = list()
for signal_id in self.signal_ids:
methods_names = list()
snr_out_mean = np.zeros((1, len([col for col in df.columns.values[4::]])))
snr_out_std = np.zeros((1, len([col for col in df.columns.values[4::]])))
aux_dic = dict()
df2 = df[df["Signal_id"] == signal_id]
for metodo in self.methods_and_params_dic:
tag = metodo
aux = df2[df2["Method"] == metodo]
if len(self.methods_and_params_dic[metodo]) > 1:
for params in self.methods_and_params_dic[metodo]:
methods_names.append(tag + params)
valores = df2[
(df2["Method"] == metodo) & (df2["Parameter"] == params)
]
# Compute the means
valores_mean = valores.iloc[:, 4::].to_numpy().mean(axis=0)
valores_mean.resize((1, valores_mean.shape[0]))
snr_out_mean = np.concatenate((snr_out_mean, valores_mean))
# Compute the std
# valores_std = valores.iloc[:,4::].to_numpy().std(axis = 0)
# valores_std.resize((1,valores_std.shape[0]))
# snr_out_std = np.concatenate((snr_out_std,valores_std))
else:
methods_names.append(tag)
valores = df2[df2["Method"] == metodo]
# Compute the means
valores_mean = valores.iloc[:, 4::].to_numpy().mean(axis=0)
valores_mean.resize((1, valores_mean.shape[0]))
snr_out_mean = np.concatenate((snr_out_mean, valores_mean))
# Compute the std
# valores_std = valores.iloc[:,4::].to_numpy().std(axis = 0)
# valores_std.resize((1,valores_std.shape[0]))
# snr_out_std = np.concatenate((snr_out_std,valores_std))
snr_out_mean = snr_out_mean[1::]
# snr_out_std = snr_out_std[1::]
aux_dic[column_names[0]] = methods_names
for i in range(1, len(column_names)):
aux_dic[str(column_names[i])] = snr_out_mean[:, i - 1]
df_means.append(pd.DataFrame(aux_dic))
return df_means
# ---------------------------------------------------------------------------------------
def _get_df_std(self, df=None, varfun=None, idx=0):
"""Generates a DataFrame with a variability function from the results.
Returns an string to create a .md file.
Args:
df (_type_, optional): Main DataFrame with the results. Defaults to None.
varfun (str, optional): A variability function. Defaults to 'std', which uses numpy.std().
idx (int, optional): Index of the corresponding performance function. Defaults to 0.
Returns:
Two DataFrames with the lower and upper variability indicator.
"""
# Use the benchmark results if no DataFrame is passed as parameter.
if df is None:
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
# Choose an appropriate variability measure:
if varfun is None:
if self.task == "detection": # Bonferroni corrected Clopper-Pearson CI
varfun = lambda x: _clopper_pearson(
x,
alpha=0.05,
bonferroni=self.ncomparisons,
)
# For denoising or misc, use bootstrap CI.
else:
# varfun = lambda x: (np.std(x),np.std(x))
varfun = lambda x: _ci(
x,
alpha=0.05,
bonferroni=self.ncomparisons,
)
# Check if matrix values are bools:
aux = df.iloc[:, 4::].to_numpy()
if type(aux[0, 0]) is bool:
df.iloc[:, 4::] = aux.astype(int)
column_names = ["Method + Param"] + [col for col in df.columns.values[4::]]
output_string = ""
df_std_upper = list()
df_std_lower = list()
for signal_id in self.signal_ids:
methods_names = list()
snr_out_std = np.zeros(
(1, 2 * len([col for col in df.columns.values[4::]]))
)
aux_dic = dict()
aux_dic_2 = dict()
df2 = df[df["Signal_id"] == signal_id]
for metodo in self.methods_and_params_dic:
tag = metodo
aux = df2[df2["Method"] == metodo]
if len(self.methods_and_params_dic[metodo]) > 1:
for params in self.methods_and_params_dic[metodo]:
methods_names.append(tag + params)
valores = df2[
(df2["Method"] == metodo) & (df2["Parameter"] == params)
]
# Compute the std
valores_std = np.array(
[varfun(col) for col in valores.iloc[:, 4::].to_numpy().T]
)
valores_std.resize((1, valores_std.size))
snr_out_std = np.concatenate((snr_out_std, valores_std))
else:
methods_names.append(tag)
valores = df2[df2["Method"] == metodo]
# Compute the std
# valores_std = valores.iloc[:,4::].to_numpy().std(axis = 0)
valores_std = np.array(
[varfun(col) for col in valores.iloc[:, 4::].to_numpy().T]
)
valores_std.resize((1, valores_std.size))
snr_out_std = np.concatenate((snr_out_std, valores_std))
snr_out_std = snr_out_std[1::]
aux_dic[column_names[0]] = methods_names
for i, j in zip(
range(1, len(column_names)), range(0, snr_out_std.shape[1], 2)
):
aux_dic[str(column_names[i])] = snr_out_std[:, j]
df_std_lower.append(pd.DataFrame(aux_dic))
aux_dic_2[column_names[0]] = methods_names
for i, j in zip(
range(1, len(column_names)), range(1, snr_out_std.shape[1], 2)
):
aux_dic_2[str(column_names[i])] = snr_out_std[:, j]
df_std_upper.append(pd.DataFrame(aux_dic_2))
return df_std_upper, df_std_lower
# --------------------------------------------------------------------------------------
def _get_table_means_and_std(self, link="", pm_name=None, idx=0):
"""Generates a table of mean and std results to .md file.
Highlights the best results.
Returns:
str: String containing the table.
"""
task = self.benchmark.task
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
column_names = ["Method + Param"] + [col for col in df.columns.values[4::]]
output_string = ""
# Get dataframes with means and some variability measure
dfs_means = self._get_df_means()
dfs_std_upper, dfs_std_lower = self._get_df_std()
for dfu, dfl, dfm in zip(dfs_std_upper, dfs_std_lower, dfs_means):
dfu.iloc[:, 1::] = dfu.iloc[:, 1::] + dfm.iloc[:, 1::]
dfl.iloc[:, 1::] = dfm.iloc[:, 1::] - dfl.iloc[:, 1::]
# TODO: Round std to 2 decimal places.
# def round_to_2dec(input):
# if type(input) is float:
# return '{:.2f}'.format(input)
# dfs_std_lower = [df.applymap(round_to_2dec) for df in dfs_std_lower]
# dfs_std_upper = [df.applymap(round_to_2dec) for df in dfs_std_upper]
dfs_std = [
{
col: [[low, upp] for low, upp in zip(dfl[col], dfu[col])]
for col in dfu.columns.values[1::]
}
for dfu, dfl in zip(dfs_std_upper, dfs_std_lower)
]
dfs_std = [pd.DataFrame(df) for df in dfs_std]
for signal_id, df_means, df_std in zip(self.signal_ids, dfs_means, dfs_std):
df_means_aux = df_means.copy()
# Format all floats with two decimals:
nparray_aux = df_means.iloc[:, 1::].to_numpy()
maxinds = np.argmax(nparray_aux, axis=0)
for col, max_ind in enumerate(maxinds):
for i in range(
df_means_aux.shape[0]
): # range(len(df_means.loc[col+1])):
df_means_aux.iloc[i, col + 1] = "{:.2f}".format(
df_means.iloc[i, col + 1]
)
# Also, highlight maxima
df_means_aux.iloc[max_ind, col + 1] = (
"**" + "{:.2f}".format(df_means.iloc[max_ind, col + 1]) + "**"
)
# Change column names to make it more human-readable
df_results = pd.DataFrame()
df_results[column_names[0]] = df_means[column_names[0]]
for col_ind in range(1, len(column_names)):
df_results["SNRin=" + str(column_names[col_ind]) + "dB (average)"] = (
df_means_aux[str(column_names[col_ind])]
)
# df_results['SNRin='+str(column_names[col_ind])+'dB (CI)'] = df_std[str(column_names[col_ind])]
# Make sure to have two decimals precision in each std column.
df_results["SNRin=" + str(column_names[col_ind]) + "dB (CI)"] = [
["{:.2f}".format(a) for a in row]
for row in df_std[str(column_names[col_ind])]
]
# df_results['SNRin='+str(column_names[col_ind])+'dB (CI)'] = df_std[str(column_names[col_ind])]
# Make sure having two decimals precision in each std column
if pm_name is not None:
df_results.rename(columns={"QRF": pm_name})
# Table header with links
aux_string = (
"### Signal: "
+ signal_id
+ " [[View Plot]]("
+ link
+ "/plot_"
+ signal_id
+ ".html) "
+ " [[Get .csv]]("
+ link
+ "/results_"
+ signal_id
+ ".csv"
+ ")"
+ "\n"
+ df_results.to_markdown(floatfmt=".2f")
+ "\n"
)
output_string += aux_string
return output_string
# --------------------------------------------------------------------------------------
def _get_report_preamble(self, link=None):
"""Creates the preamble of the .md file with a table summarizing the benchmark results.
Returns:
str: String with the table header.
"""
if link is None:
link_text = ""
else:
link_text = " [[Results .csv]](" + link + "/results.csv" + ") " + "\n"
lines = [
"# Benchmark Report" + link_text + "\n",
"## Configuration" + "\n",
# 'Parallelize' + str(self.benchmark.parallel_flag) + '\n',
"Length of signals: " + str(self.N) + "\n",
"Repetitions: " + str(self.repetitions) + "\n",
"SNRin values: ",
]
lines = lines + [str(val) + ", " for val in self.snr_values] + ["\n"]
lines = (
lines
+ ["### Methods \n"]
+ ["* " + methid + " \n" for methid in self.methods_ids]
)
lines = (
lines
+ ["### Signals \n"]
+ ["* " + signid + " \n" for signid in self.signal_ids]
)
# lines = lines + ['## Figures:\n  \n']
lines = lines + ["## Mean results tables: \n"]
# TODO Check this part for different perf. functions
if self.task == "denoising":
lines = lines + [
"The results shown here are the average and 95\% CI of \
the performance metric with Bonferroni correction. \
Best performances are **bolded**. \n"
]
if self.task == "detection":
lines = lines + [
"The results shown here are the average and 95\% Clopper-Pearson CI of \
the estimated detection power with Bonferroni correction. \
Best performances are **bolded**. \n"
]
return lines
# --------------------------------------------------------------------------------------
[docs]
def save_report(
self, filename=None, path="results", bars=False, link="", pm_name=None
):
"""This function generates a report of the results given in the Benchmark-class
object. The report is saved in a Markdown syntax to be viewed as a .md file,
while a .csv file is generated with the results.
Args:
filename (str, optional): Path for saving the report. Defaults to None.
"""
if filename is None:
filename = "results_" + self.task + ".md"
# Check path availability, otherwise create folder.
if not os.path.exists(path):
os.makedirs(path)
output_path = os.path.join(path, filename)
# Generate table header:
lines = self._get_report_preamble(link=link)
with open(output_path, "w") as f:
f.write("\n".join(lines))
# f.writelines(lines)
# Append table under header
table_string = self._get_table_means_and_std(link=link, pm_name=pm_name)
with open(output_path, "a") as f:
f.write(table_string)
return True
# -------------------------------------------------------------------------------------
[docs]
def get_snr_plot(
self,
df,
x=None,
y=None,
hue=None,
axis=None,
ylabel=None,
plot_type="linear",
**kwargs
):
"""Generates a Quality Reconstruction Factor (QRF) vs. SNRin plot. The QRF is
computed as:
QRF = 20 log ( norm(x) / norm(x-x_r)) [dB]
where x is the noiseless signal and x_r is de denoised estimation of x.
Args:
df (DataFrame): DataFrame with the results of the simulation.
x (str, optional): Column name to use as the horizontal axis.
Defaults to None.
y (str, optional): Column name to use as the vertical axis.
Defaults to None.
hue (str, optional): Column name with the methods' name. Defaults to None.
axis (matplotlib.Axes, optional): The axis object where the plot will be
generated. Defaults to None.
"""
markers = ["o", "d", "s", "*"]
aux = np.unique(df[hue].to_numpy())
# print(aux)
# fig, axis2 = plt.subplots(1,1)
plots = [
(method_name, markers[np.mod(i, 4)]) for i, method_name in enumerate(aux)
]
u_offset = np.linspace(-2, 2, len(plots))
u_offset = np.zeros_like(u_offset)
for offs_idx, plots_info in enumerate(plots):
method_name, marker = plots_info
df_aux = df[df[hue] == method_name]
u = np.unique(df_aux[x].to_numpy())
v = np.zeros_like(u, dtype=float)
label = "".join(
[c for c in string.capwords(method_name, sep="_") if c.isupper()]
)
if method_name.find("-") > -1:
label = label + method_name[method_name.find("-") : :]
for uind, j in enumerate(u):
df_aux2 = df_aux[df_aux[x] == j]
no_nans = df_aux2[y].dropna()
if no_nans.shape != (0,):
v[uind] = np.nanmean(no_nans.to_numpy())
if plot_type == "linear":
axis.plot(
u + u_offset[offs_idx],
v,
"-" + marker,
# ms = 5,
# linewidth = 1.0,
label=label,
**kwargs
)
if plot_type == "semilogy":
axis.semilogy(
u + u_offset[offs_idx],
v + 1e-10,
"-" + marker,
label=label,
**kwargs
)
axis.yaxis.set_minor_locator(
mticker.LogLocator(numticks=999, subs="auto")
)
# axis.plot([np.min(u), np.max(u)],[np.min(u), np.max(u)],'r',
# linestyle = (0, (5, 10)),
# linewidth = 0.75)
axis.set_xticks(u)
axis.set_yticks(v)
# axis.set_xlabel(x + ' (dB)')
# axis.set_ylabel(y + ' (dB)')
axis.set_xlabel("SNRin (dB)")
if ylabel is None:
if self.benchmark.task == "denoising":
axis.set_ylabel("QRF (dB)")
if self.benchmark.task == "detection":
axis.set_ylabel("Detection Power")
else:
axis.set_ylabel(ylabel)
return True
# --------------------------------------------------------------------------------------
[docs]
def get_snr_plot_bars(
self,
df,
x=None,
y=None,
hue=None,
errbar_fun=("ci", 95),
axis=None,
errbar_params=None,
):
"""Generates a Quality Reconstruction Factor (QRF) vs. SNRin barplot.
The QRF is computed as:
QRF = 20 log ( norm(x) / norm(x-x_r)) [dB]
where x is the noiseless signal and x_r is de denoised estimation of x.
Args:
df (DataFrame): DataFrame with the results of the simulation.
x (str, optional): Column name to use as the horizontal axis.
Defaults to None.
y (str, optional): Column name to use as the vertical axis.
Defaults to None.
hue (str, optional): Column name with the methods' name. Defaults to None.
axis (matplotlib.Axes, optional): The axis object where the plot will be
generated. Defaults to None.
"""
if errbar_params is None:
errbar_params = {
"errwidth": 0.1,
"capsize": 0.02,
}
barfig = sns.barplot(
x="SNRin",
y="QRF", # It's QRF in the DataFrame, but changed later.
hue="Method",
data=df,
dodge=True,
errorbar=errbar_fun,
ax=axis,
**errbar_params
)
axis.set_xlabel("SNRin (dB)")
if self.benchmark.task == "denoising":
axis.set_ylabel("QRF (dB)")
if self.benchmark.task == "detection":
axis.set_ylabel("Detection Power")
return barfig
# --------------------------------------------------------------------------------------
[docs]
def get_summary_plots(
self,
df_rearr=None,
size=(3, 3),
savetofile=True,
path=None,
filter_crit="all",
filter_str=None,
errbar_fun=("ci", 95),
errbar_params=None,
ax=None,
plot_type="bars",
bars=True,
magnitude="absolute",
):
"""Generates individual performance plots for each signal, displaying the
performance measure of all methods for all noise conditions.
Args:
size (tuple, optional): Size (in inches) of the figures. Defaults to (3,3).
savetofile (bool, optional): Whether to save or not the figures.
Defaults to True.
filename (_type_, optional): Path and file name to save the figures. If None
figures are saved in "results/figures" . Defaults to None.
filter_str (_type_, optional): A string, or a list of strings, to select
the methods to plot. If None, plots all the methods. Defaults to None.
filter_crit (str, optional): A criterion to use the strings passed in
filter_str. If 'all', only choose those methods where all the strings
appear in the "Method" column of the resuls DataFrame. If 'any', select the methods for wich any of the strings appear in the mentioned column.
Defaults to 'all'.
plot_type (str, optional): _description_. Defaults to 'lines'.
Returns:
list: A list with matplotlib figures.
"""
# Nsignals = len(self.signal_ids)
if df_rearr is None:
df_rearr = self.rearrange_data_frame()
if magnitude == "difference":
df_rearr["QRF"] = df_rearr["QRF"] - df_rearr["SNRin"]
list_figs = list()
for signal_id in self.signal_ids:
if ax is None:
fig, ax = plt.subplots(1, 1)
else:
fig = plt.gca()
print(signal_id)
# sns.set_theme()
df_aux = df_rearr[df_rearr["Signal_id"] == signal_id]
# If the method has different parameters, add the parameters to the name.
indexes = df_aux["Parameter"] != "{(),{}}"
method_and_params = (
df_aux.loc[indexes, "Method"] + "-" + df_aux.loc[indexes, "Parameter"]
)
df_aux.loc[indexes, "Method"] = method_and_params
# Filter methods using the string given:
if filter_str is not None:
if filter_crit == "all":
a = [np.all([j in i for j in filter_str]) for i in df_aux["Method"]]
if filter_crit == "any":
a = [np.any([j in i for j in filter_str]) for i in df_aux["Method"]]
method_and_params = df_aux.iloc[a, :]
df_aux = method_and_params
if bars == False or plot_type == "lines":
self.get_snr_plot(df_aux, x="SNRin", y="QRF", hue="Method", axis=ax)
else:
self.get_snr_plot_bars(
df_aux,
x="SNRin",
y="QRF",
hue="Method",
errbar_fun=errbar_fun,
errbar_params=errbar_params,
axis=ax,
)
if self.benchmark.task == "denoising" and magnitude == "difference":
ax.set_ylabel("QRF - SNR")
if self.benchmark.task == "detection":
ax.set_ylabel("Detection Power")
ax.set_ylim([0, 1.2])
ax.grid(linewidth=0.1)
ax.set_axisbelow(True)
ax.set_title(signal_id)
ax.legend(loc="upper left", frameon=False, fontsize="small")
fig.set_size_inches(size)
# Save figures to file.
if savetofile:
if path is None:
filename = os.path.join(
"results", self.task, "figures", "plot_" + signal_id + ".pdf"
)
else:
filename = os.path.join(path, "plot_" + signal_id + ".pdf")
fig.savefig(filename, bbox_inches="tight")
list_figs.append(fig)
ax = None # Create new figure in the next interation of the loop.
return list_figs
# --------------------------------------------------------------------------------------
[docs]
def get_summary_plotlys(
self, df=None, bars=True, difference=False, varfun=None, ylabel=None, idx=0
):
"""Generates interactive plots with plotly.
Returns:
list : A list with plotly figures.
"""
if df is None:
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
if difference:
for col in df.columns.values[4::]:
df[col] = df[col] - col
figs = []
# Get dataframes with means and some variability measure (varfun)
dfs_means = self._get_df_means(df=df)
dfs_std_upper, dfs_std_lower = self._get_df_std(df=df, varfun=varfun)
for signal_id, df_means, df_std_upper, df_std_lower in zip(
self.signal_ids, dfs_means, dfs_std_upper, dfs_std_lower
):
df3 = df_means.set_index("Method + Param").stack().reset_index()
df3.rename(columns={"level_1": "SNRin", 0: "QRF"}, inplace=True)
df3_std = df_std_upper.set_index("Method + Param").stack().reset_index()
df3_std.rename(columns={"level_1": "SNRin", 0: "std"}, inplace=True)
df3_std_minus = (
df_std_lower.set_index("Method + Param").stack().reset_index()
)
df3_std_minus.rename(columns={"level_1": "SNRin", 0: "std"}, inplace=True)
df3["std"] = df3_std["std"]
df3["std-minus"] = df3_std_minus["std"]
# print(df3)
if bars:
fig = px.bar(
df3,
x="SNRin",
y="QRF",
color="Method + Param",
# markers=True,
barmode="group",
error_x="SNRin",
error_y="std",
error_y_minus="std-minus",
title=signal_id,
)
else:
fig = px.line(
df3,
x="SNRin",
y="QRF",
color="Method + Param",
markers=True,
error_x="SNRin",
error_y="std",
error_y_minus="std-minus",
title=signal_id,
)
fig.update_layout(xaxis_title="SNRin (dB)")
if ylabel is not None:
fig.update_layout(yaxis_title=ylabel)
figs.append(fig)
return figs
[docs]
def get_csv_files(self, path, idx=0):
"""Generates a table of mean results to .md file.
Saves a .csv file with the results per signal.
Returns:
str: String containing the table.
"""
# TODO Deal with results for different perf. metrics here!
df = self.benchmark.get_results_as_df()
if type(df) == list:
df = df[idx]
# Save a .csv with all the results.
df_aux = self.rearrange_data_frame()
filename = os.path.join(path, "results.csv")
df_aux.to_csv(filename)
# Save one .csv per signal.
for signal_id in self.signal_ids:
# Generate DataFrame with only signal information
df2 = df[df["Signal_id"] == signal_id]
# # Save .csv file for the signal.
if path is None:
path = "results"
# Check path availability, otherwise create folder.
if not os.path.exists(path):
os.makedirs(path)
filename = os.path.join(path, "results_" + signal_id + ".csv")
df2.to_csv(filename)
return True
[docs]
def elapsed_time_summary(self):
"""Get a DataFrame summarizing the elapsed times of the methods.
Returns:
DataFrame: Pandas DataFrame with a summary of the elapsed times.
"""
mydict = self.benchmark.elapsed_time
auxdic = {}
for signal_id in mydict:
for k in mydict[signal_id]:
for k2 in mydict[signal_id][k]:
auxdic[signal_id + "-" + k + "-" + k2] = pd.DataFrame(
mydict[signal_id][k][k2]
)
df3 = pd.concat(auxdic, axis=0)
df3 = df3.reset_index()
df3 = df3.drop(labels="level_1", axis=1)
methid = np.unique(df3["level_0"])
auxdic = {}
for i in methid:
auxdic[i] = (
np.mean(df3[0][df3["level_0"] == i]),
np.std(df3[0][df3["level_0"] == i]),
)
df = pd.DataFrame(auxdic)
df = df.transpose()
df.columns = ("Average time (s)", "Std")
return df
# Use this function only for the CP CI shown in the interactive figures using Plotly:
def _clopper_pearson(x, alpha=0.05, bonferroni=1):
"""
Clopper-Pearson confidence interval for Bernoulli parameter
alpha: confidence level
k: number of successes
n: number of observations
"""
alpha = alpha / bonferroni
n = len(x) # k: number of successes
k = sum(x) # n: number of observations
lb = spst.beta.ppf(alpha / 2, k, n - k + 1)
ub = spst.beta.ppf(1 - alpha / 2, k + 1, n - k)
if np.isnan(ub):
ub = 1.0
return np.mean(x) - lb, ub - np.mean(x)
def _ci(x, alpha=0.05, bonferroni=1):
significance = alpha / bonferroni * 100
lb, ub = sns.utils.ci(sns.algorithms.bootstrap(x), which=100 - significance)
return np.mean(x) - lb, ub - np.mean(x)