diff --git a/.gitignore b/.gitignore index 93e82b4..45a2c88 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ venv/ ENV/ env.bak/ venv.bak/ +*.lock # Spyder project settings .spyderproject diff --git a/adastop/_version.py b/adastop/_version.py deleted file mode 100644 index ae73625..0000000 --- a/adastop/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.3" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..07155b2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "adastop" +version = "0.1.3" +description = "AdaStop implements a *statistical test to adaptively choose the number of runs of stochastic algorithms* necessary to compare these algorithms and be able to rank them with a theoretically controlled family-wise error rate" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "click>=7.0", + "matplotlib>=3.9", + "numpy>=1.23.2", + "pandas>=1.5.0", + "scipy>=1.9.2", + "tabulate>=0.8.9", +] + +[project.scripts] +adastop = "adastop.cli:adastop" + +[build-system] +requires = ["uv_build>=0.13.6,<0.14.0"] +build-backend = "uv_build" + +[dependency-groups] +dev = [ + "pytest>6.2.5", +] diff --git a/adastop/__init__.py b/src/adastop/__init__.py similarity index 100% rename from adastop/__init__.py rename to src/adastop/__init__.py diff --git a/src/adastop/_version.py b/src/adastop/_version.py new file mode 100644 index 0000000..2e36870 --- /dev/null +++ b/src/adastop/_version.py @@ -0,0 +1,3 @@ +from importlib.metadata import version + +__version__ = version("adastop") \ No newline at end of file diff --git a/adastop/cli.py b/src/adastop/cli.py similarity index 97% rename from adastop/cli.py rename to src/adastop/cli.py index 5d9ff13..cb49489 100644 --- a/adastop/cli.py +++ b/src/adastop/cli.py @@ -81,7 +81,7 @@ def compare(ctx, input_file, n_groups, size_group, n_permutations, alpha, seed, data[agent]=comparator.eval_values[agent] comparator.partial_compare(data, False) - if not("continue" in list(comparator.decisions.values())): + if "continue" not in list(comparator.decisions.values()): click.echo('') click.echo("Test is finished, decisions are") click.echo(comparator.get_results().to_markdown(index=False)) @@ -131,7 +131,7 @@ def plot(ctx, folder, target_file,width, height ): if os.path.isfile(path_lf): with open(path_lf, 'rb') as fp: comparator = pickle.load(fp) - assert not("continue" in list(comparator.decisions.values())), "Testing process not finished yet, cannot plot yet." + assert "continue" not in list(comparator.decisions.values()), "Testing process not finished yet, cannot plot yet." else: raise ValueError('Comparator save file not found.') diff --git a/adastop/compare_agents.py b/src/adastop/compare_agents.py similarity index 96% rename from adastop/compare_agents.py rename to src/adastop/compare_agents.py index 0e3978e..b67df07 100644 --- a/adastop/compare_agents.py +++ b/src/adastop/compare_agents.py @@ -1,9 +1,6 @@ import logging import numpy as np from copy import copy -import matplotlib.pyplot as plt -from matplotlib.colors import LinearSegmentedColormap -import matplotlib.patches as mpatches import math import pandas as pd import itertools @@ -134,7 +131,7 @@ def compute_mean_diffs(self, k, Z): else: # Eliminate for conditional mean_diffs = {str(comp): [] for comp in self.mean_diffs} - to_remove = [] + for i in range(len(self.mean_diffs[str(comparisons[0])])): zval = [] for comp in self.mean_diffs: @@ -254,7 +251,6 @@ def partial_compare(self, eval_values, verbose=True): print("Step {}".format(k)) current_decisions = np.array(["continue"] * len(self.id_tracked)) - current_sign = np.zeros(len(current_decisions)) for j in range(len(current_decisions)): current_comparisons = self.current_comparisons[ @@ -293,7 +289,6 @@ def partial_compare(self, eval_values, verbose=True): Tmax = 0 Tmin = np.inf Tmaxsigned = 0 - Tminsigned = 0 for i, comp in enumerate( self.current_comparisons[current_decisions == "continue"] ): @@ -311,11 +306,6 @@ def partial_compare(self, eval_values, verbose=True): if Ti < Tmin: Tmin = Ti - imin = i - Tminsigned = (k + 1) * ( - np.mean(Z[comp[0]][: ((k + 1) * self.n[comp[0]])]) - - np.mean(Z[comp[1]][: ((k + 1) * self.n[comp[1]])]) - ) if Tmax > bk_sup: id_reject = np.arange(len(current_decisions))[ @@ -348,7 +338,7 @@ def partial_compare(self, eval_values, verbose=True): id_decided = np.array(current_decisions) != "continue" - if not ("continue" in self.decisions.values()): + if "continue" not in self.decisions.values(): self.is_finished = True self.id_tracked = self.id_tracked[~id_decided] diff --git a/adastop/confidence_sequences.py b/src/adastop/confidence_sequences.py similarity index 100% rename from adastop/confidence_sequences.py rename to src/adastop/confidence_sequences.py diff --git a/adastop/plotting.py b/src/adastop/plotting.py similarity index 92% rename from adastop/plotting.py rename to src/adastop/plotting.py index 9e9cd01..1d85e76 100644 --- a/adastop/plotting.py +++ b/src/adastop/plotting.py @@ -58,24 +58,21 @@ def plot_results(comparator, agent_names=None, axes = None): else: (ax1, ax2) = axes - n_iterations = [comparator.n_iters[comparator.agent_names[i]] for i in id_sort] - the_table = ax1.table( - cellText=[n_iterations], rowLabels=["n_iter"], loc="top", cellLoc="center" - ) + # Draw the heatmap with the mask and correct aspect ratio colors = mpl.colormaps["Pastel1"].colors colors = [colors[0], "lightgray", colors[1], "white"] cmap = ListedColormap(colors, name="my_cmap") - im = ax1.imshow(links, cmap=cmap, vmin=-1, vmax=2, aspect='auto') + ax1.imshow(links, cmap=cmap, vmin=-1, vmax=2, aspect='auto') ax1.set_yticks(np.arange(len(id_sort)), labels=np.array(agent_names)[id_sort]) ax1.set_xticks([], labels=[]) # Loop over data dimensions and create text annotations. for i in range(len(annot)): for j in range(len(annot[0])): - text = ax1.text(j, i, annot[i][j], + ax1.text(j, i, annot[i][j], ha="center", va="center", color="k") @@ -149,13 +146,13 @@ def plot_results_sota(comparator, agent_names=None, axes = None): colors = mpl.colormaps["Pastel1"].colors colors = [colors[0], "lightgray", colors[1], "white"] cmap = ListedColormap(colors, name="my_cmap") - im = ax1.imshow([links], cmap=cmap, vmin=-1, vmax=2, aspect='auto') + ax1.imshow([links], cmap=cmap, vmin=-1, vmax=2, aspect='auto') ax1.set_yticks([], labels=[]) ax1.set_xticks([], labels=[]) # Loop over data dimensions and create text annotations. for i in range(len(annot)): - text = ax1.text(i, 0, annot[i], + ax1.text(i, 0, annot[i], ha="center", va="center", color="k") ax1.autoscale(False) diff --git a/tests/test_cli.py b/tests/test_cli.py index 36d6e4e..6a1cd40 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -30,7 +30,7 @@ def test_cli(): assert result.exit_code == 0 result = runner.invoke(adastop, ['status', 'examples']) assert result.exit_code == 0 - assert os.path.exists(test_pdf_path) == True + assert os.path.exists(test_pdf_path) result = runner.invoke(adastop, ['reset', 'examples']) diff --git a/tests/test_compare_agents.py b/tests/test_compare_agents.py index 2a2e51e..027ac3d 100644 --- a/tests/test_compare_agents.py +++ b/tests/test_compare_agents.py @@ -9,7 +9,6 @@ def test_partial_compare(): rng = np.random.RandomState(seed) - idxs = [] comparator = MultipleAgentsComparator(n=3, K=3, B=B, alpha=alpha, seed=42) evals = {"Agent "+str(k): rng.normal(size=3) for k in range(3)} comparator.partial_compare(evals) @@ -38,7 +37,7 @@ def test_type1(K,n): else: evals = {"Agent "+str(k): rng.normal(size=n) for k in range(n_agents)} comparator.partial_compare(evals) - idxs.append(not("equal" in comparator.decisions.values())) + idxs.append("equal" not in comparator.decisions.values()) print(comparator.get_results().to_string(index=False)) assert np.mean(idxs) < 2*alpha + 1/4/(np.sqrt(n_runs)), "type 1 error seems to be too large." @@ -59,7 +58,7 @@ def test_type2(K,n): else: evals = {"Agent "+str(k): rng.normal(size=n)+2*k for k in range(n_agents)} comparator.partial_compare(evals) - idxs.append(not("equal" in comparator.decisions.values())) + idxs.append("equal" not in comparator.decisions.values()) assert np.mean(idxs) > 0.3, "type 2 error seems to be too large." diff --git a/tests/test_plot.py b/tests/test_plot.py index fb6bf23..37866d5 100644 --- a/tests/test_plot.py +++ b/tests/test_plot.py @@ -1,4 +1,3 @@ -import pytest from adastop import MultipleAgentsComparator import numpy as np import matplotlib.pyplot as plt @@ -23,7 +22,6 @@ def test_plot(): comparator.partial_compare(evals) comparator.plot_results() - # plt.savefig('fig.pdf') fig, axes= plt.subplots(1,2) comparator.plot_results(axes=axes) @@ -44,7 +42,7 @@ def test_plot_sota(): evals = {"Agent "+str(k): rng.normal(size=n) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() - # plt.savefig('fig2.pdf') + fig, axes= plt.subplots(1,2) comparator.plot_results_sota(axes=axes) @@ -61,7 +59,7 @@ def test_plot_noteq(): else: evals = {"Agent "+str(k): rng.normal(size=10)+k for k in range(n_agents)} comparator.partial_compare(evals) - # plt.savefig('fig2.pdf') + fig, axes= plt.subplots(1,2) comparator.plot_results(axes=axes) @@ -81,7 +79,7 @@ def test_plot_sota_noteq(): evals = {"Agent "+str(k): rng.normal(size=10) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() - # plt.savefig('fig2.pdf') + fig, axes= plt.subplots(1,2) comparator.plot_results_sota(axes=axes) @@ -100,7 +98,7 @@ def test_plot_noteq2(): else: evals = {"Agent "+str(k): rng.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} comparator.partial_compare(evals) - # plt.savefig('fig2.pdf') + fig, axes= plt.subplots(1,2) comparator.plot_results(axes=axes) @@ -120,7 +118,7 @@ def test_plot_sota_noteq2(): evals = {"Agent "+str(k): rng.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() - # plt.savefig('fig2.pdf') + fig, axes= plt.subplots(1,2) comparator.plot_results_sota(axes=axes)