scikit-learn · andportnoy · Sep 10, 2022 · Sep 24, 2022 · Sep 25, 2022 · Oct 1, 2022
diff --git a/pca-sparse-debug/scripts/mismatch-csv.sh b/pca-sparse-debug/scripts/mismatch-csv.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+LOGFILE=$1
+paste -d ',' \
+      <(echo 'seed,rtol,solver,layout,k,density'; grep -Po '(?<=test_pca_sparse\[).+?(?=\])' $LOGFILE | sed -e 's/1e-/1e@/g' -e 's/-/,/g' -e 's/@/-/g') \
+      <(echo 'bad,total'; grep -Po '(?<=Mismatched elements: )\d+ / \d+' $LOGFILE | sed -e 's/ //g' -e 's/\//,/g')
diff --git a/pca-sparse-debug/scripts/mismatch-log.sh b/pca-sparse-debug/scripts/mismatch-log.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" OMP_NUM_THREADS=1 pytest --color=no -n "$(nproc --all)" sklearn/decomposition/tests/test_pca.py::test_pca_sparse
diff --git a/pca-sparse-debug/scripts/mismatch-main.sh b/pca-sparse-debug/scripts/mismatch-main.sh
@@ -0,0 +1,19 @@
+set -euxo pipefail
+
+SCRIPTDIR=$(dirname "$0")
+DATADIR=$SCRIPTDIR/../data
+PLOTDIR=$SCRIPTDIR/../plots
+TIMESTAMP=$(date +"%Y%m%d-%H%M%S")
+GITHASH=$(git rev-parse --short HEAD)
+
+BASENAME=pca-sparse-mismatch-$GITHASH-$TIMESTAMP
+LOGFILE=$DATADIR/$BASENAME.log
+CSVFILE=$DATADIR/$BASENAME.csv
+PLOTFILE=$PLOTDIR/$BASENAME.png
+
+mkdir -p $DATADIR
+mkdir -p $PLOTDIR
+
+bash $SCRIPTDIR/mismatch-log.sh > $LOGFILE || true
+bash $SCRIPTDIR/mismatch-csv.sh $LOGFILE > $CSVFILE
+python $SCRIPTDIR/mismatch-plot.py $CSVFILE $PLOTFILE
diff --git a/pca-sparse-debug/scripts/mismatch-plot.py b/pca-sparse-debug/scripts/mismatch-plot.py
@@ -0,0 +1,52 @@
+import sys
+import pandas as pd
+import matplotlib.pyplot as plt
+
+csv = sys.argv[1]
+plot = sys.argv[2]
+
+df = pd.read_csv(csv)
+df = df[df.solver != 'auto']
+
+df['rate'] = df.bad/df.total
+
+def mismatch_by(x):
+    gb = df.groupby(x)
+    return gb['rate'].mean()
+
+fig, axes = plt.subplots(2, 2, figsize=(10, 10), dpi=300)
+
+ax=axes[0][0]
+seed = mismatch_by('seed').hist(ax=ax)
+seed.set_title('mismatch rate by seed (histogram)')
+seed.set_xlabel('mismatch rate')
+seed.set_ylabel('seed count')
+seed.set_ylim(top=100)
+seed.set_xlim(right=1)
+
+ax=axes[0][1]
+solver = mismatch_by('solver').plot.bar(ax=ax)
+solver.set_title('mismatch rate by solver')
+solver.set_xlabel('solver')
+solver.set_ylabel('mismatch rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+ax=axes[1][0]
+density = mismatch_by('density').plot.bar(ax=ax)
+density.set_title('mismatch rate by density')
+density.set_xlabel('density')
+density.set_ylabel('mismatch rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+ax=axes[1][1]
+ncomp = mismatch_by('k').plot.bar(ax=ax)
+ncomp.set_title('mismatch rate by number of components')
+ncomp.set_xlabel('# components')
+ncomp.set_ylabel('mismatch rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+for bp in (solver, density, ncomp):
+    bp.set_xticklabels(bp.get_xticklabels(), rotation=0)
+    bp.set_ylim(top=1)
+fig.tight_layout()
+fig.savefig(plot, facecolor='white', transparent=False)
diff --git a/pca-sparse-debug/scripts/mismatch-tolerance-plot.py b/pca-sparse-debug/scripts/mismatch-tolerance-plot.py
@@ -0,0 +1,40 @@
+import sys
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+csv = sys.argv[1]
+plot = sys.argv[2]
+
+df = pd.read_csv(csv)
+df = df[df.solver != 'auto']
+
+df['rate'] = df.bad/df.total
+
+def mismatch_by(x):
+    gb = df.groupby(x)
+    return gb['rate'].mean()
+
+gb = mismatch_by(['solver', 'rtol'])
+gb = gb.reindex(pd.MultiIndex.from_product(gb.index.levels)).fillna(0)
+
+def formatter(x):
+    if x == 1:
+        return '1e-00'
+    else:
+        return format(x, ".0e")
+gb.index = gb.index.set_levels(map(formatter, gb.index.levels[1]), level=1)
+solvers = gb.index.levels[0]
+fig, axes = plt.subplots(1, len(solvers), figsize=(24, 4), dpi=300)
+fig.suptitle('Elementwise mismatch rate by solver and relative tolerance', fontsize=20)
+for solver, ax in zip(solvers, axes.flat):
+    bar = gb[solver].plot.bar(ax=ax)
+    bar.set_title(solver, pad=15, fontsize=20)
+    bar.set_xlabel('relative tolerance', fontsize=14)
+    bar.set_ylabel('mismatch rate', fontsize=14)
+    bar.set_ylim(bottom=0, top=1)
+    bar.tick_params(axis='both', which='major', labelsize=14)
+    ax.bar_label(ax.containers[0], fmt="%.2g")
+
+fig.tight_layout()
+fig.savefig(plot, facecolor='white', transparent=False)
diff --git a/pca-sparse-debug/scripts/passrate-csv.sh b/pca-sparse-debug/scripts/passrate-csv.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+LOGFILE=$1
+echo 'seed,rtol,solver,format,k,density,outcome'
+grep -P 'PASSED|FAILED' $LOGFILE | sed -E -e 's/^.*(FAILED|PASSED).*\[(.*)\]/\2 \1/' -e 's/1e-/1e@/g' -e 's/-/ /g' -e 's/@/-/g' -e 's/ $//' -e 's/ /,/g'
diff --git a/pca-sparse-debug/scripts/passrate-log.sh b/pca-sparse-debug/scripts/passrate-log.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# writes pca sparse pass/fail results to stdout
+
+SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" OMP_NUM_THREADS=1 pytest --color=no -v --tb=no -n "$(nproc --all)" sklearn/decomposition/tests/test_pca.py::test_pca_sparse
diff --git a/pca-sparse-debug/scripts/passrate-main.sh b/pca-sparse-debug/scripts/passrate-main.sh
@@ -0,0 +1,19 @@
+set -euxo pipefail
+
+SCRIPTDIR=$(dirname "$0")
+DATADIR=$SCRIPTDIR/../data
+PLOTDIR=$SCRIPTDIR/../plots
+TIMESTAMP=$(date +"%Y%m%d-%H%M%S")
+GITHASH=$(git rev-parse --short HEAD)
+
+BASENAME=pca-sparse-passrate-$GITHASH-$TIMESTAMP
+LOGFILE=$DATADIR/$BASENAME.log
+CSVFILE=$DATADIR/$BASENAME.csv
+PLOTFILE=$PLOTDIR/$BASENAME.png
+
+mkdir -p $DATADIR
+mkdir -p $PLOTDIR
+
+bash $SCRIPTDIR/passrate-log.sh > $LOGFILE || true
+bash $SCRIPTDIR/passrate-csv.sh $LOGFILE > $CSVFILE
+python $SCRIPTDIR/passrate-plot.py $CSVFILE $PLOTFILE
diff --git a/pca-sparse-debug/scripts/passrate-plot.py b/pca-sparse-debug/scripts/passrate-plot.py
@@ -0,0 +1,55 @@
+import sys
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+csv = sys.argv[1]
+plot = sys.argv[2]
+
+df = pd.read_csv(csv)
+df = df[df.solver != 'auto']
+
+df['pass'] = df.outcome.apply(lambda x: True if x=='PASSED' else False)
+
+def passrate_by(x):
+    passes = df.groupby(x)['pass']
+    counts = passes.count()
+    sums = passes.sum()
+    return sums / counts
+
+fig, axes = plt.subplots(2, 2, figsize=(10, 10), dpi=300)
+
+ax=axes[0][0]
+seed = passrate_by('seed').hist(ax=ax)
+seed.set_title('pass rate by seed (histogram)')
+seed.set_xlabel('pass rate')
+seed.set_ylabel('seed count')
+seed.set_ylim(top=100)
+seed.set_xlim(right=1)
+
+ax=axes[0][1]
+solver = passrate_by('solver').plot.bar(ax=ax)
+solver.set_title('pass rate by solver')
+solver.set_xlabel('solver')
+solver.set_ylabel('pass rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+ax=axes[1][0]
+density = passrate_by('density').plot.bar(ax=ax)
+density.set_title('pass rate by density')
+density.set_xlabel('density')
+density.set_ylabel('pass rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+ax=axes[1][1]
+ncomp = passrate_by('k').plot.bar(ax=ax)
+ncomp.set_title('pass rate by number of components')
+ncomp.set_xlabel('# components')
+ncomp.set_ylabel('pass rate')
+ax.bar_label(ax.containers[0], fmt="%.3f")
+
+for bp in (solver, density, ncomp):
+    bp.set_xticklabels(bp.get_xticklabels(), rotation=0)
+    bp.set_ylim(top=1)
+fig.tight_layout()
+fig.savefig(plot, facecolor='white', transparent=False)
diff --git a/pca-sparse-debug/scripts/passrate-tolerance-plot.py b/pca-sparse-debug/scripts/passrate-tolerance-plot.py
@@ -0,0 +1,38 @@
+import sys
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+csv = sys.argv[1]
+plot = sys.argv[2]
+
+df = pd.read_csv(csv)
+df['pass'] = df.outcome.apply(lambda x: True if x=='PASSED' else False)
+df = df[df.solver != 'auto']
+
+def passrate_by(x):
+    passes = df.groupby(x)['pass']
+    counts = passes.count()
+    sums = passes.sum()
+    return sums / counts
+
+gb = passrate_by(['solver', 'rtol'])
+def formatter(x):
+    if x == 1:
+        return '1e-00'
+    else:
+        return format(x, ".0e")
+gb.index = gb.index.set_levels(map(formatter, gb.index.levels[1]), level=1)
+fig, axes = plt.subplots(1, 4, figsize=(24, 4), dpi=300)
+fig.suptitle('Test pass rate by solver and relative tolerance', fontsize=20)
+for solver, ax in zip(gb.index.levels[0], axes.flat):
+    bar = gb[solver].plot.bar(ax=ax)
+    bar.set_title(solver, pad=15, fontsize=20)
+    bar.set_xlabel('relative tolerance', fontsize=14)
+    bar.set_ylabel('pass rate', fontsize=14)
+    bar.set_ylim(bottom=0, top=1)
+    bar.tick_params(axis='both', which='major', labelsize=14)
+    ax.bar_label(ax.containers[0], fmt="%.2g")
+
+fig.tight_layout()
+fig.savefig(plot, facecolor='white', transparent=False)
diff --git a/pca-sparse-debug/scripts/search-mismatch-stats.sh b/pca-sparse-debug/scripts/search-mismatch-stats.sh
@@ -0,0 +1,4 @@
+LOGFILE=$1
+paste -d ',' \
+      <(echo 'seed,solver,format,k,density'; grep -Po '(?<=_ test_pca_sparse\[).+?(?=\])' $LOGFILE | sed 's/-/,/g') \
+      <(echo 'bad,total'; grep -Po '(?<=Mismatched elements: )\d+ / \d+' $LOGFILE | sed -e 's/ //g' -e 's/\//,/g')
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/bash

		SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" OMP_NUM_THREADS=1 pytest --color=no -n "$(nproc --all)" sklearn/decomposition/tests/test_pca.py::test_pca_sparse