def __init__(self, *args, flip=True, **kwargs):

super().__init__(*args, **kwargs)

self.flip = flip

def __call__(self, x, i=None):

if self.flip:

x = 1 - x

def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):

self.midpoint = midpoint

matplotlib.colors.Normalize.__init__(self, vmin, vmax, clip)

def __call__(self, value, clip=None):

x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]

if len(x) == 0:

return []

maxidx = 0

maxval = x[0]

res = [maxidx]

for i in range(1, len(x)):

if x[i] > maxval:

maxidx = i

maxval = x[i]

res.append(maxidx)

# base_url like 'resnet20/prune_global_49/v5/finetune/finetune_130'

com = 'select * from data where (path, step) in (select path, max(step) from data where path like "%{}/trial_%/iter_0%" group by path)'.format(base_url)

cur = conn.execute(com)

base_map = {

path.split('/')[-3]: acc

for (path, _, step, _, acc) in cur.fetchall()

}

for it, expts in iters.items():

for expt, points in expts.items():

for point, trials in points.items():

for t in trials:

try:

trials[t] -= base_map[t]

except KeyError:

for it, expts in iters.items():

if it == 'iter_0':

continue

for expt, points in expts.items():

for point, trials in points.items():

for t in trials:

# base_url like 'resnet20/prune_global_%/v1'

# allowed_names like ['lottery', 'oneshot_lottery', 'reinit']

allowed_names = ['/{}/'.format(name.strip('/')) for name in allowed_names]

# iter -> expt name -> point -> trial -> delta

iters = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))

com = 'select path, step, test_acc from data where path like "%{}%"'.format(base_url)

cur = conn.execute(com)

for (name, step, acc) in cur.fetchall():

name = '/'.join(name.split('/')[:-1])

if not any(allowed_name in name for allowed_name in allowed_names):

continue

trial = name.split('/')[-2]

if trial not in ('trial_1', 'trial_2', 'trial_3'):

continue

expt_name = name.split('/')[-4]

iter_int = int(name.split('/')[-3].split('_')[-1])

it = 'iter_{}'.format(iter_int)

if it == 'iter_0':

continue

point = round((step - iterations_per_epoch * epochs) / (iterations_per_epoch * iter_int))

try:

prev = iters[it][expt_name][point][trial]

except KeyError:

iters[it][expt_name][point][trial] = acc

res = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))

reinit = 'reinit_long'

for i in iters:

for e in iters[i]:

for p in iters[i][e]:

for t in iters[i][e][p]:

a = iters[i][e][p][t]

d = res[i][reinit][p]

if t in d:

d[t] = max(a, d[t])

else:

d[t] = a

network_long_bases = {

'resnet20': '%resnet20/prune_global_20/v3/lottery/prune_%/trial_%/iter_%',

'resnet50': '%resnet50/prune_global_20/v10/lottery/prune_11259/trial_%/iter_%',

'vgg16': '%vgg_16_nofc/prune_global_20/v6/base/trial_%/iter_%',

}

network_iter_bases = {

'resnet20': [4701, 4701, 4701, 28139, 4701, 12514, 12514, 12514, 12514, 12514],

'resnet50': [11259] * 10,

'vgg16': [0] * 30,

}

com = 'select path,test_acc from data where (path, step) in (select path, max(step) from data where path like "{}" group by path)'.format(network_long_bases[network])

cur = conn.execute(com)

res_dict = {}

for path,test_acc in cur.fetchall():

if 'resnet' in network:

prune = int(path.split('/')[-4].split('_')[-1])

trial = path.split('/')[-3]

it = int(path.split('/')[-2].split('_')[-1])

res_dict[prune,trial,it] = test_acc

else:

prune = 0

trial = path.split('/')[-3]

it = int(path.split('/')[-2].split('_')[-1])

res_dict[prune,trial,it] = test_acc

for it_s in iters:

it = int(it_s.split('_')[-1]) - 1

for expt in iters[it_s]:

if 'best' in expt:

m_it = it

elif 'oneshot' in expt:

m_it = 0

else:

raise ValueError()

for point in iters[it_s][expt]:

for trial in iters[it_s][expt][point]:

# base_url like 'resnet20/prune_global_%/v1'

# allowed_names like ['lottery', 'oneshot_lottery', 'reinit']

allowed_names = ['/{}/'.format(name.strip('/')) for name in allowed_names]

# iter -> expt name -> point -> trial -> delta

iters = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))

com = 'select * from data where (path, step) in (select path, max(step) from data where path like "%{}%" group by path)'.format(base_url)

cur = conn.execute(com)

for (name, _, step, _, acc) in cur.fetchall():

name = '/'.join(name.split('/')[:-1])

if not any(allowed_name in name for allowed_name in allowed_names):

continue

trial = name.split('/')[-2]

if trial not in ('trial_1', 'trial_2', 'trial_3'):

continue

expt_name = name.split('/')[-4]

point = int(name.split('/')[-3].split('_')[-1])

it = name.split('/')[it_idx]

if 'lottery' in expt_name:

point = int(round((epochs * iterations_per_epoch - point) / iterations_per_epoch))

elif 'real_reinit' in expt_name:

point = int(point / iterations_per_epoch)

try:

prev = iters[it][expt_name][point][trial]

except KeyError:

iters[it][expt_name][point][trial] = acc

iters = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))

for i in [i1,i2]:

for a in i:

for b in i[a]:

for c in i[a][b]:

for d in i[a][b][c]:

iters[a][b][c][d] = i[a][b][c][d]

iters = get_iter_dict('resnet20/prune_global_20/v3', ['lottery', 'finetune'], 50000/128, 182, -1)

fix_deltas_iter(iters)

long_it = get_long_iter_dict('%resnet20%v4%', ['reinit_best_long', 'reinit_oneshot_long'], 50000/128, 182)

fix_long_deltas(long_it, 'resnet20')

long_it = merge_long_reinits(long_it)

res = merge(iters, long_it)

remove_other_points(0, 10, 30, 50, 70, 90, 110, 130, 150, 170, 182, 192, 212, 232, 252, 272, 292, 312, 332, 352, 364)(res)

iters = get_iter_dict('resnet50%v10', ['lottery', 'finetune'], 1251, 90, -1)

fix_deltas_iter(iters)

long_it = get_long_iter_dict('%resnet50%v10%', ['reinit_best_long', 'reinit_oneshot_long'], 1251, 90)

fix_long_deltas(long_it, 'resnet50')

long_it = merge_long_reinits(long_it)

res = merge(iters, long_it)

remove_other_points(*range(0, 181, 9))(res)

iters = get_iter_dict('vgg_16_nofc/prune_global_20/v8', ['lottery', 'finetune'], 50000/128, 182, -1)

fix_deltas_iter(iters)

long_it = get_long_iter_dict('%vgg_16_nofc/prune_global_20/v6%', ['reinit_best_long', 'reinit_oneshot_long'], 50000/128, 182)

fix_long_deltas(long_it, 'vgg16')

long_it = merge_long_reinits(long_it)

res = merge(iters, long_it)

remove_other_points(0, 10, 30, 50, 70, 90, 110, 130, 150, 170, 182, 192, 212, 232, 252, 272, 292, 312, 332, 352, 364)(res)

# iters = get_iter_dict('vgg_19_nofc/prune_global_20/v8', ['lottery', 'finetune'], 50000/128, 182, -1)

# fix_deltas_iter(iters)

iters = get_iter_dict('vgg_19_nofc/prune_global_20_fc/v3', ['lottery', 'finetune'], 50000/128, 182, -1)

fix_deltas_iter(iters)

long_it = get_long_iter_dict('%vgg_19_nofc/prune_global_20/v6%', ['reinit_best_long', 'reinit_oneshot_long'], 50000/128, 182)

fix_long_deltas(long_it, 'vgg19')

long_it = merge_long_reinits(long_it)

res = merge(iters, long_it)

remove_other_points(0, 10, 30, 50, 70, 90, 110, 130, 150, 170, 182, 192, 212, 232, 252, 272, 292, 312, 332, 352, 364)(res)

def c(*col):

return (col[0]/255, col[1]/255, col[2]/255, 1)

cmap = {

'lottery': '#1b9e77',

'finetune': '#d95f02',

'reinit': '#7570b3',

}

if 'lottery' in expt:

return 'Rewind'

elif 'finetune' in expt:

return 'Fine-tune'

elif 'reinit' in expt:

return 'Reinitialize'

else:

if 'lottery' in expt:

return 'o--'

elif 'finetune' in expt:

return '^:'

elif 'reinit' in expt:

return 'x-.'

else:

date_suffix = ["th", "st", "nd", "rd"]

if myDate % 10 in [1, 2, 3] and myDate not in [11, 12, 13]:

return date_suffix[myDate % 10]

else:

if 'lottery' in x.lower():

return 0

elif 'finetune' in x.lower():

return 1

elif 'reinit' in x.lower():

return 2

else:

if show:

plt.ion()

else:

plt.ioff()

xs_map = {}

for expt, expts in iters.items():

xs_map[expt] = set()

for points in expts.values():

xs_map[expt] = xs_map[expt].union(set(points.keys()))

all_xs = set.union(*xs_map.values())

for (it, expts) in sorted(iters.items(), key=lambda x: float(x[0].split('_')[-1])):

if it == 'iter_0':

continue

elif it == 'iter_1' and name == 'resnet50' and not is_iterative:

continue

plt.figure()

for (expt, points) in sorted(expts.items(), key=lambda x: legend_sort_key(x[0])):

xs = []

y_means = []

y_errs_lower = []

y_errs_upper = []

for (point, delta_dict) in sorted(points.items()):

deltas = list(delta_dict.values())

xs.append(point)

if RANGE_METHOD == MEDIAN_MAX:

y_means.append(np.median(deltas))

y_errs_lower.append(y_means[-1] - np.min(deltas))

y_errs_upper.append(np.max(deltas) - y_means[-1])

elif RANGE_METHOD == MEAN_STD:

y_means.append(np.mean(deltas))

std = np.std(deltas) if len(deltas) > 1 else 0

if std > 0.03:

std = 0

y_errs_lower.append(std)

y_errs_upper.append(std)

else:

raise ValueError(RANGE_METHOD)

xs = np.array(xs)

y_means = np.array(y_means)

y_errs_lower = np.array(y_errs_lower)

y_errs_upper = np.array(y_errs_upper)

idxs = [i for i in range(len(xs)) if xs[i] in all_xs]

max_idxs = argcummax(y_means[idxs])

plt.errorbar(xs[idxs], y_means[idxs],

[y_errs_lower[idxs], y_errs_upper[idxs]],

fmt=fmtof(expt), label='{} {}'.format(labof(expt), _FLAVOR),

color=get_color(expt),

capsize=5,

)

if name == 'resnet50':

lr_changes = [10, 30, 60]

else:

lr_changes = [46, 91]

# for ch in lr_changes:

# plt.plot([ch, ch], [-0.03, 0.01], '--', color=(0,0,0,0.3))

plt.plot([0, max(all_xs)], [0, 0], '--', color=(0,0,0,0.3))

plt.ylim(-0.03, 0.01)

if is_iterative:

plt.xlabel(r'Re-training Epochs (epochs per iteration $\times$ iterations)')

else:

plt.xlabel('Re-training Epochs')

plt.ylabel('$\Delta$ Accuracy')

plt.legend()

if is_iterative:

it = int(it.split('_')[1])

density = 0.8 ** it

plt.title('{name} $\Delta$ accuracy after {idx}{suffix} pruning iteration ({sparsity:.2%} sparsity)'.format(

name=netname(name),

idx=it,

suffix=suffix_of_number(it),

sparsity=1-density,

).replace('%', r'\%'))

else:

density = float(it.split('_')[1]) / 100

plt.title('{name} $\Delta$ accuracy after one-shot prune to {sparsity:.2%} sparsity'.format(

name=netname(name),

sparsity=1-density,

).replace('%', r'\%'))

plt.tight_layout()

format_axes(plt.gca())

plt.gca().set_xticks([round(i) for i in sorted(all_xs)])

vals = plt.gca().get_xticks()

if is_iterative:

plt.gca().set_xticklabels([r'${} \times {}$'.format(int(x), it) if x > 0 else '0' for x in vals], rotation=30, ha='right')

vals = plt.gca().get_yticks()

plt.gca().set_yticklabels(['{:+,.2%}'.format(x).replace('%', r'\%') for x in vals])

if savename:

os.makedirs(os.path.join(_DIRNAME, 'figures', savename), exist_ok=True)

plt.savefig(os.path.join(_DIRNAME, 'figures', savename, '{:.5}.pdf'.format(1 - density)), dpi=FIG_DPI)

if not show:

plt.close('all')

if show:

from numpy import array

x = array(x)

if any(x < 0) or any(x > 1):

raise ValueError('x must be between 0 and 1 inclusive.')

red = (0.237 - 2.13 * x + 26.92 * x ** 2 - 65.5 * x ** 3 +

63.5 * x ** 4 - 22.36 * x ** 5)

grn = ((0.572 + 1.524 * x - 1.811 * x ** 2) /

(1 - 0.291 * x + 0.1574 * x ** 2)) ** 2

blu = 1. / (1.579 - 4.03 * x + 12.92 * x ** 2 - 31.4 * x ** 3 +

48.6 * x ** 4 - 23.36 * x ** 5)

bars_to_show=['horizontal', 'vertical', 'finetune'],

zones_to_show=['Safe', 'Dominant', 'Fine-tuning Plateau'],

):

real_xs, safes, dominants, ft_deltas = get_zones(iters, name, is_iterative, just_get_data=True)

cmap_lim = 0.005

norm = MidpointNormalize(midpoint=0, vmin=-cmap_lim, vmax=cmap_lim)

cmap = matplotlib.cm.get_cmap('bwr')

for (zone, ztype) in ([

(safes, 'Safe'),

(dominants, 'Dominant'),

(ft_deltas, 'Fine-tuning Plateau'),

]):

if ztype.lower() not in [z.lower() for z in zones_to_show]:

continue

fig, ax = plt.subplots()

plt.title('{} {}, {} pruning'.format(

netname(name),

{

'Safe': 'Rewinding Safety',

'Dominant': 'Rewinding Dominance',

}.get(ztype, ztype),

'Iterative' if is_iterative else 'One-shot',

))

if zone is not ft_deltas:

zone = [z for i, z in enumerate(zone) if 0 <= real_xs[i] <= 1]

C = np.empty((len(zone), len(zone[0][-2])))

delta = real_xs[4] - real_xs[3]

for i, z in enumerate(zone):

for j, diff in enumerate(z[-4]):

C[i, j] = diff

all_xs = np.array(dominants[0][-2])

plt.imshow(C[::-1, :], cmap=cmap, aspect='auto')

ax.invert_yaxis()

format_axes(ax)

class FooSparsityFormatter(ticker.PercentFormatter):

def __init__(self, *args, flip=True, **kwargs):

super().__init__(*args, **kwargs)

self.flip = flip

def __call__(self, x, i=None):

zero = np.log(max(all_xs))/np.log(.8)

x = 0.8 ** (x + zero)

if self.flip:

x = 1 - x

return super().__call__(x, i)

class RetrainFormatter(ticker.PercentFormatter):

def __init__(self, zone, *args, **kwargs):

super().__init__(*args, **kwargs)

self.zone = zone

def __call__(self, x, i=None):

x = np.interp(

len(real_xs) - x - 1,

np.arange(len(real_xs)),

2 - real_xs

)

if self.zone is ft_deltas:

x = 2 - x

else:

x = x

return super().__call__(x, i)

ax.xaxis.set_major_locator(ticker.FixedLocator(np.linspace(0, len(zone[0][-2]), 10)))

ax.xaxis.set_major_formatter(FooSparsityFormatter(1, None if 'resnet' in name else 1, flip=True))

ax.xaxis.set_minor_locator(ticker.NullLocator())

ax.set_xlabel('Sparsity')

if zone is ft_deltas:

ax.set_ylabel('Re-training Time')

else:

ax.set_ylabel('Rewind Point')

ax.yaxis.set_major_locator(ticker.FixedLocator(np.linspace(0, len(zone), 11)))

ax.yaxis.set_major_formatter(RetrainFormatter(zone, 1))

ax.yaxis.set_minor_locator(ticker.NullLocator())

plt.clim(-cmap_lim, cmap_lim)

plt.tight_layout()

def format_coord(x,y):

return "text_string_made_from({:.2f},{:.2f})".format(x,y)

ax.format_coord = format_coord

if savename:

os.makedirs(os.path.join(_DIRNAME, 'figures', savename), exist_ok=True)

plt.savefig(os.path.join(_DIRNAME, 'figures', savename, '{}.pdf'.format(ztype.replace(' ', ''))), dpi=FIG_DPI)

for orientation in bars_to_show:

fig, ax = plt.subplots(figsize=(6, 1.15) if orientation in ('horizontal', 'finetune') else (1.85, 4))

cbar = matplotlib.colorbar.ColorbarBase(ax, cmap=cmap, orientation={'finetune': 'horizontal'}.get(orientation, orientation))

if orientation == 'finetune':

cbar.set_label('$\Delta$ Accuracy between Fine-tuning and Plateau')

else:

cbar.set_label('$\Delta$ Accuracy between Rewinding and Fine-tuning')

cbar.set_ticks(np.linspace(0, 1, 3))

ticks = [t * (cmap_lim * 2) - cmap_lim for t in cbar.get_ticks()]

if orientation == 'finetune':

ticks = [-cmap_lim, -cmap_lim/2, 0]

cbar.set_clim(0, 2)

cbar.set_ticklabels([

r'$\leq {:+.2%}$'.format(ticks[0]).replace(r'%', r'\%')

] + [

r'${:+.2%}$'.format(x).replace(r'%', r'\%')

for x in list(ticks)[1:-1]

] + [

r'$\geq {:+.2%}$'.format(ticks[-1]).replace(r'%', r'\%')

])

fig.tight_layout()

if savename:

os.makedirs(os.path.join(_DIRNAME, 'figures', savename), exist_ok=True)

plt.savefig(os.path.join(_DIRNAME, 'figures', savename, 'bar_{}.pdf'.format(orientation)), dpi=FIG_DPI)

if show:

plt.show()

else:

if show:

plt.ion()

else:

plt.ioff()

lines = collections.defaultdict(list)

all_xs = set()

for (it, expts) in sorted(iters.items(), key=lambda x: float(x[0].split('_')[-1])):

if it == 'iter_0':

continue

if name == 'resnet20' and is_iterative and int(it.split('_')[1]) > 10:

continue

if is_iterative:

density = 0.8 ** int(it.split('_')[1])

else:

density = float(it.split('_')[1]) / 100

if 'vgg' in name and density > 0.2:

continue

for (expt, points) in sorted(expts.items()):

if 'resnet' in name and is_iterative == False and 'prune' not in it:

continue

if plot_zones and 'reinit' in expt:

continue

xs = []

y_means = []

y_errs_lower = []

y_errs_upper = []

for (point, delta_dict) in sorted(points.items()):

deltas = list(delta_dict.values())

xs.append(point)

if RANGE_METHOD == MEDIAN_MAX:

y_means.append(np.median(deltas))

y_errs_lower.append(y_means[-1] - np.min(deltas))

y_errs_upper.append(np.max(deltas) - y_means[-1])

elif RANGE_METHOD == MEAN_STD:

y_means.append(np.mean(deltas))

y_errs_lower.append(np.std(deltas) if len(deltas) > 1 else 0)

y_errs_upper.append(np.std(deltas) if len(deltas) > 1 else 0)

else:

raise ValueError(RANGE_METHOD)

idx = np.argmax(y_means)

sparsity = 1 - density

all_xs.add(density)

if y_means[idx] < -0.1:

continue

lines[expt].append((density, xs[idx], y_means[idx], y_errs_lower[idx], y_errs_upper[idx]))

all_xs = sorted(set(x[0] for line in lines.values() for x in line))

fig, lax = plt.subplots()

for expt in sorted(lines.keys(), key=legend_sort_key):

plt.errorbar([x[0] for x in lines[expt]], [x[2] for x in lines[expt]],

[[x[3] for x in lines[expt]], [x[4] for x in lines[expt]]],

fmt=fmtof(expt), label='{} {}'.format(labof(expt), _FLAVOR),

color=get_color(expt),

capsize=5,

)

if is_iterative:

plt.title('{name} best $\Delta$ accuracy across sparsities, iterative pruning'.format(

name=netname(name),

))

else:

plt.title('{name} best $\Delta$ accuracy across sparsities, one-shot pruning'.format(

name=netname(name),

))

lax.set_ylabel('$\Delta$ accuracy')

lax.legend(loc='lower left')

lax.plot([0, max(all_xs)], [0, 0], '--', color=(0,0,0,0.3))

lax.set_ylim(-0.03, 0.01)

lax.set_xlabel('Sparsity (log scale)')

lax.set_xscale('log')

lax.invert_xaxis()

xs = np.power(10, np.linspace(np.log10(min(all_xs)), np.log10(max(all_xs)), 10))

lax.xaxis.set_major_locator(ticker.FixedLocator(xs))

lax.xaxis.set_major_formatter(SparsityFormatter(1))

lax.xaxis.set_minor_locator(ticker.NullLocator())

plt.tight_layout()

format_axes(lax, plot_zones)

vals = lax.get_yticks()

lax.set_yticklabels(['{:+,.2%}'.format(x).replace('%', r'\%') for x in vals])

if savename:

os.makedirs(os.path.join(_DIRNAME, 'figures', savename), exist_ok=True)

plt.savefig(os.path.join(_DIRNAME, 'figures', savename, 'lth.pdf'), dpi=FIG_DPI)

if not show:

plt.close('all')

if show:

print('Regular {} {}'.format(net, style))

latexify()

iters = globals()['{}_{}_deltas'.format(net, style)]()

if filters and not callable(filters[0]):

filters = globals()[filters[0]][filters[1]][1]

for filt in filters:

filt(iters)

print('LTH {} {}'.format(net, style))

if do_latexify:

latexify()

iters = globals()['{}_{}_deltas'.format(net, style)]()

for filt in filters:

filt(iters)

print('LTH {} {}'.format(net, style))

latexify()

iters = globals()['{}_{}_deltas'.format(net, style)]()

for filt in filters:

filt(iters)

if show:

plt.ion()

else:

plt.ioff()

print('Zones: {} {}'.format(net, style))

if do_latexify:

latexify()

iters = globals()['{}_{}_deltas'.format(net, style)]()

for filt in [remove_experiments('reinit'), remove_points(0)]:

filt(iters)

get_zones(iters, net, style=='iterative', *args, **kwargs)

if show:

plt.show()

else:

if show:

plt.ion()

else:

plt.ioff()

print('Implots: {} {}'.format(net, style))

if do_latexify:

latexify()

iters = globals()['{}_{}_deltas'.format(net, style)]()

for filt in [remove_experiments('reinit'), remove_points(0)]:

filt(iters)

if isinstance(experiments, str):

experiments = [experiments]

def remover(d):

for it in d:

for expt in d[it]:

if experiments is not None and not any(e in expt for e in experiments):

continue

for point in points:

if point in d[it][expt]:

del d[it][expt][point]

flip=False,

just_get_data=False,

gen_individual_plots=False,

savename=None,

):

fullname = '{} {}'.format(netname(name), 'Iterative' if is_iterative else 'Oneshot')

lines = collections.defaultdict(list)

if is_iterative:

lname = 'lottery'

fname = 'finetune'

else:

lname = 'oneshot_lottery'

fname = 'oneshot_finetune'

if flip:

fname, lname = lname, fname

all_xs = set()

all_expts = set()

if 'resnet50' in name:

max_x = 90

else:

max_x = 182

for (it, expts) in sorted(iters.items(), key=lambda x: float(x[0].split('_')[-1])):

if it == 'iter_0':

continue

if name == 'resnet20' and is_iterative and int(it.split('_')[1]) > 10:

continue

if is_iterative:

density = 0.8 ** int(it.split('_')[1])

else:

density = float(it.split('_')[1]) / 100

if 'vgg' in name and density > 0.2:

continue

for (expt, points) in sorted(expts.items()):

if 'resnet' in name and is_iterative == False and 'prune' not in it:

continue

xs = []

y_means = []

y_errs_lower = []

y_errs_upper = []

for (point, delta_dict) in sorted(points.items()):

deltas = list(delta_dict.values())

xs.append(point)

if RANGE_METHOD == MEDIAN_MAX:

y_means.append(np.median(deltas))

y_errs_lower.append(y_means[-1] - np.min(deltas))

y_errs_upper.append(np.max(deltas) - y_means[-1])

elif RANGE_METHOD == MEAN_STD:

y_means.append(np.mean(deltas))

y_errs_lower.append(np.std(deltas) if len(deltas) > 1 else 0)

y_errs_upper.append(np.std(deltas) if len(deltas) > 1 else 0)

else:

raise ValueError(RANGE_METHOD)

all_expts.add(expt)

for plotter in xs:

plotter /= max_x

if name == 'resnet50':

trep = 90

else:

trep = 182

if lname in expt:

idx = max(i for i in range(len(xs)) if xs[i] <= trep*plotter)

else:

idx = max((y_means[i], i) for i in range(len(xs)) if xs[i] <= trep*plotter)[1]

sparsity = 1 - density

all_xs.add(plotter)

lines[(expt, plotter)].append((density, xs[idx], y_means[idx], y_errs_lower[idx], y_errs_upper[idx]))

if gen_individual_plots:

for expt, x in list(lines.keys()):

if (lname, x) not in lines and (expt, x) != (fname, 2):

del lines[(expt, x)]

all_xs = sorted(set(line[0] for line in lines[(fname, 2)]))

def fmt_plot(ax, title_plotter):

ax.plot([0, max(all_xs)], [0, 0], '--', color=(0,0,0,0.3))

ax.set_ylim(-0.03, 0.01)

ax.set_xlabel('Sparsity (log scale)')

ax.set_ylabel('$\Delta$ accuracy')

ax.legend(loc='lower left')

ax.set_xscale('log')

ax.invert_xaxis()

xs = np.power(10, np.linspace(np.log10(min(all_xs)), np.log10(max(all_xs)), 10))

ax.xaxis.set_major_locator(ticker.FixedLocator(xs))

ax.xaxis.set_major_formatter(SparsityFormatter(1))

ax.xaxis.set_minor_locator(ticker.NullLocator())

plt.tight_layout()