"""Reshapes high-dimension input to a vector.

dim = 1

for d in variable.get_shape()[1:].as_list():

dim *= d

"""Clear all layer names in set_keep['_layers_name_list'],

"""Enable or disable reuse layer name. By default, each layer must has unique

"""Return the initialized RNN state.

try: # TF1.0

LSTMStateTuple = tf.contrib.rnn.LSTMStateTuple

except:

LSTMStateTuple = tf.nn.rnn_cell.LSTMStateTuple

if isinstance(state, LSTMStateTuple):

c = state.c.eval()

h = state.h.eval()

return (c, h)

else:

new_state = state.eval()

"""Print all trainable and non-trainable variables

# tvar = tf.trainable_variables() if train_only else tf.all_variables()

if train_only:

t_vars = tf.trainable_variables()

print(" [*] printing trainable variables")

else:

try: # TF1.0+

t_vars = tf.global_variables()

except: # TF0.12

t_vars = tf.all_variables()

print(" [*] printing global variables")

for idx, v in enumerate(t_vars):

"""Get variable list by a given name scope.

print(" [*] geting variables with %s" % name)

# tvar = tf.trainable_variables() if train_only else tf.all_variables()

if train_only:

t_vars = tf.trainable_variables()

else:

try: # TF1.0+

t_vars = tf.global_variables()

except: # TF0.12

t_vars = tf.all_variables()

d_vars = [var for var in t_vars if name in var.name]

if printable:

for idx, v in enumerate(d_vars):

print(" got {:3}: {:15} {}".format(idx, v.name, str(v.get_shape())))

"""Get layer list in a network by a given name scope.

assert network is not None

print(" [*] geting layers with %s" % name)

layers = []

i = 0

for layer in network.all_layers:

# print(type(layer.name))

if name in layer.name:

layers.append(layer)

if printable:

print(" got {:3}: {:15} {}".format(i, layer.name, str(layer.get_shape())))

i = i + 1

"""Remove the repeated items in a list, and return the processed list.

l2 = []

[l2.append(i) for i in l if not i in l2]

"""Merge all parameters, layers and dropout probabilities to a :class:`Layer`.

layer = layers[0]

all_params = []

all_layers = []

all_drop = {}

for l in layers:

all_params.extend(l.all_params)

all_layers.extend(l.all_layers)

all_drop.update(l.all_drop)

layer.all_params = list(all_params)

layer.all_layers = list(all_layers)

layer.all_drop = dict(all_drop)

"""Excute ``sess.run(tf.global_variables_initializer())`` for TF12+ or

assert sess is not None

# try: # TF12+

sess.run(tf.global_variables_initializer())

# except: # TF11

"""

def __init__(

self,

inputs = None,

name ='layer'

):

self.inputs = inputs

scope_name = tf.get_variable_scope().name

if scope_name:

name = scope_name + '/' + name

if (name in set_keep['_layers_name_list']) and set_keep['name_reuse'] == False:

raise Exception("Layer '%s' already exists, please choice other 'name' or reuse this layer\

\nHint : Use different name for different 'Layer' (The name is used to control parameter sharing)" % name)

else:

self.name = name

if name not in ['', None, False]:

set_keep['_layers_name_list'].append(name)

def print_params(self, details=True):

''' Print all info of parameters in the network'''

for i, p in enumerate(self.all_params):

if details:

try:

# print(" param {:3}: {:15} (mean: {:<18}, median: {:<18}, std: {:<18}) {}".format(i, str(p.eval().shape), p.eval().mean(), np.median(p.eval()), p.eval().std(), p.name))

val = p.eval()

print(" param {:3}: {:20} {:15} {} (mean: {:<18}, median: {:<18}, std: {:<18}) ".format(i, p.name, str(val.shape), p.dtype.name, val.mean(), np.median(val), val.std()))

except Exception as e:

print(str(e))

raise Exception("Hint: print params details after tl.layers.initialize_global_variables(sess) or use network.print_params(False).")

else:

print(" param {:3}: {:20} {:15} {}".format(i, p.name, str(p.get_shape()), p.dtype.name))

print(" num of params: %d" % self.count_params())

def print_layers(self):

''' Print all info of layers in the network '''

for i, layer in enumerate(self.all_layers):

# print(" layer %d: %s" % (i, str(layer)))

print(" layer {:3}: {:20} {:15} {}".format(i, layer.name, str(layer.get_shape()), layer.dtype.name))

def count_params(self):

''' Return the number of parameters in the network '''

n_params = 0

for i, p in enumerate(self.all_params):

n = 1

# for s in p.eval().shape:

for s in p.get_shape():

try:

s = int(s)

except:

s = 1

if s:

n = n * s

n_params = n_params + n

return n_params

def __str__(self):

# print("\nIt is a Layer class")

# self.print_params(False)

# self.print_layers()

"""

def __init__(

self,

inputs = None,

name ='input_layer'

):

Layer.__init__(self, inputs=inputs, name=name)

print(" [TL] InputLayer %s: %s" % (self.name, inputs.get_shape()))

self.outputs = inputs

self.all_layers = []

self.all_params = []

"""

def __init__(

self,

inputs = None,

depth = None,

on_value = None,

off_value = None,

axis = None,

dtype=None,

name ='input_layer'

):

Layer.__init__(self, inputs=inputs, name=name)

assert depth != None, "depth is not given"

print(" [TL]:Instantiate OneHotInputLayer %s: %s" % (self.name, inputs.get_shape()))

self.outputs = tf.one_hot(inputs, depth, on_value=on_value, off_value=off_value, axis=axis, dtype=dtype)

self.all_layers = []

self.all_params = []

"""

def __init__(

self,

inputs = None,

train_labels = None,

vocabulary_size = 80000,

embedding_size = 200,

num_sampled = 64,

nce_loss_args = {},

E_init = tf.random_uniform_initializer(minval=-1.0, maxval=1.0),

E_init_args = {},

nce_W_init = tf.truncated_normal_initializer(stddev=0.03),

nce_W_init_args = {},

nce_b_init = tf.constant_initializer(value=0.0),

nce_b_init_args = {},

name ='word2vec_layer',

):

Layer.__init__(self, name=name)

self.inputs = inputs

print(" [TL] Word2vecEmbeddingInputlayer %s: (%d, %d)" % (self.name, vocabulary_size, embedding_size))

# Look up embeddings for inputs.

# Note: a row of 'embeddings' is the vector representation of a word.

# for the sake of speed, it is better to slice the embedding matrix

# instead of transfering a word id to one-hot-format vector and then

# multiply by the embedding matrix.

# embed is the outputs of the hidden layer (embedding layer), it is a

# row vector with 'embedding_size' values.

with tf.variable_scope(name) as vs:

embeddings = tf.get_variable(name='embeddings',

shape=(vocabulary_size, embedding_size),

initializer=E_init,

**E_init_args)

embed = tf.nn.embedding_lookup(embeddings, self.inputs)

# Construct the variables for the NCE loss (i.e. negative sampling)

nce_weights = tf.get_variable(name='nce_weights',

shape=(vocabulary_size, embedding_size),

initializer=nce_W_init,

**nce_W_init_args)

nce_biases = tf.get_variable(name='nce_biases',

shape=(vocabulary_size),

initializer=nce_b_init,

**nce_b_init_args)

# Compute the average NCE loss for the batch.

# tf.nce_loss automatically draws a new sample of the negative labels

# each time we evaluate the loss.

self.nce_cost = tf.reduce_mean(

tf.nn.nce_loss(weights=nce_weights, biases=nce_biases,

inputs=embed, labels=train_labels,

num_sampled=num_sampled, num_classes=vocabulary_size,

**nce_loss_args))

self.outputs = embed

self.normalized_embeddings = tf.nn.l2_normalize(embeddings, 1)

self.all_layers = [self.outputs]

self.all_params = [embeddings, nce_weights, nce_biases]

"""

def __init__(

self,

inputs = None,

vocabulary_size = 80000,

embedding_size = 200,

E_init = tf.random_uniform_initializer(-0.1, 0.1),

E_init_args = {},

name ='embedding_layer',

):

Layer.__init__(self, name=name)

self.inputs = inputs

print(" [TL] EmbeddingInputlayer %s: (%d, %d)" % (self.name, vocabulary_size, embedding_size))

with tf.variable_scope(name) as vs:

embeddings = tf.get_variable(name='embeddings',

shape=(vocabulary_size, embedding_size),

initializer=E_init,

**E_init_args)

embed = tf.nn.embedding_lookup(embeddings, self.inputs)

self.outputs = embed

self.all_layers = [self.outputs]

self.all_params = [embeddings]

"""The :class:`AverageEmbeddingInputlayer` averages over embeddings of inputs, can be used as the input layer for models like DAN[1] and FastText[2].

def __init__(

self, inputs, vocabulary_size, embedding_size,

pad_value=0,

name='average_embedding_layer',

embeddings_initializer=tf.random_uniform_initializer(-0.1, 0.1),

embeddings_kwargs=None,

):

super().__init__(name=name)

# if embeddings_kwargs is None:

# embeddings_kwargs = {}

if inputs.get_shape().ndims != 2:

raise ValueError(

'inputs must be of size batch_size * batch_sentence_length')

self.inputs = inputs

print(" [TL] AverageEmbeddingInputlayer %s: (%d, %d)" % (name, vocabulary_size, embedding_size))

with tf.variable_scope(name):

self.embeddings = tf.get_variable(

name='embeddings',

shape=(vocabulary_size, embedding_size),

initializer=embeddings_initializer,

**(embeddings_kwargs or {})

# **embeddings_kwargs

) # **(embeddings_kwargs or {}),

word_embeddings = tf.nn.embedding_lookup(

self.embeddings, self.inputs,

name='word_embeddings',

)

# Zero out embeddings of pad value

masks = tf.not_equal(self.inputs, pad_value, name='masks')

word_embeddings *= tf.cast(

tf.expand_dims(masks, axis=-1),

tf.float32,

)

sum_word_embeddings = tf.reduce_sum(word_embeddings, axis=1)

# Count number of non-padding words in each sentence

sentence_lengths = tf.count_nonzero(

masks,

axis=1,

keep_dims=True,

dtype=tf.float32,

name='sentence_lengths',

)

sentence_embeddings = tf.divide(

sum_word_embeddings,

sentence_lengths + 1e-8, # Add epsilon to avoid dividing by 0

name='sentence_embeddings'

)

self.outputs = sentence_embeddings

self.all_layers = [self.outputs]

self.all_params = [self.embeddings]

"""

def __init__(

self,

layer = None,

n_units = 100,

act = tf.identity,

W_init = tf.truncated_normal_initializer(stddev=0.1),

b_init = tf.constant_initializer(value=0.0),

W_init_args = {},

b_init_args = {},

name ='dense_layer',

):

Layer.__init__(self, name=name)

self.inputs = layer.outputs

if self.inputs.get_shape().ndims != 2:

raise Exception("The input dimension must be rank 2, please reshape or flatten it")

n_in = int(self.inputs.get_shape()[-1])

self.n_units = n_units

print(" [TL] DenseLayer %s: %d %s" % (self.name, self.n_units, act.__name__))

with tf.variable_scope(name) as vs:

W = tf.get_variable(name='W', shape=(n_in, n_units), initializer=W_init, **W_init_args )

if b_init is not None:

try:

b = tf.get_variable(name='b', shape=(n_units), initializer=b_init, **b_init_args )

except: # If initializer is a constant, do not specify shape.

b = tf.get_variable(name='b', initializer=b_init, **b_init_args )

self.outputs = act(tf.matmul(self.inputs, W) + b)

else:

self.outputs = act(tf.matmul(self.inputs, W))

# Hint : list(), dict() is pass by value (shallow), without them, it is

# pass by reference.

self.all_layers = list(layer.all_layers)

self.all_params = list(layer.all_params)

self.all_drop = dict(layer.all_drop)

self.all_layers.extend( [self.outputs] )

if b_init is not None:

self.all_params.extend( [W, b] )

else:

"""

def __init__(

self,

layer = None,

x_recon = None,

name = 'recon_layer',

n_units = 784,

act = tf.nn.softplus,

):

DenseLayer.__init__(self, layer=layer, n_units=n_units, act=act, name=name)

print(" [TL] %s is a ReconLayer" % self.name)

# y : reconstruction outputs; train_params : parameters to train

# Note that: train_params = [W_encoder, b_encoder, W_decoder, b_encoder]

y = self.outputs

self.train_params = self.all_params[-4:]

# =====================================================================

#

# You need to modify the below cost function and optimizer so as to

# implement your own pre-train method.

#

# =====================================================================

lambda_l2_w = 0.004

learning_rate = 0.0001

print(" lambda_l2_w: %f" % lambda_l2_w)

print(" learning_rate: %f" % learning_rate)

# Mean-square-error i.e. quadratic-cost

mse = tf.reduce_sum(tf.squared_difference(y, x_recon), 1)

mse = tf.reduce_mean(mse) # in theano: mse = ((y - x) ** 2 ).sum(axis=1).mean()

# mse = tf.reduce_mean(tf.reduce_sum(tf.square(tf.sub(y, x_recon)), 1))

# mse = tf.reduce_mean(tf.squared_difference(y, x_recon)) # <haodong>: Error

# mse = tf.sqrt(tf.reduce_mean(tf.square(y - x_recon))) # <haodong>: Error

# Cross-entropy

# ce = cost.cross_entropy(y, x_recon) # <haodong>: list , list , Error (only be used for softmax output)

# ce = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y, x_recon)) # <haodong>: list , list , Error (only be used for softmax output)

# ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, x_recon)) # <haodong>: list , index , Error (only be used for softmax output)

L2_w = tf.contrib.layers.l2_regularizer(lambda_l2_w)(self.train_params[0]) \

+ tf.contrib.layers.l2_regularizer(lambda_l2_w)(self.train_params[2]) # faster than the code below

# L2_w = lambda_l2_w * tf.reduce_mean(tf.square(self.train_params[0])) + lambda_l2_w * tf.reduce_mean( tf.square(self.train_params[2]))

# DropNeuro

P_o = cost.lo_regularizer(0.03)(self.train_params[0]) # + cost.lo_regularizer(0.5)(self.train_params[2]) # <haodong>: if add lo on decoder, no neuron will be broken

P_i = cost.li_regularizer(0.03)(self.train_params[0]) # + cost.li_regularizer(0.001)(self.train_params[2])

# L1 of activation outputs

activation_out = self.all_layers[-2]

L1_a = 0.001 * tf.reduce_mean(activation_out) # <haodong>: theano: T.mean( self.a[i] ) # some neuron are broken, white and black

# L1_a = 0.001 * tf.reduce_mean( tf.reduce_sum(activation_out, 0) ) # <haodong>: some neuron are broken, white and black

# L1_a = 0.001 * 100 * tf.reduce_mean( tf.reduce_sum(activation_out, 1) ) # <haodong>: some neuron are broken, white and black

# KL Divergence

beta = 4

rho = 0.15

p_hat = tf.reduce_mean(activation_out, 0) # theano: p_hat = T.mean( self.a[i], axis=0 )

try: ## TF1.0

KLD = beta * tf.reduce_sum( rho * tf.log(tf.divide(rho, p_hat)) + (1- rho) * tf.log((1- rho)/ (tf.subtract(float(1), p_hat))) )

except: ## TF0.12

KLD = beta * tf.reduce_sum( rho * tf.log(tf.div(rho, p_hat)) + (1- rho) * tf.log((1- rho)/ (tf.sub(float(1), p_hat))) )

# KLD = beta * tf.reduce_sum( rho * tf.log(rho/ p_hat) + (1- rho) * tf.log((1- rho)/(1- p_hat)) )

# theano: L1_a = l1_a[i] * T.sum( rho[i] * T.log(rho[i]/ p_hat) + (1- rho[i]) * T.log((1- rho[i])/(1- p_hat)) )

# Total cost

if act == tf.nn.softplus:

print(' use: mse, L2_w, L1_a')

self.cost = mse + L1_a + L2_w

elif act == tf.nn.sigmoid:

# ----------------------------------------------------

# Cross-entropy was used in Denoising AE

# print(' use: ce, L2_w, KLD')

# self.cost = ce + L2_w + KLD

# ----------------------------------------------------

# Mean-squared-error was used in Vanilla AE

print(' use: mse, L2_w, KLD')

self.cost = mse + L2_w + KLD

# ----------------------------------------------------

# Add DropNeuro penalty (P_o) can remove neurons of AE

# print(' use: mse, L2_w, KLD, P_o')

# self.cost = mse + L2_w + KLD + P_o

# ----------------------------------------------------

# Add DropNeuro penalty (P_i) can remove neurons of previous layer

# If previous layer is InputLayer, it means remove useless features

# print(' use: mse, L2_w, KLD, P_i')

# self.cost = mse + L2_w + KLD + P_i

else:

raise Exception("Don't support the given reconstruct activation function")

self.train_op = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999,

epsilon=1e-08, use_locking=False).minimize(self.cost, var_list=self.train_params)

# self.train_op = tf.train.GradientDescentOptimizer(1.0).minimize(self.cost, var_list=self.train_params)

def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10,

save=True, save_name='w1pre_'):

# ====================================================

#

# You need to modify the cost function in __init__() so as to

# get your own pre-train method.

#

# ====================================================

print(" [*] %s start pretrain" % self.name)

print(" batch_size: %d" % batch_size)

if denoise_name:

print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]])

dp_denoise = self.all_drop[set_keep[denoise_name]]

else:

print(" no denoising layer")

for epoch in range(n_epoch):

start_time = time.time()

for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):

dp_dict = utils.dict_to_one( self.all_drop )

if denoise_name:

dp_dict[set_keep[denoise_name]] = dp_denoise

feed_dict = {x: X_train_a}

feed_dict.update(dp_dict)

sess.run(self.train_op, feed_dict=feed_dict)

if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:

print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))

train_loss, n_batch = 0, 0

for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):

dp_dict = utils.dict_to_one( self.all_drop )

feed_dict = {x: X_train_a}

feed_dict.update(dp_dict)

err = sess.run(self.cost, feed_dict=feed_dict)

train_loss += err

n_batch += 1

print(" train loss: %f" % (train_loss/ n_batch))

val_loss, n_batch = 0, 0

for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True):

dp_dict = utils.dict_to_one( self.all_drop )

feed_dict = {x: X_val_a}

feed_dict.update(dp_dict)

err = sess.run(self.cost, feed_dict=feed_dict)

val_loss += err

n_batch += 1

print(" val loss: %f" % (val_loss/ n_batch))

if save:

try:

visualize.W(self.train_params[0].eval(), second=10, saveable=True, shape=[28,28], name=save_name+str(epoch+1), fig_idx=2012)

files.save_npz([self.all_params[0]] , name=save_name+str(epoch+1)+'.npz')

except: