"""Deep Belief Network

def __init__(self, numpy_rng, theano_rng=None, n_ins=784,

hidden_layers_sizes=[500, 500], n_outs=10):

"""This class is made to support a variable number of layers.

self.sigmoid_layers = []

self.rbm_layers = []

self.params = []

self.n_layers = len(hidden_layers_sizes)

assert self.n_layers > 0

if not theano_rng:

theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))

# allocate symbolic variables for the data

self.x = T.matrix('x') # the data is presented as rasterized images

self.y = T.ivector('y') # the labels are presented as 1D vector

# of [int] labels

# end-snippet-1

# The DBN is an MLP, for which all weights of intermediate

# layers are shared with a different RBM. We will first

# construct the DBN as a deep multilayer perceptron, and when

# constructing each sigmoidal layer we also construct an RBM

# that shares weights with that layer. During pretraining we

# will train these RBMs (which will lead to chainging the

# weights of the MLP as well) During finetuning we will finish

# training the DBN by doing stochastic gradient descent on the

# MLP.

for i in xrange(self.n_layers):

# construct the sigmoidal layer

# the size of the input is either the number of hidden

# units of the layer below or the input size if we are on

# the first layer

if i == 0:

input_size = n_ins

else:

input_size = hidden_layers_sizes[i - 1]

# the input to this layer is either the activation of the

# hidden layer below or the input of the DBN if you are on

# the first layer

if i == 0:

layer_input = self.x

else:

layer_input = self.sigmoid_layers[-1].output

sigmoid_layer = HiddenLayer(rng=numpy_rng,

input=layer_input,

n_in=input_size,

n_out=hidden_layers_sizes[i],

activation=T.nnet.sigmoid)

# add the layer to our list of layers

self.sigmoid_layers.append(sigmoid_layer)

# its arguably a philosophical question... but we are

# going to only declare that the parameters of the

# sigmoid_layers are parameters of the DBN. The visible

# biases in the RBM are parameters of those RBMs, but not

# of the DBN.

self.params.extend(sigmoid_layer.params)

# Construct an RBM that shared weights with this layer

rbm_layer = RBM(numpy_rng=numpy_rng,

theano_rng=theano_rng,

input=layer_input,

n_visible=input_size,

n_hidden=hidden_layers_sizes[i],

W=sigmoid_layer.W,

hbias=sigmoid_layer.b)

self.rbm_layers.append(rbm_layer)

# We now need to add a logistic layer on top of the MLP

self.logLayer = LogisticRegression(

input=self.sigmoid_layers[-1].output,

n_in=hidden_layers_sizes[-1],

n_out=n_outs)

self.params.extend(self.logLayer.params)

# compute the cost for second phase of training, defined as the

# negative log likelihood of the logistic regression (output) layer

self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)

# compute the gradients with respect to the model parameters

# symbolic variable that points to the number of errors made on the

# minibatch given by self.x and self.y

self.errors = self.logLayer.errors(self.y)

def pretraining_functions(self, train_set_x, batch_size, k):

'''Generates a list of functions, for performing one step of

# index to a [mini]batch

index = T.lscalar('index') # index to a minibatch

learning_rate = T.scalar('lr') # learning rate to use

# number of batches

n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

# begining of a batch, given `index`

batch_begin = index * batch_size

# ending of a batch given `index`

batch_end = batch_begin + batch_size

pretrain_fns = []

for rbm in self.rbm_layers:

# get the cost and the updates list

# using CD-k here (persisent=None) for training each RBM.

# TODO: change cost function to reconstruction error

cost, updates = rbm.get_cost_updates(learning_rate,

persistent=None, k=k)

# compile the theano function

fn = theano.function(

inputs=[index, theano.Param(learning_rate, default=0.1)],

outputs=cost,

updates=updates,

givens={

self.x: train_set_x[batch_begin:batch_end]

}

)

# append `fn` to the list of functions

pretrain_fns.append(fn)

return pretrain_fns

def build_finetune_functions(self, datasets, batch_size, learning_rate):

'''Generates a function `train` that implements one step of

(train_set_x, train_set_y) = datasets[0]

(valid_set_x, valid_set_y) = datasets[1]

(test_set_x, test_set_y) = datasets[2]

# compute number of minibatches for training, validation and testing

n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]

n_valid_batches /= batch_size

n_test_batches = test_set_x.get_value(borrow=True).shape[0]

n_test_batches /= batch_size

index = T.lscalar('index') # index to a [mini]batch

# compute the gradients with respect to the model parameters

gparams = T.grad(self.finetune_cost, self.params)

# compute list of fine-tuning updates

updates = []

for param, gparam in zip(self.params, gparams):

updates.append((param, param - gparam * learning_rate))

train_fn = theano.function(

inputs=[index],

outputs=self.finetune_cost,

updates=updates,

givens={

self.x: train_set_x[

index * batch_size: (index + 1) * batch_size

],

self.y: train_set_y[

index * batch_size: (index + 1) * batch_size

]

}

)

test_score_i = theano.function(

[index],

self.errors,

givens={

self.x: test_set_x[

index * batch_size: (index + 1) * batch_size

],

self.y: test_set_y[

index * batch_size: (index + 1) * batch_size

]

}

)

valid_score_i = theano.function(

[index],

self.errors,

givens={

self.x: valid_set_x[

index * batch_size: (index + 1) * batch_size

],

self.y: valid_set_y[

index * batch_size: (index + 1) * batch_size

]

}

)

# Create a function that scans the entire validation set

def valid_score():

return [valid_score_i(i) for i in xrange(n_valid_batches)]

# Create a function that scans the entire test set

def test_score():

return [test_score_i(i) for i in xrange(n_test_batches)]

pretrain_lr=0.01, k=1, training_epochs=1000,

dataset='mnist.pkl.gz', batch_size=10):

"""

datasets = load_data(dataset)

train_set_x, train_set_y = datasets[0]

valid_set_x, valid_set_y = datasets[1]

test_set_x, test_set_y = datasets[2]

# compute number of minibatches for training, validation and testing

n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

# numpy random generator

numpy_rng = numpy.random.RandomState(123)

print '... building the model'

# construct the Deep Belief Network

dbn = DBN(numpy_rng=numpy_rng, n_ins=28 * 28,

hidden_layers_sizes=[1000, 1000, 1000],

n_outs=10)

# start-snippet-2

#########################

# PRETRAINING THE MODEL #

#########################

print '... getting the pretraining functions'

pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,

batch_size=batch_size,

k=k)

print '... pre-training the model'

start_time = time.clock()

## Pre-train layer-wise

for i in xrange(dbn.n_layers):

# go through pretraining epochs

for epoch in xrange(pretraining_epochs):

# go through the training set

c = []

for batch_index in xrange(n_train_batches):

c.append(pretraining_fns[i](index=batch_index,

lr=pretrain_lr))

print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),

print numpy.mean(c)

end_time = time.clock()

# end-snippet-2

print >> sys.stderr, ('The pretraining code for file ' +

os.path.split(__file__)[1] +

' ran for %.2fm' % ((end_time - start_time) / 60.))

########################

# FINETUNING THE MODEL #

########################

# get the training, validation and testing function for the model

print '... getting the finetuning functions'

train_fn, validate_model, test_model = dbn.build_finetune_functions(

datasets=datasets,

batch_size=batch_size,

learning_rate=finetune_lr

)

print '... finetuning the model'

# early-stopping parameters

patience = 4 * n_train_batches # look as this many examples regardless

patience_increase = 2. # wait this much longer when a new best is

# found

improvement_threshold = 0.995 # a relative improvement of this much is

# considered significant

validation_frequency = min(n_train_batches, patience / 2)

# go through this many

# minibatches before checking the network

# on the validation set; in this case we

# check every epoch

best_validation_loss = numpy.inf

test_score = 0.

start_time = time.clock()

done_looping = False

epoch = 0

while (epoch < training_epochs) and (not done_looping):

epoch = epoch + 1

for minibatch_index in xrange(n_train_batches):

minibatch_avg_cost = train_fn(minibatch_index)

iter = (epoch - 1) * n_train_batches + minibatch_index

if (iter + 1) % validation_frequency == 0:

validation_losses = validate_model()

this_validation_loss = numpy.mean(validation_losses)

print(

'epoch %i, minibatch %i/%i, validation error %f %%'

% (

epoch,

minibatch_index + 1,

n_train_batches,

this_validation_loss * 100.

)

)

# if we got the best validation score until now

if this_validation_loss < best_validation_loss:

#improve patience if loss improvement is good enough

if (

this_validation_loss < best_validation_loss *

improvement_threshold

):

patience = max(patience, iter * patience_increase)

# save best validation score and iteration number

best_validation_loss = this_validation_loss

best_iter = iter

# test it on the test set

test_losses = test_model()

test_score = numpy.mean(test_losses)

print((' epoch %i, minibatch %i/%i, test error of '

'best model %f %%') %

(epoch, minibatch_index + 1, n_train_batches,

test_score * 100.))

if patience <= iter:

done_looping = True

break

end_time = time.clock()

print(

(

'Optimization complete with best validation score of %f %%, '

'obtained at iteration %i, '

'with test performance %f %%'

) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)

)

print >> sys.stderr, ('The fine tuning code for file ' +

os.path.split(__file__)[1] +

' ran for %.2fm' % ((end_time - start_time)