FilesExpand file tree

 master

/

tutorial_imdb_fasttext.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

157 lines (112 loc) · 5.35 KB

 master

/

tutorial_imdb_fasttext.py

Copy path

Top

File metadata and controls

• Code
• Blame

157 lines (112 loc) · 5.35 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

#!/usr/bin/env python

"""

This demo implements FastText[1] for sentence classification.

FastText is a simple model for text classification with performance often close

to state-of-the-art, and is useful as a solid baseline.

There are some important differences between this implementation and what

is described in the paper. Instead of Hogwild! SGD[2], we use Adam optimizer

with mini-batches. Hierarchical softmax is also not supported; if you have

a large label space, consider utilizing candidate sampling methods provided

by TensorFlow[3].

After 5 epochs, you should get test accuracy close to 90.9%.

[1] Joulin, A., Grave, E., Bojanowski, P., & Mikolov, T. (2016).

Bag of Tricks for Efficient Text Classification.

http://arxiv.org/abs/1607.01759

[2] Recht, B., Re, C., Wright, S., & Niu, F. (2011).

Hogwild: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent.

In Advances in Neural Information Processing Systems 24 (pp. 693–701).

[3] https://www.tensorflow.org/api_guides/python/nn#Candidate_Sampling

"""

import array

import hashlib

import time

import numpy as np

import tensorflow as tf

import tensorlayer as tl

from tensorlayer.layers import *

# Hashed n-grams with 1 < n <= N_GRAM are included as features

# in addition to unigrams.

N_GRAM = 2

# Size of vocabulary; less frequent words will be treated as "unknown"

VOCAB_SIZE = 100000

# Number of buckets used for hashing n-grams

N_BUCKETS = 1000000

# Size of the embedding vectors

EMBEDDING_SIZE = 50

# Number of epochs for which the model is trained

N_EPOCH = 5

# Size of training mini-batches

BATCH_SIZE = 32

# Path to which to save the trained model

MODEL_FILE_PATH = 'model.npz'

class FastTextClassifier(object):

"""Simple wrapper class for creating the graph of FastText classifier."""

def __init__(self, vocab_size, embedding_size, n_labels):

self.vocab_size = vocab_size

self.embedding_size = embedding_size

self.n_labels = n_labels

self.inputs = tf.placeholder(tf.int32, shape=[None, None], name='inputs')

self.labels = tf.placeholder(tf.int32, shape=[None], name='labels')

# Network structure

network = AverageEmbeddingInputlayer(self.inputs, self.vocab_size, self.embedding_size)

self.network = DenseLayer(network, self.n_labels)

# Training operation

cost = tl.cost.cross_entropy(self.network.outputs, self.labels, name='cost')

self.train_op = tf.train.AdamOptimizer().minimize(cost)

# Predictions

self.prediction_probs = tf.nn.softmax(self.network.outputs)

self.predictions = tf.argmax(self.network.outputs, axis=1, output_type=tf.int32)

# self.predictions = tf.cast(tf.argmax( # for TF < 1.2

# self.network.outputs, axis=1), tf.int32)

# Evaluation

are_predictions_correct = tf.equal(self.predictions, self.labels)

self.accuracy = tf.reduce_mean(tf.cast(are_predictions_correct, tf.float32))

def save(self, sess, filename):

tl.files.save_npz(self.network.all_params, name=filename, sess=sess)

def load(self, sess, filename):

tl.files.load_and_assign_npz(sess, name=filename, network=self.network)

def augment_with_ngrams(unigrams, unigram_vocab_size, n_buckets, n=2):

"""Augment unigram features with hashed n-gram features."""

def get_ngrams(n):

return list(zip(*[unigrams[i:] for i in range(n)]))

def hash_ngram(ngram):

bytes_ = array.array('L', ngram).tobytes()

hash_ = int(hashlib.sha256(bytes_).hexdigest(), 16)

return unigram_vocab_size + hash_ % n_buckets

return unigrams + [hash_ngram(ngram) for i in range(2, n + 1) for ngram in get_ngrams(i)]

def load_and_preprocess_imdb_data(n_gram=None):

"""Load IMDb data and augment with hashed n-gram features."""

X_train, y_train, X_test, y_test = \

tl.files.load_imdb_dataset(nb_words=VOCAB_SIZE)

if n_gram is not None:

X_train = np.array([augment_with_ngrams(x, VOCAB_SIZE, N_BUCKETS, n=n_gram) for x in X_train])

X_test = np.array([augment_with_ngrams(x, VOCAB_SIZE, N_BUCKETS, n=n_gram) for x in X_test])

return X_train, y_train, X_test, y_test

def train_test_and_save_model():

X_train, y_train, X_test, y_test = load_and_preprocess_imdb_data(N_GRAM)

classifier = FastTextClassifier(

vocab_size=VOCAB_SIZE + N_BUCKETS,

embedding_size=EMBEDDING_SIZE,

n_labels=2,

)

with tf.Session() as sess:

tl.layers.initialize_global_variables(sess)

for epoch in range(N_EPOCH):

start_time = time.time()

print('Epoch %d/%d' % (epoch + 1, N_EPOCH))

for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size=BATCH_SIZE, shuffle=True):

sess.run(

classifier.train_op, feed_dict={

classifier.inputs: tl.prepro.pad_sequences(X_batch),

classifier.labels: y_batch,

})

print(" took %.5fs" % (time.time() - start_time))

test_accuracy = sess.run(

classifier.accuracy, feed_dict={

classifier.inputs: tl.prepro.pad_sequences(X_test),

classifier.labels: y_test,

})

print('Test accuracy: %.5f' % test_accuracy)

classifier.save(sess, MODEL_FILE_PATH)

if __name__ == '__main__':

train_test_and_save_model()