FilesExpand file tree

 master

/

LDA.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

executable file

·

143 lines (125 loc) · 4.36 KB

 master

/

LDA.py

Copy path

Top

File metadata and controls

• Code
• Blame

executable file

·

143 lines (125 loc) · 4.36 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

# -*- coding: utf-8 -*-

# writer : lgy

# data : 2017-08-29

# function 解决LDA 的话题分析数据处理以及数据矩阵构建

import sys

sys.path.append('../..')

reload(sys)

import pickle

from MyCode import config

import logging

import gensim

import jieba

logging.basicConfig(filename='logger.log', format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

class LDA(object):

def __init__(self, modelName="LDATopicWord_1", word_dic_name="id2word"):

self.modelname = modelName

self.word_dic_name = word_dic_name

self.Lda_model = None

self.word_dic = None

self.topics = None

def setModelName(self, modelName="LDATopicWord_1"):

self.modelname = modelName

def build_word_dic(self, documents):

print "build word dic"

self.word_dic = gensim.corpora.Dictionary(documents=documents)

# print len(self.word_dic)

print "build finished and write dic to file"

with open(config.ModelPath + self.word_dic_name + ".model", 'wb') as fp:

pickle.dump(self.word_dic, fp)

logging.info("build word dic finished and write dic to file")

def buildModel(self, documents, num_topics=100):

modelfile = config.ModelPath + self.modelname + ".model"

print "Train LDA Model ... ..."

# logging.info("构建词典")

# self.build_word_dic(documents)

logging.info("训练模型")

trainCorpus = [self.word_dic.doc2bow(docment) for docment in documents]

tfidf = gensim.models.TfidfModel(trainCorpus)

corpustfidf = tfidf[trainCorpus]

self.Lda_model = gensim.models.LdaModel(corpustfidf, id2word=self.word_dic, num_topics=num_topics, alpha=0.2)

logging.info("存储模型{0}".format(self.modelname))

print "Finished Train LDA Model and saving ... ..."

with open(modelfile, "wb") as model_fp:

pickle.dump(self.Lda_model, model_fp)

print "Finished save LDA Model"

logging.info("build LDA model finished and write dic to file")

def load_word_dic(self):

word_dic_file = config.ModelPath + self.word_dic_name + ".model"

try:

if self.word_dic:

return self.word_dic

logging.info("load word dic ")

print "load word dic ... ..."

with open(word_dic_file, "rb") as fp:

self.word_dic = pickle.load(fp)

print "load success"

except:

print "word dic not found !"

return None

def load_LdaModel(self):

ladmodel_file = config.ModelPath + self.modelname + ".model"

try:

if self.Lda_model:

return self.Lda_model

logging.info("load LDA Model ")

print "load LDA Model ... ..."

with open(ladmodel_file, "rb") as fp:

self.Lda_model = pickle.load(fp)

print "load success"

except:

print "LDA Model not found !"

return None

# # 处理语句成query

# def getQueries(sentences):

# par_sentences = Participle.Participle(sentences)

# # thl = thulac.thulac()

# # par_sentences = [thl.cut(sen.encode("utf-8")) for sen in sentences]

# # documents = filterStopWords.filterStopWordFromSentences(par_sentences)

# documents = par_sentences

# return documents

def getWordsId(self, document):

documentId = self.word_dic.doc2bow(document)

return documentId

def getQuery(self, sentence):

par_sentences = jieba.cut(sentence.strip())

documents = par_sentences

return documents

def get_topic_distribute(self, sentence):

logging.info("计算文档话题分布")

sentence = list(jieba.cut(sentence))

# print sentence

sentence_id = self.word_dic.doc2bow(sentence)

return self.Lda_model

# 处理query 的话题

def getQuerySimilarly(self, query, top_n=100):

logging.info("计算文档集的话题")

sentence = list(jieba.cut(query))

# print sentence

sentence_id = self.word_dic.doc2bow(sentence)

doc_topic = self.Lda_model.get_document_topics(sentence_id)[:top_n]

if doc_topic is None:

return None

doc_topic = sorted(doc_topic, key=lambda X: X[1], reverse=True)

return doc_topic

def getTopicofWOrd(self, query, query_id):

"""

返回query words most probobily topic

:param query:

:return:

"""

sentence = query.strip().split(' ')

# print sentence

sentence_id = self.word_dic.doc2bow(sentence)

self.Lda_model.get_document_topics(sentence_id)

words_topic = []

# print sentence_id

for word_id in sentence_id:

# print word_id

word_topic = self.Lda_model.get_term_topics(word_id[0])

# print word_topic

if word_topic:

words_topic.append([(query_id, word_topic[0][0]), word_topic[0][1]])

return words_topic

def show(self):

self.Lda_model.show_topics(num_topics=10, num_words=10)