FilesExpand file tree

 master

/

preProcess.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

45 lines (37 loc) · 1.12 KB

 master

/

preProcess.py

Copy path

Top

File metadata and controls

• Code
• Blame

45 lines (37 loc) · 1.12 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

# -*- coding: utf-8 -*-

# writer : lgy

# data : 2017-07-24

# today i want to write my first code file in company

def check_contain_chinese(check_str):

for ch in check_str.decode('utf-8'):

if u'\u4e00' <= ch <= u'\u9fff':

return True

return False

def InputFileProcess():

fwfp = open("../Result/corpus.txt","w")

with open("../Data/corpus_lgy.txt",'r') as fp:

line = fp.readline()

while line:

line = line.replace("<content>",'')

line = line.replace('</content>','')

if len(line) > 5:

fwfp.write(line)

line = fp.readline()

fp.close()

# 句子过滤 sentences filter remove contecxt latter '\t'

def removeContext(Sentences):

new_sentences = []

for sentence in Sentences:

line = sentence.split("\t")[0]

new_sentences.append(line)

return new_sentences

def getWordsFromSentences(sentences):

Words = []

for sentence in sentences:

for w in sentence:

Words.append(w)

return list(set(Words))

def main():

InputFileProcess()

if __name__ == '__main__':

main()