FilesExpand file tree

 master

/

threading_queue.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

81 lines (64 loc) · 1.95 KB

 master

/

threading_queue.py

Copy path

Top

File metadata and controls

• Code
• Blame

81 lines (64 loc) · 1.95 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

#!/usr/bin/env python

import Queue

import threading

import urllib2

import time

from BeautifulSoup import BeautifulSoup

hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com", "http://ibm.com", "http://apple.com"]

queue = Queue.Queue()

out_queue = Queue.Queue()

class ThreadUrl(threading.Thread):

"""

Threaded Url Grab

"""

def __init__(self, queue, out_queue):

threading.Thread.__init__(self)

self.queue = queue

self.out_queue = out_queue

def run(self):

while True:

# grabs host from queue

host = self.queue.get()

# grabs urls of hosts and then grabs chunk of webpage

url = urllib2.urlopen(host)

chunk = url.read()

# place chunk into out queue

self.out_queue.put(chunk)

# signals to queue job is done

self.queue.task_done()

class DatamineThread(threading.Thread):

"""

Thread Url Grab

"""

def __init__(self, out_queue):

threading.Thread.__init__(self)

self.out_queue = out_queue

def run(self):

while True:

# grabs hosts from queue

chunk = self.out_queue.get()

# parse the chunk

soup = BeautifulSoup(chunk)

print(soup.findAll(['title']))

# signals to queue job is done

self.out_queue.task_done()

def main():

# spawn a pool of threads, and pass them queue instance

for i in range(5):

t = ThreadUrl(queue, out_queue)

t.setDaemon(True)

t.start()

# populate queue with data

for host in hosts:

queue.put(host)

for i in range(5):

dt = DatamineThread(out_queue)

dt.setDaemon(True)

dt.start()

# wait on the queue until everything has been processed

queue.join()

out_queue.join()

if __name__ == "__main__":

start = time.time()

main()

print("Elapsed Time: %s" % (time.time() - start))