FilesExpand file tree

 main

/

browser-example.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

49 lines (38 loc) · 1.11 KB

 main

/

browser-example.py

Copy path

Top

File metadata and controls

• Code
• Blame

49 lines (38 loc) · 1.11 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

#!/usr/bin/python3

"""

Date Updated: 2019-06-19

Author: James Campbell

What: Loads CNN.com using mechanize and saves all the links to assets/cnn.txt

"""

import mechanize

base_url = "http://www.cnn.com"

title = "cnn"

def crawl(site):

seed_url = site

br = mechanize.Browser()

br.set_handle_robots(False)

br.set_handle_equiv(False)

br.open(seed_url)

link_bank = []

for link in br.links():

if link.url[0:4] == "http":

link_bank.append(link.url)

if link.url[0] == "/":

url = link.url

if url.find(".com") == -1:

if url.find(".org") == -1:

link_bank.append(base_url + link.url)

else:

link_bank.append(link.url)

else:

link_bank.append(link.url)

if link.url[0] == "#":

link_bank.append(base_url + link.url)

link_bank = list(set(link_bank))

my_file = open(f"./assets/{title}.txt", "w")

for link in link_bank:

my_file.write(link + "\n")

my_file.close()

return link_bank

crawl(base_url)

raise SystemExit(0)