FilesExpand file tree

 master

/

Spider.java

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

80 lines (68 loc) · 1.77 KB

 master

/

Spider.java

Copy path

Top

File metadata and controls

• Code
• Blame

80 lines (68 loc) · 1.77 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

package spider;

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.net.URL;

import java.net.URLConnection;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import bean.Imooc;

/**

* 1.获取网页源码

*

*/

public class Spider {

/**

* 获取网页源代码

* @param url 网址

* @return

*/

public static String getSource(String url){

BufferedReader reader=null;

String result = "";

try {

URL realUrl = new URL(url);

// 打开和url之间的连接

URLConnection conn = realUrl.openConnection();

reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));

String line="";

while((line=reader.readLine())!=null){

result+=line;

}

// System.out.println(result);

if(reader!=null)

reader.close();

} catch (Exception e) {

e.printStackTrace();

}

return result;

}

// 测试是否获得慕课问答页面源码

// 测试是否获取问题的url

public static void main(String[] args) {

String url = "http://www.imooc.com/wenda";

String regex = "class=\"content\".+?href=\"(.+?)\".+?</a>";

String result = getSource(url);

List<String> wendaUrl = getImoocPage(result,regex);

System.out.println(wendaUrl);

}

// 没有用到

/**

* 获取问题页面url

* @param quesSource 页面源码

* @param regex 匹配正则表达式

* @return

*/

public static List<String> getImoocPage(String quesSource,String regex){

Pattern pattern = Pattern.compile(regex);

Matcher matcher = pattern.matcher(quesSource);

List<String> quesUrl = new ArrayList<String>();

while(matcher.find()){

String url = "http://www.imooc.com"+matcher.group(1);

quesUrl.add(url);

}

// System.out.println(quesUrl);

return quesUrl;

}

}