FilesExpand file tree

 master

/

Selenium_tutorial.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

70 lines (55 loc) · 2.47 KB

 master

/

Selenium_tutorial.py

Copy path

Top

File metadata and controls

• Code
• Blame

70 lines (55 loc) · 2.47 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

from parsel import Selector

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.chrome.options import Options

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

import time

# ----------------------------------------------------------------------------------------

# ----------------------------------------------------------------------------------------

# https://scrapfly.io/blog/web-scraping-with-selenium-and-python/

# ----------------------------------------------------------------------------------------

# ----------------------------------------------------------------------------------------

# configure webdriver

options = Options()

options.headless = True # hide GUI

options.add_argument("--window-size=1920,1080") # set window size to native GUI size

options.add_argument("start-maximized") # ensure window is full-screen

# configure chrome browser to not load images and javascript

chrome_options = webdriver.ChromeOptions()

chrome_options.add_experimental_option(

"prefs", {"profile.managed_default_content_settings.images": 2}

)

driver = webdriver.Chrome(options=options, chrome_options=chrome_options)

driver.get("https://www.twitch.tv/directory/game/Art")

# wait for page to load

element = WebDriverWait(driver=driver, timeout=5).until(

EC.presence_of_element_located((By.CSS_SELECTOR, 'div[data-target=directory-first-item]'))

)

#print(driver.page_source)

sel = Selector(text=driver.page_source)

parsed = []

for item in sel.xpath("//div[contains(@class,'tw-tower')]/div[@data-target]"):

parsed.append({

'title': item.css('h3::text').get(),

'url': item.css('.tw-link::attr(href)').get(),

'username': item.css('.tw-link::text').get(),

'tags': item.css('.tw-tag ::text').getall(),

'viewers': ''.join(item.css('.tw-media-card-stat::text').re(r'(\d+)')),

})

driver.quit()

# -------------------------------------------------------------------

from selenium import webdriver

from selenium.webdriver.common.keys import Keys

driver = webdriver.Chrome()

driver.get("https://www.twitch.tv/")

search_box = driver.find_element_by_css_selector('input[aria-label="Search Input"]')

search_box.send_keys(

'fast painting'

)

# either press the enter key

search_box.send_keys(Keys.ENTER)

# or click search button

search_button = driver.find_element_by_css_selector('button[icon="NavSearch"]')

search_button.click()

driver.quit()