admin 管理员组

文章数量: 894082

斗鱼爬虫

from selenium import webdriver
import time
import json
from lxml import etree

class DouYuSpider:
def init(self):
self.url = “”
self.driver = webdriver.Chrome()

def get_content_list(self):li_list = self.driver.find_elements_by_xpath('//ul[@id="live-list-contentbox"]/li')content_list = []for li in li_list:item = {}item["room_name"] = li.find_element_by_xpath('.//div[@class="mes-tit"]/h3').textitem["tag"] = li.find_element_by_xpath('.//div[@class="mes-tit"]/span').textitem["room_img"] = li.find_element_by_xpath('.//span[@class="imgbox"]/img').get_attribute("src")item["room_url"] = li.find_element_by_xpath('./a[1]').get_attribute("href")item["user_name"] = li.find_element_by_xpath('.//div[@class="mes"]/p/span[1]').textitem["hot"] = li.find_element_by_xpath('.//div[@class="mes"]/p/span[2]').textprint(item)content_list.append(item)next_url = self.driver.find_elements_by_xpath('//a[@class="shark-pager-next"]')next_url = next_url[0] if len(next_url)>0 else Nonereturn content_list,next_urldef save_data(self,content_list):for content in content_list:with open("斗鱼房间数据.txt","a",encoding="utf-8") as f:f.write(json.dumps(content,ensure_ascii=False,indent=4))def _get_content_list(self,html_str):html = etree.HTML(html_str)li_list = html.xpath('//ul[@id="live-list-contentbox"]/li')content_list = []for li in li_list:item = {}item["room_name"] = li.xpath('.//div[@class="mes-tit"]/h3/text()')[0].replace("\n","").strip()item["tag"] = li.xpath('.//div[@class="mes-tit"]/span/text()')[0]item["room_img"] = li.xpath('.//span[@class="imgbox"]/img/@src')[0]item["room_url"] = li.xpath('./a[1]/@href')[0]item["user_name"] = li.xpath('.//div[@class="mes"]/p/span[1]/text()')[0]item["hot"] = li.xpath('.//div[@class="mes"]/p/span[2]/text()')[0]print(item)content_list.append(item)def run(self):self.driver.get(self.url)content_list,next_url = self.get_content_list()self.save_data(content_list)while next_url is not None:next_url.click()time.sleep(3)content_list,next_url = self.get_content_list()self.save_data(content_list)self.driver.quit()

if name == ‘main’:
douyu = DouYuSpider()
douyu.run()

本文标签: 斗鱼爬虫