admin 管理员组文章数量: 894082
斗鱼爬虫
from selenium import webdriver
import time
import json
from lxml import etree
class DouYuSpider:
def init(self):
self.url = “”
self.driver = webdriver.Chrome()
def get_content_list(self):li_list = self.driver.find_elements_by_xpath('//ul[@id="live-list-contentbox"]/li')content_list = []for li in li_list:item = {}item["room_name"] = li.find_element_by_xpath('.//div[@class="mes-tit"]/h3').textitem["tag"] = li.find_element_by_xpath('.//div[@class="mes-tit"]/span').textitem["room_img"] = li.find_element_by_xpath('.//span[@class="imgbox"]/img').get_attribute("src")item["room_url"] = li.find_element_by_xpath('./a[1]').get_attribute("href")item["user_name"] = li.find_element_by_xpath('.//div[@class="mes"]/p/span[1]').textitem["hot"] = li.find_element_by_xpath('.//div[@class="mes"]/p/span[2]').textprint(item)content_list.append(item)next_url = self.driver.find_elements_by_xpath('//a[@class="shark-pager-next"]')next_url = next_url[0] if len(next_url)>0 else Nonereturn content_list,next_urldef save_data(self,content_list):for content in content_list:with open("斗鱼房间数据.txt","a",encoding="utf-8") as f:f.write(json.dumps(content,ensure_ascii=False,indent=4))def _get_content_list(self,html_str):html = etree.HTML(html_str)li_list = html.xpath('//ul[@id="live-list-contentbox"]/li')content_list = []for li in li_list:item = {}item["room_name"] = li.xpath('.//div[@class="mes-tit"]/h3/text()')[0].replace("\n","").strip()item["tag"] = li.xpath('.//div[@class="mes-tit"]/span/text()')[0]item["room_img"] = li.xpath('.//span[@class="imgbox"]/img/@src')[0]item["room_url"] = li.xpath('./a[1]/@href')[0]item["user_name"] = li.xpath('.//div[@class="mes"]/p/span[1]/text()')[0]item["hot"] = li.xpath('.//div[@class="mes"]/p/span[2]/text()')[0]print(item)content_list.append(item)def run(self):self.driver.get(self.url)content_list,next_url = self.get_content_list()self.save_data(content_list)while next_url is not None:next_url.click()time.sleep(3)content_list,next_url = self.get_content_list()self.save_data(content_list)self.driver.quit()
if name == ‘main’:
douyu = DouYuSpider()
douyu.run()
本文标签: 斗鱼爬虫
版权声明:本文标题:斗鱼爬虫 内容由网友自发贡献,该文观点仅代表作者本人, 转载请联系作者并注明出处:http://www.freenas.com.cn/jishu/1688190000h190004.html, 本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌抄袭侵权/违法违规的内容,一经查实,本站将立刻删除。
发表评论