前面我们写了Python 爬虫 抓取豆瓣小组图片 通过api提交入库到 Chevereto 图床，但是现在有些群组需要登录才行，所以我们这里做了一个半自动脚本并使用chromedriver自动登录豆瓣并提交图片到 Chevereto 图床。
注：老季是Python新手，且本脚本大部分内容来自网络。
chormedriver下载地址
http://npm.taobao.org/mirrors/chromedriver/
Python代码实例
#!/usr/bin/python # -*- coding: UTF-8 -*- # 作者：老季 # 网址：https://www.laoji.org #导入所需的库 import logging import random import time import urllib.request import requests , json from selenium import webdriver # 实例化driver # option= webdriver.ChromeOptions() # driver = webdriver.Chrome(chrome_options=option) driver = webdriver.Chrome('D:\Downloads\chromedriver.exe') # chromedirver所在路径 # 登陆 driver.get('https://accounts.douban.com/passport/login') driver.find_element_by_class_name('account-tab-account').click() # 模拟点击 time.sleep(2) driver.find_element_by_id('username').send_keys('豆瓣用户名') # 输入用户名 driver.find_element_by_id('password').send_keys('豆瓣密码') # 输入密码 driver.find_element_by_css_selector("[class='btn btn-account btn-active']").click() # 模拟点击 time.sleep(5) ''' TODO:验证码 ''' # 获取cookie session = requests.Session() cookies = driver.get_cookies() for cookie in cookies: session.cookies.set(cookie['name'], cookie['value']) # print(cookies) time.sleep(3) logging.FileHandler(filename='Selenium_img_douban.log', encoding='utf-8') logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%Y-%m-%d %A %H:%M:%S', # filename='F:\豆瓣下载\Selenium_img_douban.log', # 日志保存路径 filemode='w') # 模拟浏览器提交图片 def submit_img(url): submit_url = 'http://tutie.org/api/1/upload/?key='+urllib.parse.quote_plus('f14fe8e8b11c08902a489706d7d99941')+'&source='+urllib.parse.quote_plus(url) # print(submit_url) req = urllib.request.Request(submit_url) try: res = urllib.request.urlopen(req) data = json.loads(res.read().decode()) return str(data['status_code']) except urllib.error.HTTPError as e: print(e.read()) def get_headers(): ''' 随机获取一个headers ''' user_agents = ['Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'] headers = {'User-Agent': random.choice(user_agents)} return headers def get(url,group_url, start_page, end_page, params=None): ''' 向一个url发送get请求，返回response对象 :param url: 网页链接 :param params: URL参数字典 :return: 发送请求后获取的response对象 ''' # 等待一个随机的时间，防止被封IP，这里随机等待0~6秒，亲测可有效地避免触发豆瓣的反爬虫机制 time.sleep(6 * random.random()) resp = session.get(url, params=params, headers=get_headers()) print(resp) logging.info(resp) if resp: logging.info( '[get] url = {0}, status_code = {1}'.format(url, resp.status_code)) resp.encoding = 'utf-8' # 这里很重要，每次发送请求后，都更新session的cookie，防止cookie过期 if resp.cookies.get_dict(): session.cookies.update(resp.cookies) logging.info('[get] updated cookies, new cookies = {0}'.format( resp.cookies.get_dict())) else: logging.info('[get] url = {0}, response is None'.format(url)) logging.info('[info] 寻找小组列表地址...') logging.info('[get] 已进入小组首页') while start_page <= end_page: driver.get(group_url + str((start_page - 1) * 25)) # 晒晒你最性感的照片小组 logging.info('已进入第' + str(start_page) + '页') print('已进入第' + str(start_page) + '页') # 每一页讨论链接列表 links = driver.find_elements_by_xpath(r"//tbody/tr/td[@class='title']/a") topic_links = {} for link in links: topic_link = link.get_attribute('href') title_name = link.get_attribute('title') # 获取的图片以小组话题名称命名 img_name = title_name.replace('?', '？').replace('\', '').replace('/', '').replace('"', '').replace(':','').replace('*','').replace('<', '').replace('>', '').replace('|', '').replace('n', '') topic_links[topic_link] = img_name topic_num = 0 for topic_link, img_name in topic_links.items(): logging.info('进入第' + str(start_page) + '页，第' + str(topic_num + 1) + '话题。') topic_num += 1 driver.get(topic_link) # 判断该话题下有无图片 img_links = driver.find_elements_by_class_name('image-wrapper') if len(img_links) > 0: i = 0 for img_link in img_links: img_name = topic_links[topic_link] + '_' + str(i) img_src = img_link.find_element_by_xpath('./img').get_attribute('src') print('[get] 话题：' + img_name + ', 图片链接：' + img_src) img = img_src.replace('.webp','.jpg') print('提交图片链接：'+img) submit_img( img ) # download_img = urllib.request.urlretrieve(img_src, 'F:\豆瓣下载\%s.jpg' % img_name) # time.sleep(3 * random.random()) i += 1 else: print('该标题下无图片') img_links2 = driver.find_elements_by_css_selector("[class='topic-figure cc']") time.sleep(3 * random.random()) start_page += 1 time.sleep(3 * random.random()) time.sleep(5) driver.quit() if __name__ == '__main__': home_url = 'https://www.douban.com/' group_url = 'https://www.douban.com/group/481977/discussion?start=' get(home_url , group_url, 0, 10000) 
评论已关闭