python3 爬取知乎 模拟登录
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
try:
import cookielib
except:
import http.cookiejar as cookielib
import re
import time
import os.path
try:
from PIL import Image
except:
pass
from bs4 import BeautifulSoup
# 构造 Request headers
#从配置表获取
# agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'
#
# headers = {
#
# "Host": "www.zhihu.com",
#
# "Referer": "https://www.zhihu.com/",
#
# 'User-Agent': agent
#
# }
cookie_file = 'cookie.txt'
header={
'Host': 'www.zhihu.com',
'Referer': 'http://www.zhihu.com',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36',
}
# 使用登录cookie信息
session = requests.session()
session.cookies = cookielib.LWPCookieJar(filename=cookie_file)
try:
session.cookies.load(ignore_discard=True)
except:
print("Cookie 未能加载")
def get_xsrf():
'''_xsrf 是一个动态变化的参数'''
index_url = 'https://www.zhihu.com'
# 获取登录时需要用到的_xsrf
index_page = session.get(index_url, headers=header)
# html = index_page.cookies
# pattern = r'name="_xsrf" value="(.*?)"'
#
# # 这里的_xsrf 返回的是一个list
#
# _xsrf = re.findall(pattern, html)
xsrf = index_page.request._cookies.get("_xsrf")
return xsrf
# 获取验证码
def get_captcha():
t = str(int(time.time() * 1000))
captcha_url = 'https://www.zhihu.com/captcha.gif?r=' + t + "&type=login"
r = session.get(captcha_url, headers=header)
with open('captcha.jpg', 'wb') as f:
f.write(r.content)
f.close()
# 用pillow 的 Image 显示验证码
# 如果没有安装 pillow 到源代码所在的目录去找到验证码然后手动输入
try:
im = Image.open('captcha.jpg')
im.show()
im.close()
except:
print(u'请到 %s 目录找到captcha.jpg 手动输入' % os.path.abspath('captcha.jpg'))
captcha = input("please input the captcha\n>")
return captcha
def isLogin():
# 通过查看用户个人信息来判断是否已经登录
url = "https://www.zhihu.com/settings/profile"
login_code = session.get(url, headers=header, allow_redirects=False).status_code
if login_code == 200:
return True
else:
return False
def login(secret, account):
# 通过输入的用户名判断是否是手机号
if re.match(r"^1\d{10}$", account):
print("手机号登录 \n")
post_url = 'https://www.zhihu.com/login/phone_num'
postdata = {
'_xsrf': get_xsrf(),
'password': secret,
'remember_me': 'true',
'phone_num': account,
}
else:
if "@" in account:
print("邮箱登录 \n")
else:
print("你的账号输入有问题，请重新登录")
return 0
post_url = 'https://www.zhihu.com/login/email'
postdata = {
'_xsrf': get_xsrf(),
'password': secret,
'remember_me': 'true',
'email': account,
}
try:
# 不需要验证码直接登录成功
login_page = session.post(post_url, data=postdata, headers=header)
login_code = login_page.text
print(login_page.status_code)
print(login_code)
except:
# 需要输入验证码后才能登录成功
postdata["captcha"] = get_captcha()
login_page = session.post(post_url, data=postdata, headers=header)
login_code = eval(login_page.text)
print(login_code['msg'])
cookie_path=cookie_file
session.cookies.save(cookie_path)
# try:
#
# input = raw_input
#
# except:
#
# pass
##将主页面的用户提问print到shell上
def getpage(url2):
mainpage = session.get(url2, headers=header)
soup=BeautifulSoup(mainpage.text,'html.parser')
tags=soup.find_all("a",class_="question_link")
#print tags
for tag in tags:
print (tag.string)
def get_login_cookie(url):
'''
获取保存cookie
:param url:
:return:
'''
if not os.path.exists(cookie_file):
account = input('请输入你的用户名\n> ')
secret = input("请输入你的密码\n> ")
user_name = account
passwd = secret
login(passwd, user_name)
try:
cookie_jar = cookielib.LWPCookieJar(cookie_file)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
print ('Load cookie succeeded')
except cookielib.LoadError:
return None
else:
cookie_d = {}
for cookie in cookie_jar:
domain = cookie.domain
if url.find(domain) > 0:
cookie_d[cookie.name] = cookie.value
return cookie_d
if __name__ == '__main__':
if isLogin():
print('您已经登录')
url2='https://www.zhihu.com'
getpage(url2)
else:
account = input('请输入你的用户名\n> ')
secret = input("请输入你的密码\n> ")
login(secret, account) 1 import json
2 import pickle
3 import time
4
5 from selenium import webdriver
6 #代码中的chrome_options.add_argument()非常关键，一是要以无界面形式运行，二是禁用沙盒，否则程序报错。
7 #linux上使用,打开方式以无界面打开，windows不需要这些
8 chrome_options = webdriver.ChromeOptions()
9 chrome_options.add_argument('--headless') # 指定无界面形式运行
10 chrome_options.add_argument('no-sandbox') # 禁止沙盒
11
12
13 tianyan_cookie_file = 'tianyancha_cookie.txt'
14 qichacha_cookie_file = 'qichacha_cookie.txt'
15
16 #注:这些方法还需要一个插件 下面 chromedriver.exe ,也可以用火狐
17 # 同目录下(脚本运行目录)放置 chromedriver.exe 这个文件, 下载地址http://npm.taobao.org/mirrors/chromedriver/
18 # 部署的话最好部署在windows服务器下,安装最新的谷歌浏览器,linux也可以使用,但是不能使用界面操作
19
20 #天眼查账号登录
21 def login(url):
22 # dirver.maximize_window()
23 dirver = webdriver.Chrome(chrome_options=chrome_options)
24 dirver.get(url)
25 time.sleep(3)
26 # page_source = dirver.page_source
27 # print(page_source)
28 phoneNumber = dirver.find_elements_by_xpath('//*[@>)
29 phone = input("你的电话")
30 phoneNumber[0].send_keys(phone)
31
32 password = dirver.find_elements_by_xpath('//*[@>)
33 # 密码
34 passwords = input("你的密码")
35 password[0].send_keys(passwords)
36
37 time.sleep(3)
38
39 sublit = dirver.find_elements_by_xpath('//*[@>)
40 sublit[0].click()
41 time.sleep(3)
42 print(dirver.get_cookies())
43 print(type(dirver.get_cookies()))
44
45 with open(tianyan_cookie_file, 'wb') as handle:
46 pickle.dump(dirver.get_cookies(), handle)
47 time.sleep(60)
48 # dirver.quit()
49 #使用微博模拟登陆
50 def login_qichacha(url):
51 dirver = webdriver.Chrome(chrome_options=chrome_options)
52 dirver.get(url)
53 time.sleep(3)
54
55 weibologin = dirver.find_elements_by_xpath('//*[@>)
56 weibologin[0].click()
57 time.sleep(5)
58 #用户名 或者微博名
59 phone = input("你的电话")
60 dirver.find_elements_by_xpath('//*[@>)[0].send_keys(phone)
61 time.sleep(3)
62 #密码
63 password = input("你的微博密码")
64 dirver.find_elements_by_xpath('//*[@>)[0].send_keys(password)
65 time.sleep(3)
66 sublimt = dirver.find_elements_by_xpath('//*[@>)
67 sublimt[0].click()
68 time.sleep(10)
69 print(dirver.get_cookies())
70 print(type(dirver.get_cookies()))
71
72 with open(qichacha_cookie_file, 'wb') as handle:
73 pickle.dump(dirver.get_cookies(), handle)
74 dirver.close()
75 # time.sleep(60)
76 def getCookieStr():
77 #打开文件
78 with open(qichacha_cookie_file, 'rb') as handle:
79 b = pickle.load(handle)
80 print(b)
81 cookie_str = ''
82 for index in b:
83 cookie_str += index['name'] + '=' + index['value'] + ';'
84 cookie_str = cookie_str[:-1]
85 print(cookie_str)
86 return cookie_str
87
88
89
90 if __name__ == '__main__':
91 #天眼查
92 # url = 'https://www.tianyancha.com/login'
93 # login(url)
94
95 # 企查查
96 url = 'https://www.qichacha.com/user_login'
97 login_qichacha(url)
98
99 #保存cookie
100 # with open(cookie_file, 'wb') as handle:
101 # pickle.dump(data, handle)
102 # #打开文件
103 # with open(qichacha_cookie_file, 'rb') as handle:
104 # b = pickle.load(handle)
105 # print(b)
106 # cookie_str = ''
107 # for index in b:
108 # cookie_str += index['name'] + '=' + index['value'] + ';'
109 # cookie_str = cookie_str[:-1]
110 # print(cookie_str)大家注意linux环境下(centos7.0下)安装依赖参考
安装最新版本google-chrome
yum install https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm
1
测试安装
google-chrome-stable --headless --disable-gpu --screenshot http://www.baidu.com/
1
报错：
[1026/154358.043744:ERROR:zygote_host_impl_linux.cc(89)] Running as
root without --no-sandbox is not supported. See
https://crbug.com/638180.
需要指定以no-sandbox方式运行。
google-chrome-stable --no-sandbox --headless --disable-gpu --screenshot http://www.baidu.com/
1
Fontconfig warning: “/etc/fonts/fonts.conf”, line 86: unknown element “blank”
[1026/154710.301159:ERROR:gpu_process_transport_factory.cc(980)] Lost UI shared context.
[1026/154711.126035:INFO:headless_shell.cc(538)] Written to file screenshot.png.
安装chrome-driver
wget http://npm.taobao.org/mirrors/chromedriver/70.0.3538.67/chromedriver_linux64.zip
这个是解压zip 到指定文件夹
tar -zcvf chromedriver_linux64.zip /usr/local/bin/
或 (二选一)
unzip -d /usr/local/bin chromedriver_linux64.zip
1
2
python3脚本
代码中的chrome_options.add_argument()非常关键，一是要以无界面形式运行，二是禁用沙盒，否则程序报错。
# -*- coding: utf-8 -*-
import time
from selenium import webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get('http://www.baidu.com/')
time.sleep(10)
print(driver.page_source)
driver.close()
driver.quit()
--------------------- - 上一篇 »python3爬虫再探之EXCEL
- 下一篇 »python3爬虫爬取网页思路及常见问题，原创
