requests库

1.基本用法

import requests
rq=requests.get('https://www.baidu.com')
#rq=requests.post('http://httpbin.org/post')
#rq=requests.put('http://httpbin.org/put')
#rq=requests.delete('http://httpbin.org/delete')
#rq=requests.head('http://httpbin.org/get')
print(type(rq))
print (rq.status_code)
print(type(rq.text))
print(rq.text.encode('utf-8'))
print(rq.cookies)

• GET请求

基本使用

import requests
data={
'name':'Tom',
'age':'120'
}
r=requests.get('http://httpbin.org/get',params=data)#该网站会根据用户的请求返回不同结果
print(r.text)

抓取网页

import requests
import re
headers={
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'

}
r=requests.get('https://zhihu.com/explore',headers=headers)
print(r.text[:1000])
pattern=re.compile('explore-feed.*?question_link.*?>(.*?)</a>',re.S)

titles=re.findall(pattern,r.text)
print(titles)

抓取二进制数据(github图标)

import requests
r=requests.get('https://github.com/favicon.ico')
#print(r.text)
#print(r.content)
with open('favicon.ico','wb') as f:
    f.write(r.content)

• POST请求

  import requests
  
  data={
  
  'name':'Tom',
  'age':'12'
  }
  r=requests.post("http://httpbin.org/post",data=data)
  print(r.text)

• 响应

  import requests
  headers={
  'User-Agent':'Mozilla/5.0'
  }
  r=requests.get('http://www.jianshu.com',headers=headers)
  print(type(r.status_code),r.status_code)
  print(type(r.headers),r.headers)
  print(type(r.cookies),r.cookies)
  

2.高级用法

• 文件上传

  import requests
  files={'file':open('favicon.ico','rb')}
  r=requests.post("http://httpbin.org/post",files=files)
  print(r.text)

• Cookies

  使用cookies维持登录状态

  import requests
  '''r=requests.get('http://www.baidu.com')
  print(r.cookies)
  for key,value in r.cookies.items(): #将cookies转化为元组进行遍历
      print(key+" = "+value)
  '''
  headers={
      'Cookie':'_zap=bface54d-d40e-4a91-a5a0-f9e714c77426; d_c0="ABCi0WXv5Q-PTjX-MOH69gxKYOZ2aSX7Syw=|1565908724"; _xsrf=r2XRKTVaACovZGHVhjD0tT9Qy7x0gwz5; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1574087462,1574219897,1574318671,1574499180; tgw_l7_route=a37704a413efa26cf3f23813004f1a3b; capsion_ticket="2|1:0|10:1574512981|14:capsion_ticket|44:NTIxZjBhOTViMDQzNDc0YmFhZjVjMzZlZTk1YjEwM2Y=|8992301dded8f83da8c35879cce196a1ccfa61143c7f48f1edb65022768f3316"; z_c0="2|1:0|10:1574513011|4:z_c0|92:Mi4xbUVWbUNBQUFBQUFBRUtMUlplX2xEeWNBQUFDRUFsVk5jN1lBWGdDSjNHTUZ6QjVYU2ZrNTZiV1U0QXRDem1MS2d3|cbbb282d1e4255038b92361ad909dcf8e0781f4992840c6ee3aa47db3ff0d82b"; tst=r; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1574513012',
      'Host':'www.zhihu.com',
      'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
  }
  r=requests.get('https://www.zhihu.com',headers=headers)
  print(r.text)

• 会话维持

  import requests
  
  '''
  requests.get('http://httpbin.org/cookies/set/number/123456')
  r=requests.get('http://httpbin.org/cookies')
  print(r.text)
  '''
  s=requests.session()
  s.get('http://httpbin.org/cookies/set/number1/123456')
  r=s.get('http://httpbin.org/cookies')
  print(r.text)

• 代理设置

  import requests
  
  proxies={
  
      'http':'http://127.0.0.1:37279',
      'https':'http://127.0.0.1:37000'
  }
  r=requests.get("https://www.taobao.com",proxies=proxies)
  print(r.text)

• Prepared Request

  将请求表示为数据结构

  from requests import Request,Session
  url='http://httpbin.org/post'
  data={
      'name':'Tom'
  }
  headers={
  
      'User-Agent':'Mozilla/5.0'
  }
  s=Session()
  req=Request('POST',url,data=data,headers=headers)
  prepare=s.prepare_request(req)
  r=s.send(prepare)
  print(r.text)