urllib

py

发送请求

import urllib2
response = urllib.urlopen(url[
    , data     #访问URL时要传入的数据，要urlencode处理
    , timeout])#延迟
html     = response.read()

构造请求

import urllib2
headers  = {'User-agent' : user_agent}            #修改请求信息
request  = urllib2.Request(url, headers = headers)#发送请求
response = urllib2.urlopen(request)               #获得回应
html     = response.read()

代理设置

proxy_handler = urllib2.ProxyHandler({"http" : 'http://some-proxy.com:8080'})
opener        = urllib2.build_opener(proxy_handler)
None          = urllib2.install_opener(opener)

异常处理

1. URLError
   
   • 注意:无错误代码
   • 无法上网
   • 连接不到服务器
   • 服务器不存在
2. HTTPError
   
   • 注意：存在错误代码
   • 各种HTTP状态码

try:
    response = urllib2.urlopen(url)
except: urllib2.URLError, urllib2.HTTPError as e:
    e.reason #错误信息
    e.code #错误代码

使用Cookie

CookieJar -> FileCookJar -> MozillaCookieJar & LWPCookieJar

import urllib2
import cookielib
cookie   = cookielib.CookieJar()              #声明实例保存cookie
handler  = urllib2.HTTPCookieProcessor(cookie)#创建cookie处理器
opener   = urllib2.build_opener(handler)      #类似urlopen
response = opener.open(url)
#此时cookie保存有从url中存在的cookie

保存Cookie到文件

filename = 'cookie.txt'
cookie   = cookielib.MozillaCookieJar(filename)
handler  = urllib2.HTTPCookieProcessor(cookie)
opener   = urllib2.build_opener(handler)
response = opener.open(url)
cookie.save(ignore_discard = True, #即使Cookie被丢弃也保存
    ignore_expires         = True) #覆盖原文件写入

从文件中获取Cookie

import http.cookiejar as cookielib
cookie = cookielib.MozillaCookieJar()
cookie.load('cookie.txt', ignore_discard=True, ignore_expires=True)
request = urllib2.Request(url)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
response = opener.open(request)

其他

import urlparse
urlparse.urljoin(seed_url, link) #将相对路径转化为绝对路径