1、BeautifulSoup 解析html如何使用
'''
想要学习Python？Python学习交流群：973783996满足你的需求，资料都已经上传群文件，可以自行下载！
'''
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import re
#待分析字符串
html_doc = """
<html>
<head>
<title>The Dormouse's story</title>
</head>
<body>
<p class="title aq">
<b>
The Dormouse's story
</b>
</p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.
</p>
<p class="story">...</p>
"""
# html字符串创建BeautifulSoup对象
soup = BeautifulSoup(html_doc, 'html.parser', from_encoding='utf-8')
#输出第一个 title 标签
print soup.title
#输出第一个 title 标签的标签名称
print soup.title.name
#输出第一个 title 标签的包含内容
print soup.title.string
#输出第一个 title 标签的父标签的标签名称
print soup.title.parent.name
#输出第一个 p 标签
print soup.p
#输出第一个 p 标签的 class 属性内容
print soup.p['class']
#输出第一个 a 标签的 href 属性内容
print soup.a['href']
'''
soup的属性可以被添加,删除或修改. 再说一次, soup的属性操作方法与字典一样
'''
#修改第一个 a 标签的href属性为 http://www.baidu.com/
soup.a['href'] = 'http://www.baidu.com/'
#给第一个 a 标签添加 name 属性
soup.a['name'] = u'百度'
#删除第一个 a 标签的 class 属性为
del soup.a['class']
##输出第一个 p 标签的所有子节点
print soup.p.contents
#输出第一个 a 标签
print soup.a
#输出所有的 a 标签，以列表形式显示
print soup.find_all('a')
#输出第一个 id 属性等于 link3 的 a 标签
print soup.find(id="link3")
#获取所有文字内容
print(soup.get_text())
#输出第一个 a 标签的所有属性信息
print soup.a.attrs
for link in soup.find_all('a'):
#获取 link 的 href 属性内容
print(link.get('href'))
#对soup.p的子节点进行循环输出
for child in soup.p.children:
print(child)
#正则匹配，名字中带有b的标签
for tag in soup.find_all(re.compile("b")):
print(tag.name)
2、cookie等使用方法以及函数爬虫
参照： https://cuiqingcai.com/968.html
3、header,代理,超时,认证,异常处理
参照： http://blog.csdn.net/m_buddy/article/details/55193762
4、错误异常处理
1.URLError
# -*- coding: UTF-8 -*-
import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
if __name__ == "__main__":
#一个不存在的连接
url = "http://www.douyu.com/Jack_Cui.html"
request = urllib.request.Request(url)
try:
response = urllib.request.urlopen(request)
# html = responese.read()
except urllib.error.HTTPError as e:
print(e.code)
运行结果：
C:\Python34\python.exe G:/xiaoshuo2.py
Process finished with exit code 0
# -*- coding: UTF-8 -*-
import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
if __name__ == "__main__":
#一个不存在的连接
url = "http://www.douyu.com/Jack_Cui.html"
request = urllib.request.Request(url)
try:
response = urllib.request.urlopen(request)
html = response.read().decode('utf-8')
print(html)
except urllib.error.HTTPError as e:
print(e.code)
运行结果：
C:\Python34\python.exe G:/xiaoshuo2.py
Process finished with exit code 0
import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
url = "http://www.douyu.com/Jack_Cui.html"
rep=urllib.request.Request(url)
try:
data=urllib.request.urlopen(rep)
except urllib.error.URLError as e:
if hasattr(e,'code'):
print("HTTPError")
print(e.code)
if hasattr(e,'reason' ):
print("URLError")
print(e.reason)
输出结果：
C:\Python34\python.exe G:/xiaoshuo2.py
HTTPError
URLError
Forbidden
Process finished with exit code 0
5、python打印防止换行和换行
https://www.cnblogs.com/kfx2007/p/5970784.html
实例：
# coding=utf-8
import re
language = '''''
<table class="infobox bordered vcard" style="width: 21em; font-size: 89%; text-align: left;" cellpadding="3">
<caption style="text-align: center; font-size: larger;" class="fn"><b>jenkins</b></caption>
<tr>
<th>性別：</th>
<td>男</td>d
</tr>
<tr>
<th>異名：</th>
<td><span class="nickname">(字) 翔宇</span></td>
</tr>
<tr>
<th>爱好：</th>
<td><span class="org"><a href="../articles/%E4%B8%AD9A.html" title="篮球">篮球</a></span></td>
</tr>
<tr>
<th>籍貫：</th>
<td><a href="../articles/%E6%B5%9981.html" title="广西省">广西省</a><a href="../articles/%E7%BB%8D82.html" title="桂林市">桂林市</a></td>
</tr>
</table>
'''
#获取table中tr值
res_tr = r'<tr>(.*?)</tr>'
m_tr = re.findall(res_tr,language,re.S|re.M)
for line in m_tr:
#获取表格第一列th 属性
res_th = r'<th>(.*?)</th>'
m_th = re.findall(res_th,line,re.S|re.M)
for mm in m_th:
if "href" in mm: #如果获取加粗的th中含超链接则处理
restr = r'<a href=.*?>(.*?)</a>'
h = re.findall(restr,mm,re.S|re.M)
print (h[0],end=' ') #逗号连接属性值 防止换行
else:
print (mm,end=' ') #unicode防止乱
#获取表格第二列td 属性值
res_td = r'<td>(.*?)</td>' #r'<td .*?>(.*?)</td>'
m_td = re.findall(res_td,line,re.S|re.M)
for nn in m_td:
if "href" in nn: #处理超链接<a href=../rel=..></a>
res_value = r'<a .*?>(.*?)</a>'
m_value = re.findall(res_value,nn,re.S|re.M)
for value in m_value:
print (value,end=' ')
elif "span" in nn: #处理标签<span>
res_value = r'<span .*?>(.*?)</span>'
m_value = re.findall(res_value,nn,re.S|re.M) #<td><span class="nickname">(字) 翔宇</span></td>
for value in m_value:
print (value,end=' ')
else:
print (nn,end=' ')
print (' ') #换行
C:\Python34\python.exe G:/xiaoshuo2.py
性別： 男
異名： (字) 翔宇
爱好： 篮球
籍貫： 广西省 桂林市
6、python打印如何呢不换行
https://www.cnblogs.com/hwd9654/p/5707920.html
# -*- coding:utf-8 -*-
import urllib
import re
#import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
class Tool:
removeImg = re.compile('<img.*?>| {7}|')
removeAddr = re.compile('<a.*?>|</a>')
replaceLine = re.compile('<tr>|<div>|</div>|</p>')
replaceTD= re.compile('<td>')
replacePara = re.compile('<p.*?>')
replaceBR = re.compile('<br><br>|<br>')
removeExtraTag = re.compile('<.*?>')
def replace(self,x):
x = re.sub(self.removeImg,"",x)
x = re.sub(self.removeAddr,"",x)
x = re.sub(self.replaceLine,"\n",x)
x = re.sub(self.replaceTD,"\t",x)
x = re.sub(self.replacePara,"\n",x)
x = re.sub(self.replaceBR,"\n",x)
x = re.sub(self.removeExtraTag,"",x)
return x.strip()
class BDTB:
def __init__(self,baseUrl,seeLZ):
self.baseURL = baseUrl
self.seeLZ = '?see_lz='+str(seeLZ)
self.tool = Tool()
def getPage(self,pageNum):
try:
url = self.baseURL+ self.seeLZ + '&pn=' + str(pageNum)
request = urllib.request.Request(url)
response = urllib.request.urlopen(request).read().decode("utf8")
#print (response)
return response
except urllib.error.URLError as e:
if hasattr(e,"reason"):
print ("连接百度贴吧失败,错误原因",e.reason)
return None
def getTitle(self):
page = self.getPage(1)
pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>',re.S)
result = re.search(pattern,page)
if result:
#print (result.group(1))
return result.group(1).strip()
else:
return None
def getPageNum(self):
page = self.getPage(1)
pattern = re.compile('<li class="l_reply_num.*?</span>.*?<span.*?>(.*?)</span>',re.S)
result = re.search(pattern,page)
#print (result.group(1))
if result:
return result.group(1).strip()
else:
return None
def getContent(self,page):
pattern = re.compile('<div id="post_content_.*?>(.*?)</div>',re.S)
items = re.findall(pattern,page)
floor = 1
for item in items:
print (floor,"楼-------------------------------------------------------------------------------------\n",end='')
#print ("楼---------------------------------------------------------------------------------------------------------------\n")
print (self.tool.replace(item))
floor += 1
baseURLh = 'http://tieba.baidu.com/p/3138733512'
bdtb = BDTB(baseURLh,1)
bdtb.getContent(bdtb.getPage(1))
打印结果：
C:\Python34\python.exe C:/Users/Administrator/ceshi.py
楼-------------------------------------------------------------------------------------
很多媒体都在每赛季之前给球员排个名，我也有这个癖好…………，我会尽量理性的分析球队地位，个人能力等因素，评出我心目中的下赛季50大现役球员，这个50大是指预估他本赛季在篮球场上对球队的影响力……不是过去的荣誉什么的，所以难免有一定的主观性……如果把你喜欢的球星排低了，欢迎理性讨论！
状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
今年的新秀我就不考虑了，没上赛季参照
楼-------------------------------------------------------------------------------------
惊喜新人王 迈卡威
上赛季数据
篮板 6.2 助攻 6.3 抢断 1.9 盖帽 0.6 失误 3.5 犯规 3 得分 16.7
新赛季第50位，我给上赛季的新人王迈卡威。 上赛季迈卡威在彻底重建的76人中迅速掌握了球队，一开始就三双搞定了热火赢得了万千眼球。后来也屡屡有经验的表现，新秀赛季就拿过三双的球员不多，迈卡威现在可以说在76人站稳了脚跟。
7、python爬虫xpath的语法
http://www.cnblogs.com/lonenysky/p/4649455.html
//*[@id="AD_4586850"]/div[1]/strong/i
//*[@id="shop_list"]/div[1]/strong/i
//*[@id="shop_list"]
8、requests用法
http://cuiqingcai.com/2556.html
#-*—coding:utf8-*-
from lxml import etree
import requests
import re
#编码转换
import sys
#headers构造一个字典，里面保存了user-agent
#headers= { 'User-Agent' : 'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' }
html = requests.get('http://cuiqingcai.com')
print(html.text)
9、sub使用
http://blog.csdn.net/lovemianmian/article/details/8867613
1、去除imag标签
import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('<img.*?>')
s=re.sub(removeImg,"",text).strip()
print (s)
C:\Python34\python.exe G:/xiaoshuo2.py
he is cool, clever, and so on...
1、1 单独去除7位长空格
import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('| {7}|')
s=re.sub(removeImg,"",text).strip()
print (s)
打印
C:\Python34\python.exe G:/xiaoshuo2.py
<imgJGood is a handsome boy,> he is cool, clever, and so on...
2、去除imag标签 + 去除7位长空格
import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('<img.*?>| {7}|')
s=re.sub(removeImg,"",text).strip()
print (s)
打印：
C:\Python34\python.exe G:/xiaoshuo2.py
he is cool, clever, and so on...
Process finished with exit code 0
3、去除imag标签 + 保留7位长空格
import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('<img.*?>{7}')
s=re.sub(removeImg,"",text).strip()
print (s)
打印：
C:\Python34\python.exe G:/xiaoshuo2.py
<imgJGood is a handsome boy,> he is cool, clever, and so on...
Process finished with exit code 0
4、把两个标签中间的内容去掉
import re
text='<a href="http://jump2.bdimg.com/safecheck/index?url=x+Z5)">迈卡威</a>刷出了不错的数据'
removeImg = re.compile('<a.*?>|</a>')
s=re.sub(removeImg,"",text).strip()
print (s)
打印：
C:\Python34\python.exe G:/xiaoshuo2.py
迈卡威刷出了不错的数据
5，把<br>换行符换成/n 换行符
import re
text ='height="510"><br><br><br><br>状元维金斯镇楼<br>P.S 1 我每天都至少更新一个，不TJ。<br> 2 今年的新秀我就不考虑了，没上赛季参照'
removeImg = re.compile('<br><br>|<br>')
s=re.sub(removeImg,"\n",text).strip()
print (s)
C:\Python34\python.exe G:/xiaoshuo2.py
height="510">
状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
今年的新秀我就不考虑了，没上赛季参照
5.1，把<br>换行符换成/n 换行符
import re
text ='height="510"><br><br><br><br>状元维金斯镇楼<br>P.S 1 我每天都至少更新一个，不TJ。<br> 2 今年的新秀我就不考虑了，没上赛季参照'
removeImg = re.compile('<br>')
s=re.sub(removeImg,"\n",text).strip()
print (s)
C:\Python34\python.exe G:/xiaoshuo2.py
height="510">
状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
今年的新秀我就不考虑了，没上赛季参照
10、正则表达式
<div class="list-item">
<div class="personal-info">
<div class="pic-word">
<div class="pic s60">
<a href="//mm.taobao.com/687471686.htm" target="_blank" class="lady-avatar"> <img src="//gtd.alicdn.com/sns_logo/i2/TB1XZ1PQVXXXXaJXpXXSutbFXXX.jpg_60x60.jpg" alt="" width="60" height="60"/>
</a>
</div>
<p class="top">
<a class="lady-name" href="//mm.taobao.com/self/model_card.htm?user_id=687471686" target="_blank">jenkins</a>
<em><strong>27</strong>岁</em>
<span>广州市</span>
pattern = re.compile('<div class="list-item">.*? href="(.*?)".*? src="(.*?)".*? target="_blank">(.*?)</a>.*?<strong>(.*?)</strong>.*?<span>(.*?)</span>',re.S)
或者
pattern = re.compile('<div class="list-item">.*?<a href="(.*?)".*?<img src="(.*?)".*?<a class="lady-name".*?>(.*?)</a>.*?<strong>(.*?)</strong>.*?<span>(.*?)</span>',re.S)
https://image.baidu.com/search/index?tn=baiduimage&ct=201326592&lm=-1&cl=2&ie=gbk&word=%C3%C0%C5%AE%CD%BC%C6%AC&fr=ala&ala=1&alatpl=cover&pos=0&hs=2&xthttps=111111