安裝
pip install pyquery
驗證安裝
In [1]: import pyquery
沒報錯即表示安裝成功
pyquery
pyquery 介紹
雖然 xpath 與 Beautiful Soup 已經很強大了,不過語法還是過於囉嗦,pyquery提供了更加簡潔優雅的語法,你可以像寫jquery一般提取數據
數據初始化
In [1]: html = ''' ...: <div> ...: <ul> ...: <li class="item-0">first item</li> ...: <li class="item-1"><a href="link2.html">second item</a></li> ...: <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> ...: <li class="item-1"><a href="link4.html">fourth item</a></li> ...: <li class="item-0"><a href="link5.html">fifth item</a></li> ...: </ul> ...: </div> ...: ''' In [2]: from pyquery import PyQuery as pq In [3]: doc = pq(html) In [4]: print(doc('li')) <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li>
初始化的數據還可以從url中獲取
In [11]: doc_url = pq(url='https://www.python.org/') In [12]: print(doc_url('title')) <title>Welcome to Python.org</title>
除此,也可以傳入文件初始化
doc_file = pq(filename='demo.html')
基本css選擇器
In [2]: html = ''' ...: <div id="container"> ...: <ul class="list"> ...: <li class="item-0">first item</li> ...: <li class="item-1"><a href="link2.html">second item</a></li> ...: <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> ...: <li class="item-1"><a href="link4.html">fourth item</a></li> ...: <li class="item-0"><a href="link5.html">fifth item</a></li> ...: </ul> ...: </div> ...: ''' In [4]: from pyquery import PyQuery as pq In [5]: doc = pq(html) In [6]: print(doc('#container .list li')) <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> In [7]: print(type(doc('#container .list li'))) <class 'pyquery.pyquery.PyQuery'>
查找節點
find()
In [8]: items = doc('.list') In [10]: print(type(items)) <class 'pyquery.pyquery.PyQuery'> In [11]: print(items) <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> In [12]: lis = items.find('li') In [13]: print(type(lis)) <class 'pyquery.pyquery.PyQuery'> In [14]: print(lis) <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li>
find()的查找範圍是節點的所有後代節點
children()
查找直接子節點
In [15]: lis = items.children() In [16]: print(type(lis)) <class 'pyquery.pyquery.PyQuery'> In [17]: print(lis) <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> In [18]: items.children('.active') Out[18]: [<li.item-0.active>] In [19]: print(items.children('.active')) <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
parent()
查找父節點
In [20]: container = items.parent() In [21]: print(type(container)) <class 'pyquery.pyquery.PyQuery'> In [22]: print(container) <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div>
parents()
查找祖先節點
siblings()
兄弟節點
遍歷節點
In [26]: lis = doc('li').items() In [27]: print(type(lis)) <class 'generator'> In [28]: for li in lis: ...: print(li, type(li)) ...: <li class="item-0">first item</li> <class 'pyquery.pyquery.PyQuery'> <li class="item-1"><a href="link2.html">second item</a></li> <class 'pyquery.pyquery.PyQuery'> <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> <class 'pyquery.pyquery.PyQuery'> <li class="item-1"><a href="link4.html">fourth item</a></li> <class 'pyquery.pyquery.PyQuery'> <li class="item-0"><a href="link5.html">fifth item</a></li> <class 'pyquery.pyquery.PyQuery'>
獲取屬性
# 選中class爲item-0及active的a節點 In [29]: a = doc('.item-0.active a') In [30]: print(a, type(a)) <a href="link3.html"><span class="bold">third item</span></a> <class 'pyquery.pyquery.PyQuery'> In [31]: print(a.attr('href')) link3.html
如果是多個節點的屬性,就需要遍歷節點獲取
In [32]: a = doc('a') In [33]: for item in a.items(): ...: print(item.attr('href')) ...: link2.html link3.html link4.html link5.html
獲取文本
text()
In [35]: a = doc('.item-0.active a') In [36]: print(a) <a href="link3.html"><span class="bold">third item</span></a> In [37]: print(a.text()) third item
html()
In [38]: li = doc('.item-0.active') In [39]: print(li) <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> In [41]: print(li.html()) <a href="link3.html"><span class="bold">third item</span></a>
在多個節點的情況下,如果要獲取每個節點的內部html,需要遍歷調用html()。而text()不需要遍歷,會將所有節點取出文本後合併成一個字符串
節點操作
addClass 和 removeClass
In [42]: li = doc('.item-0.active') In [43]: print(li) <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> In [44]: li.removeClass('active') Out[44]: [<li.item-0>] In [45]: print(li) <li class="item-0"><a href="link3.html"><span class="bold">third item</span></a></li> In [46]: li.addClass('active') Out[46]: [<li.item-0.active>] In [47]: print(li) <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
attr, text 和 html
In [48]: li = doc('.item-0.active') In [49]: print(li) <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li> In [50]: li.attr('name', 'link') Out[50]: [<li.item-0.active>] In [51]: print(li) <li class="item-0 active" name="link"><a href="link3.html"><span class="bold">third item</span></a></li> In [52]: li.text('changed item') Out[52]: [<li.item-0.active>] In [53]: print(li) <li class="item-0 active" name="link">changed item</li> In [54]: li.html('<span>changed item</span>') Out[54]: [<li.item-0.active>] In [55]: print(li) <li class="item-0 active" name="link"><span>changed item</span></li>
remove
In [56]: text = ''' ...: <div class="wrap"> ...: Hello, World ...: <p>This is a paragraph</p> ...: </div> ...: ''' In [57]: doc = pq(text) In [58]: wrap = doc('.wrap') In [59]: print(wrap.text()) Hello, World This is a paragraph
如果我們不想要p節點內的text,就可以用remove將其移除
In [60]: wrap.find('p').remove() Out[60]: [<p>] In [61]: print(wrap.text()) Hello, World
僞類選擇器
In [62]: doc = pq(html) In [63]: li = doc('li:first-child') In [64]: print(li) <li class="item-0">first item</li> In [65]: li = doc('li:last-child') In [66]: print(li) <li class="item-0"><a href="link5.html">fifth item</a></li> In [67]: li = doc('li:nth-child(2)') In [68]: print(li) <li class="item-1"><a href="link2.html">second item</a></li> In [69]: li = doc('li:gt(2)') In [70]: print(li) <li class="item-1"><a href="link4.html">fourth item</a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> In [71]: li = doc('li:nth-child(2n)') In [72]: print(li) <li class="item-1"><a href="link2.html">second item</a></li> <li class="item-1"><a href="link4.html">fourth item</a></li> In [73]: li = doc('li:contains(second)') In [74]: print(li) <li class="item-1"><a href="link2.html">second item</a></li>