最近剛學習Python,做了個簡單的爬蟲,作爲一個簡單的demo希望幫助和我一樣的初學者
代碼使用python2.7做的爬蟲 抓取51job上面的職位名,公司名,薪資,發佈時間等等
直接上代碼,代碼中註釋還算比較清楚 ,沒有安裝mysql需要屏蔽掉相關代碼:
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import urllib
import urllib2
import codecs
import re
import time
import logging
import MySQLdb
class Jobs(object):
# 初始化
"""docstring for Jobs"""
def __init__(self):
super(Jobs, self).__init__()
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
#數據庫的操作,沒有mysql可以做屏蔽
self.db = MySQLdb.connect('127.0.0.1','root','rootroot','MySQL_Test',charset='utf8')
self.cursor = self.db.cursor()
#log日誌的顯示
self.logger = logging.getLogger("sjk")
self.logger.setLevel(level=logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.FileHandler('log.txt')
handler.setFormatter(formatter)
handler.setLevel(logging.DEBUG)
self.logger.addHandler(handler)
self.logger.info('初始化完成')
# 模擬請求數據
def jobshtml(self, key, page='1'):
try:
self.logger.info('開始請求第' + page + '頁')
#網頁url
searchurl = "https://search.51job.com/list/040000,000000,0000,00,9,99,{key},2,{page}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:59.0) Gecko/20100101 Firefox/59.0'
#設置請求頭
header = {'User-Agent': user_agent, 'Host': 'search.51job.com',
'Referer': 'https://www.51job.com/'}
#拼接url
finalUrl = searchurl.format(key=key, page=page)
request = urllib2.Request(finalUrl, headers=header)
response = urllib2.urlopen(request)
#等待網頁加載完成
time.sleep(3)
#gbk格式解碼
info = response.read().decode('gbk')
self.logger.info('請求網頁網頁')
self.decodeHtml(info=info, key=key, page=page)
except urllib2.HTTPError as e:
print e.reason
# 解析網頁數據
def decodeHtml(self, info, key, page):
self.logger.info('開始解析網頁數據')
#BeautifulSoup 解析網頁
soup = BeautifulSoup(info, 'html.parser')
#找到class = t1 t2 t3 t4 t5 的標籤數據
ps = soup.find_all(attrs={"class": re.compile(r'^t[1-5].*')})
#打開txt文件 a+ 代表追加
f = codecs.open(key + '.txt', 'a+', 'UTF-8')
#清除之前的數據信息
f.truncate()
f.write('\n------------' + page + '--------------\n')
count = 1
arr = []
#做一些字符串的處理,形成數據格式 iOS開發工程師 有限公司 深圳-南山區 0.9-1.6萬/月 05-16
for pi in ps:
spe = " "
finalstr = pi.getText().strip()
arr.append(finalstr)
if count % 5 == 0:
#每一條數據插入數據庫,如果沒有安裝mysql 可以將當前行註釋掉
self.connectMySQL(arr=arr)
arr = []
spe = "\n"
writestr = finalstr + spe
count += 1
f.write(writestr)
f.close()
self.logger.info('解析完成')
#數據庫操作 沒有安裝mysql 可以屏蔽掉
def connectMySQL(self,arr):
work=arr[0]
company=arr[1]
place=arr[2]
salary=arr[3]
time=arr[4]
query = "select * from Jobs_tab where \
company_name='%s' and work_name='%s' and work_place='%s' \
and salary='%s' and time='%s'" %(company,work,place,salary,time)
self.cursor.execute(query)
queryresult = self.cursor.fetchall()
#數據庫中不存在就插入數據 存在就可以更新數據 不過我這邊沒有寫
if len(queryresult) > 0:
sql = "insert into Jobs_tab(work_name,company_name,work_place,salary\
,time) values('%s','%s','%s','%s','%s')" %(work,company,place,salary,time)
try:
self.cursor.execute(sql)
self.db.commit()
except Exception as e:
self.logger.info('寫入數據庫失敗')
#模擬登陸
# def login(self):
# data = {'action':'save','isread':'on','loginname':'18086514327','password':'kui4131sjk'}
# 開始抓取 主函數
def run(self, key):
# 只要前5頁的數據 key代表搜索工做類型 這邊我是用的ios page是頁數
for x in xrange(1, 6):
self.jobshtml(key=key, page=str(x))
self.logger.info('寫入數據庫完成')
self.db.close()
if __name__ == '__main__':
Jobs().run(key='iOS')
這樣抓取網頁數據格式如下:
有什麼寫得不好的地方,歡迎指點