前言
之前寫過一篇用webdriver爬取教務系統課表的爬蟲,用的是selenium自動化的無頭瀏覽器模擬登錄,今天帶來的是用requests請求實現的爬蟲。
工具
requests庫實現對網頁的請求,execjs庫實現是js腳本的執行,bs4對爬取的數據進行清洗,csv庫實現對數據的儲存。
步驟
①首先是登錄獲取會話(session)
先在輸入框亂輸入,按F12,登錄獲取登錄頁面的表單信息格式
發現是POST請求,表單爲加密過的,此時查看頁面源代碼,檢查是如何加密的
再往下看到js,發現是根據encodeInp()進行加密的
再次檢查頁面源代碼並沒有發現此函數,那就只能是存在JS文件裏面的
雖然後來發現是Base64加密方式,但是沒關係了,用python的execjs來執行加密,記得將conwork.js下載下來放在跟python文件的同個目錄
request請求後獲取會話,準備到目的網頁進行數據爬取
def get_js(self, msg): # python 調用JS加密 返回 加密後的結果
with open('conwork.js', encoding='utf-8') as f:
js = execjs.compile(f.read())
return js.call('encodeInp', msg)
def login_requests(self):
user = '賬號'
pwd = '密碼'
csv_file_path = r'E://py//個人學期課程表.csv'
encode = str(self.get_js(user)) + "%%%" + str(self.get_js(pwd)) + "=" # 獲得加密後的東西
form_data = {
'encoded': encode
}
r_session = requests.session()
r_session.post(self.login_url, headers=self.header, data=form_data)
return r_session
②爬取數據
直接來到網頁顯示的課表頁面,將此時的URL複製下來
又發現課表有多選框,所以爲了數據準確性,需要進行post數據
周次的數據選擇全部,也就是0
學期按時間,本次選擇2019-2020-2
放大也選擇是
請求的數據如下
post_data = {
'zc': '0', #周次選擇全部
'xnxq01id': '2019-2020-2', #學期
'sfFd': '1' #放大
}
根據登錄的會話進行再次請求
post_data = {
'zc': '0', # 周次選擇全部
'xnxq01id': '2019-2020-2', # 學期
'sfFd': '1' # 放大
}
response = r_session.get(self.score_url, headers=self.header,data=post_data)
③數據清洗
使用beautifulsoup進行數據篩選,弄出符合csv的二維列表格式
soup = BeautifulSoup(response.text, 'lxml')
page = soup.find_all('div', attrs={'class': "kbcontent"})
teachers1, teachers2 = [], []
weeks1, weeks2 = [], []
classrooms1, classrooms2 = [], []
for i in page:
teachers1.append(i.find('font', attrs={'title': '老師'}))
weeks1.append(i.find('font', attrs={'title': '周次(節次)'}))
classrooms1.append(i.find('font', attrs={'title': '教室'}))
my_detail = list(page)
for i in teachers1:
if i == None:
teachers2.append('\n')
else:
teachers2.append(i.string)
for i in weeks1:
if i == None:
weeks2.append('\n')
else:
weeks2.append('\n' + i.string)
for i in classrooms1:
if i == None:
classrooms2.append('\n')
else:
classrooms2.append('\n' + i.string)
all_data = []
pitch_number = ['(上午)\n第1,2節\n(08:00-08:45)\n(08:55-09:40)', '(上午)\n第3,4節\n(10:00-10:45)\n(10:55-11:40)',
'(下午)\n第5,6節\n(14:30-15:15)\n(15:25-16:10)', '(下午)\n第7,8節\n(16:20-16:05)\n(17:15-18:00)',
'(晚上)\n第9,10節\n(19:30-20:15)\n(20:25-21:10)', '第11,12節', '第13,14節']
temp = []
temp.append(pitch_number[0])
num = 0
pnum = 0
for i in range(len(my_detail)):
if my_detail[i].text == '\xa0':
temp.append('\n\n\n')
else:
temp.append(my_detail[i].text.split(teachers2[i])[0] + '\n' + teachers2[i] + weeks2[i] + classrooms2[i])
num = num + 1
if num == 7:
all_data.append(temp)
temp = []
pnum = pnum + 1
temp.append(pitch_number[pnum])
num = 0
page2 = soup.find('td', attrs={'colspan': "7"})
BZ = ['備註:' + page2.text, '\n', '\n', '\n', '\n', '\n', '\n', '\n']
all_data.append(BZ)
④生成csv文件(可用excel打開)
f = open(self.csv_file_path, 'w', newline='')
csv_write = csv.writer(f)
csv_write.writerow(['課程時間', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日'])
for i in range(len(all_data)):
csv_write.writerow(all_data[i])
f.close()
最後附上所有代碼
工程代碼如下
import csv
import requests
import execjs
from bs4 import BeautifulSoup
class SpiderOfTimetable:
login_url = 'http://jwgln.zsc.edu.cn/jsxsd/xk/LoginToXk'
score_url = 'http://jwgln.zsc.edu.cn/jsxsd/xskb/xskb_list.do'
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/75.0.3770.100 Safari/537.36',
'Host': 'jwgln.zsc.edu.cn',
'Referer': 'http://jwgln.zsc.edu.cn/jsxsd/'
}
user = '賬號'
pwd = '密碼'
csv_file_path = r'C://個人學期課程表.csv' #生成文件的位置
def get_js(self, msg): # python 調用JS加密 返回 加密後的結果
with open('conwork.js', encoding='utf-8') as f:
js = execjs.compile(f.read())
return js.call('encodeInp', msg)
def login_requests(self):
encode = str(self.get_js(self.user)) + "%%%" + str(self.get_js(self.pwd)) + "=" # 獲得加密後的東西
form_data = {
'encoded': encode
}
r_session = requests.session()
r_session.post(self.login_url, headers=self.header, data=form_data)
return r_session
def parse_page(self,r_session):
post_data = {
'zc': '0', # 周次選擇全部
'xnxq01id': '2019-2020-2', # 學期
'sfFd': '1' # 放大
}
response = r_session.get(self.score_url, headers=self.header,data=post_data)
soup = BeautifulSoup(response.text, 'lxml')
page = soup.find_all('div', attrs={'class': "kbcontent"})
teachers1, teachers2 = [], []
weeks1, weeks2 = [], []
classrooms1, classrooms2 = [], []
for i in page:
teachers1.append(i.find('font', attrs={'title': '老師'}))
weeks1.append(i.find('font', attrs={'title': '周次(節次)'}))
classrooms1.append(i.find('font', attrs={'title': '教室'}))
my_detail = list(page)
for i in teachers1:
if i == None:
teachers2.append('\n')
else:
teachers2.append(i.string)
for i in weeks1:
if i == None:
weeks2.append('\n')
else:
weeks2.append('\n' + i.string)
for i in classrooms1:
if i == None:
classrooms2.append('\n')
else:
classrooms2.append('\n' + i.string)
all_data = []
pitch_number = ['(上午)\n第1,2節\n(08:00-08:45)\n(08:55-09:40)', '(上午)\n第3,4節\n(10:00-10:45)\n(10:55-11:40)',
'(下午)\n第5,6節\n(14:30-15:15)\n(15:25-16:10)', '(下午)\n第7,8節\n(16:20-16:05)\n(17:15-18:00)',
'(晚上)\n第9,10節\n(19:30-20:15)\n(20:25-21:10)', '第11,12節', '第13,14節']
temp = []
temp.append(pitch_number[0])
num = 0
pnum = 0
for i in range(len(my_detail)):
if my_detail[i].text == '\xa0':
temp.append('\n\n\n')
else:
temp.append(my_detail[i].text.split(teachers2[i])[0] + '\n' + teachers2[i] + weeks2[i] + classrooms2[i])
num = num + 1
if num == 7:
all_data.append(temp)
temp = []
pnum = pnum + 1
temp.append(pitch_number[pnum])
num = 0
page2 = soup.find('td', attrs={'colspan': "7"})
BZ = ['備註:' + page2.text, '\n', '\n', '\n', '\n', '\n', '\n', '\n']
all_data.append(BZ)
f = open(self.csv_file_path, 'w', newline='')
csv_write = csv.writer(f)
csv_write.writerow(['課程時間', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日'])
for i in range(len(all_data)):
csv_write.writerow(all_data[i])
f.close()
if __name__ == '__main__':
spider = SpiderOfTimetable()
session = spider.login_requests()
spider.parse_page(session)
conwork.js文件代碼
eval(function(p,a,c,k,e,d){e=function(c){return(c<a?"":e(parseInt(c/a)))+((c=c%a)>35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)d[e(c)]=k[c]||e(c);k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1;};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p;}('b 9="o+/=";p q(a){b e="";b 8,5,7="";b f,g,c,1="";b i=0;m{8=a.h(i++);5=a.h(i++);7=a.h(i++);f=8>>2;g=((8&3)<<4)|(5>>4);c=((5&s)<<2)|(7>>6);1=7&t;k(j(5)){c=1=l}v k(j(7)){1=l}e=e+9.d(f)+9.d(g)+9.d(c)+9.d(1);8=5=7="";f=g=c=1=""}u(i<a.n);r e}',32,32,'|enc4||||chr2||chr3|chr1|keyStr|input|var|enc3|charAt|output|enc1|enc2|charCodeAt||isNaN|if|64|do|length|ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789|function|encodeInp|return|15|63|while|else'.split('|'),0,{}))
工程目錄格式