基金爬蟲

use invest ;
DROP TABLE IF EXISTS fund_info ;
CREATE TABLE IF NOT EXISTS `fund_info` (
`fund_code` varchar(255) NOT NULL COMMENT '基金代碼',
`fund_name` varchar(255) DEFAULT NULL COMMENT '基金全稱',
`fund_abbr_name` varchar(255) DEFAULT NULL COMMENT '基金簡稱',
`fund_type` varchar(255) DEFAULT NULL COMMENT '基金類型',
`issue_date` varchar(255) DEFAULT NULL COMMENT '發行日期',
`establish_date` varchar(255) DEFAULT NULL COMMENT '成立日期',
`establish_scale` varchar(255) DEFAULT NULL COMMENT '成立日期規模',
`asset_value` varchar(255) DEFAULT NULL COMMENT '最新資產規模',
`asset_value_date` varchar(255) DEFAULT NULL COMMENT '最新資產規模日期',
`units` varchar(255) DEFAULT NULL COMMENT '最新份額規模',
`units_date` varchar(255) DEFAULT NULL COMMENT '最新份額規模',
`fund_manager` varchar(255) DEFAULT NULL COMMENT '基金管理人',
`fund_trustee` varchar(255) DEFAULT NULL COMMENT '基金託管人',
`funder` varchar(255) DEFAULT NULL COMMENT '基金經理人',
`total_div` varchar(255) DEFAULT NULL COMMENT '成立來分紅',
`mgt_fee` varchar(255) DEFAULT NULL COMMENT '管理費率',
`trust_fee` varchar(255) DEFAULT NULL COMMENT '託管費率',
`sale_fee` varchar(255) DEFAULT NULL COMMENT '銷售服務費率',
`buy_fee` varchar(255) DEFAULT NULL COMMENT '最高認購費率',
`buy_fee2` varchar(255) DEFAULT NULL COMMENT '最高申購費率',
`benchmark` varchar(1000) DEFAULT NULL COMMENT '業績比較基準',
`underlying` varchar(500) DEFAULT NULL COMMENT '跟蹤標的',
`data_source` varchar(255) DEFAULT 'eastmoney' COMMENT '數據來源',
`created_date` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '創建時間',
`updated_date` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新時間',
`created_by` varchar(255) DEFAULT 'eastmoney' COMMENT '創建人',
`updated_by` varchar(255) DEFAULT 'eastmoney' COMMENT '更新人',
PRIMARY KEY (`fund_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='基金基本信息表';

DROP TABLE IF EXISTS fund_nav ;
CREATE TABLE IF NOT EXISTS `fund_nav` (
`the_date` varchar(255) NOT NULL,
`nav` float(15,8) DEFAULT NULL,
`add_nav` float(15,8) DEFAULT NULL,
`nav_chg_rate` varchar(255) DEFAULT NULL,
`buy_state` varchar(255) DEFAULT NULL,
`sell_state` varchar(255) DEFAULT NULL,
`div_record` varchar(255) DEFAULT NULL,
`fund_code` varchar(255) NOT NULL,
`created_date` datetime DEFAULT NULL,
`updated_date` datetime DEFAULT NULL,
PRIMARY KEY (`the_date`,`fund_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

DROP TABLE IF EXISTS fund_nav_currency ;
CREATE TABLE IF NOT EXISTS `fund_nav_currency` (
`the_date` varchar(255) NOT NULL,
`fund_code` varchar(255) NOT NULL,
`profit_per_units` float(15,8) DEFAULT NULL,
`profit_rate` varchar(255) DEFAULT NULL,
`buy_state` varchar(255) DEFAULT NULL,
`sell_state` varchar(255) DEFAULT NULL,
`div` varchar(255) DEFAULT NULL,

`created_date` datetime DEFAULT NULL,
`updated_date` datetime DEFAULT NULL,
PRIMARY KEY (`the_date`,`fund_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

select count(*) from invest.fund_info;
select count(*) from invest.fund_nav;
select count(*) from invest.fund_nav_currency;

drop table invest.`fund_managers_chg`;
CREATE TABLE IF NOT EXISTS invest.`fund_managers_chg` (
`fund_code` varchar(255) NOT NULL,
`start_date` varchar(255) NOT NULL,
`end_date` varchar(255) NOT NULL,
`fund_managers` varchar(255) NOT NULL,
`term` varchar(255) NOT NULL,
`return_rate` varchar(255) DEFAULT NULL,
`created_date` datetime DEFAULT NULL,
`updated_date` datetime DEFAULT NULL,
`data_source` varchar(255) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE IF NOT EXISTS invest.`fund_managers_info` (
`manager_id` varchar(255) NOT NULL,
`url` varchar(255) NOT NULL,
`manager_name` varchar(255) NOT NULL,
`created_date` datetime DEFAULT NULL,
`updated_date` datetime DEFAULT NULL,
`data_source` varchar(255) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

commit;

delete from invest.fund_info;
delete from invest.fund_managers_chg;
delete from invest.fund_managers_info;
delete from invest.fund_nav;
delete from invest.fund_nav_currency;

程序：

# -*- coding:utf-8 -*-
############################################################################
'''
# 程序：東方財富網基金數據爬取
# 功能：抓取東方財富網上基金相關數據
# 創建時間：2017/02/14 基金概況數據
# 更新歷史：2017/02/15 增加基金淨值數據
#
# 使用庫：requests、BeautifulSoup4、pymysql,pandas
# 作者：yuzhucu
'''
#############################################################################
import requests
from bs4 import BeautifulSoup
import time
import random
import pymysql
import os
import pandas as pd
import re

def randHeader():
'''
隨機生成User-Agent
:return:
'''
head_connection = ['Keep-Alive', 'close']
head_accept = ['text/html, application/xhtml+xml, */*']
head_accept_language = ['zh-CN,fr-FR;q=0.5', 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3']
head_user_agent = ['Opera/8.0 (Macintosh; PPC Mac OS X; U; en)',
'Opera/9.27 (Windows NT 5.2; U; zh-cn)',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0)',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E; QQBrowser/7.3.9825.400)',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; BIDUBrowser 2.x)',
'Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3',
'Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070803 Firefox/1.5.0.12',
'Mozilla/5.0 (Windows; U; Windows NT 5.2) Gecko/2008070208 Firefox/3.0.1',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; rv:11.0) like Gecko)',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0 ',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Maxthon/4.0.6.2000 Chrome/26.0.1410.43 Safari/537.1 ',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.92 Safari/537.1 LBBROWSER',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.0 Safari/536.11',
'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0'
]
result = {
'Connection': head_connection[0],
'Accept': head_accept[0],
'Accept-Language': head_accept_language[1],
'User-Agent': head_user_agent[random.randrange(0, len(head_user_agent))]
}
return result

def getCurrentTime():
# 獲取當前時間
return time.strftime('[%Y-%m-%d %H:%M:%S]', time.localtime(time.time()))

def getURL(url, tries_num=5, sleep_time=0, time_out=10,max_retry = 5):
'''
這裏重寫get函數，主要是爲了實現網絡中斷後自動重連，同時爲了兼容各種網站不同的反爬策略及，通過sleep時間和timeout動態調整來測試合適的網絡連接參數；
通過isproxy 來控制是否使用代理，以支持一些在內網辦公的同學
:param url:
:param tries_num: 重試次數
:param sleep_time: 休眠時間
:param time_out: 連接超時參數
:param max_retry: 最大重試次數，僅僅是爲了遞歸使用
:return: response
'''
sleep_time_p = sleep_time
time_out_p = time_out
tries_num_p = tries_num
try:
res = requests.Session()
if isproxy == 1:
res = requests.get(url, headers=header, timeout=time_out, proxies=proxy)
else:
res = requests.get(url, headers=header, timeout=time_out)
res.raise_for_status() # 如果響應狀態碼不是 200，就主動拋出異常
except requests.RequestException as e:
sleep_time_p = sleep_time_p + 10
time_out_p = time_out_p + 10
tries_num_p = tries_num_p - 1
# 設置重試次數，最大timeout 時間和最長休眠時間
if tries_num_p > 0:
time.sleep(sleep_time_p)
print (getCurrentTime(), url, 'URL Connection Error: 第', max_retry - tries_num_p, u'次 Retry Connection', e)
return getURL(url, tries_num_p, sleep_time_p, time_out_p,max_retry)
return res

class PyMySQL:
# 獲取當前時間
def getCurrentTime(self):
return time.strftime('[%Y-%m-%d %H:%M:%S]', time.localtime(time.time()))
# 數據庫初始化
def _init_(self, host, user, passwd, db,port=3306,charset='utf8'):
pymysql.install_as_MySQLdb()
try:
self.db =pymysql.connect(host=host,user=user,passwd=passwd,db=db,port=3306,charset='utf8')
#self.db = pymysql.connect(ip, username, pwd, schema,port)
self.db.ping(True)#使用mysql ping來檢查連接,實現超時自動重新連接
print (self.getCurrentTime(), u"MySQL DB Connect Success:",user+'@'+host+':'+str(port)+'/'+db)
self.cur = self.db.cursor()
except Exception as e:
print (self.getCurrentTime(), u"MySQL DB Connect Error :%d: %s" % (e.args[0], e.args[1]))
# 插入數據
def insertData(self, table, my_dict):
try:
#self.db.set_character_set('utf8')
cols = ', '.join(my_dict.keys())
values = '"," '.join(my_dict.values())
sql = "replace into %s (%s) values (%s)" % (table, cols, '"' + values + '"')
#print (sql)
try:
result = self.cur.execute(sql)
insert_id = self.db.insert_id()
self.db.commit()
# 判斷是否執行成功
if result:
#print (self.getCurrentTime(), u"Data Insert Sucess")
return insert_id
else:
return 0
except Exception as e:
# 發生錯誤時回滾
self.db.rollback()
print (self.getCurrentTime(), u"Data Insert Failed: %s" % (e))
return 0
except Exception as e:
print (self.getCurrentTime(), u"MySQLdb Error: %s" % (e))
return 0

class FundSpiders():

def getCurrentTime(self):
# 獲取當前時間
return time.strftime('[%Y-%m-%d %H:%M:%S]', time.localtime(time.time()))

def getFundCodesFromCsv(self):
'''
從csv文件中獲取基金代碼清單（可從wind或者其他財經網站導出）
'''
file_path=os.path.join(os.getcwd(),'fund.csv')
fund_code = pd.read_csv(filepath_or_buffer=file_path, encoding='gbk')
Code=fund_code.trade_code
print(Code)
return Code

def getFundInfo(self,fund_code):
'''
獲取基金概況基本信息
:param fund_code:
:return:
'''
fund_url='http://fund.eastmoney.com/f10/jbgk_'+fund_code +'.html'
print(fund_url)
res = getURL(fund_url)
soup = BeautifulSoup(res.text, 'html.parser')
result = {}
try:
result['fund_code']=fund_code
'''
之前用select、find 比較多，但是一些網頁中經常出現部分字段不全導致內容和數據庫不匹配的情況導致數據錯位。這裏改爲用使用標題的next_element 來獲取數據值來規避此問題
其中也有個別字段有問題的，特殊處理下即可
'''
result['fund_name']= soup.find_all(text=u"基金全稱")[0].next_element.text.strip()
result['fund_abbr_name']= soup.find_all(text=u"基金簡稱")[0].next_element.text.strip()
#result['fund_code']= soup.find_all(text=u"基金代碼")[0].next_element )
result['fund_type']= soup.find_all(text=u"基金類型")[0].next_element.text.strip()
result['issue_date']= soup.find_all(text=u"發行日期")[0].next_element.text.strip()
result['establish_date']= soup.find_all(text=u"成立日期/規模")[0].next_element.text.split(u'/')[0].strip()
result['establish_scale']= soup.find_all(text=u"成立日期/規模")[0].next_element.text.split(u'/')[-1].strip()
result['asset_value']= soup.find_all(text=u"資產規模")[0].next_element.text.split(u'（')[0].strip()
result['asset_value_date']= soup.find_all(text=u"資產規模")[0].next_element.text.split(u'（')[1].split(u'）')[0].strip(u'截止至：')
result['units']= soup.find_all(text=u"份額規模")[0].next_element.text.strip().split(u'（')[0].strip()
result['units_date']= soup.find_all(text=u"份額規模")[0].next_element.text.strip().split(u'（')[1].strip(u'（截止至：）')
result['fund_manager']= soup.find_all(text=u"基金管理人")[0].next_element.text.strip()
result['fund_trustee']= soup.find_all(text=u"基金託管人")[0].next_element.text.strip()
result['funder']= soup.find_all(text=u"基金經理人")[0].next_element.text.strip()
result['total_div']= soup.find_all(text=u"成立來分紅")[0].next_element.text.strip()
result['mgt_fee']= soup.find_all(text=u"管理費率")[0].next_element.text.strip()
result['trust_fee']= soup.find_all(text=u"託管費率")[0].next_element.text.strip()
result['sale_fee']= soup.find_all(text=u"銷售服務費率")[0].next_element.text.strip()
result['buy_fee']= soup.find_all(text=u"最高認購費率")[0].next_element.text.strip()
result['buy_fee2']= soup.find_all(text=u"最高申購費率")[0].next_element.text.strip()
result['benchmark']= soup.find_all(text=u"業績比較基準")[0].next_element.text.strip(u'該基金暫未披露業績比較基準')
result['underlying']= soup.find_all(text=u"跟蹤標的")[0].next_element.text.strip(u'該基金無跟蹤標的')
except Exception as e:
print (self.getCurrentTime(), fund_code,fund_url,e )

try:
mySQL.insertData('fund_info', result)
#print (self.getCurrentTime(),'Fund Info Insert Sucess:', result['fund_code'],result['fund_name'],result['fund_abbr_name'],result['fund_manager'],result['funder'],result['establish_date'],result['establish_scale'],result['benchmark'] )
except Exception as e:
print (self.getCurrentTime(), fund_code,fund_url,e )

try:
print (self.getCurrentTime(),'getFundInfo:', result['fund_code'],result['fund_name'],result['fund_abbr_name'],result['fund_manager'],result['funder'],result['establish_date'],result['establish_scale'],result['benchmark']
# ,result['issue_date'],result['asset_value'],result['asset_value_date'], result['unit'],result['unit_date'],result['fund_trustee']
# ,result['total_div'],result['mg_fee'],result['trust_fee'], result['sale_fee'], result['buy_fee'],result['buy_fee2'],result['underlying']
)
except Exception as e:
print (self.getCurrentTime(), fund_code,fund_url,e )

return result

def getFundManagers(self,fund_code):
'''
獲取基金經理數據。基金投資分析關鍵在投資經理，後續在完善
:param fund_code:
:return:
'''
fund_url='http://fund.eastmoney.com/f10/jjjl_'+fund_code +'.html'
res = getURL(fund_url)
soup = BeautifulSoup(res.text, 'html.parser')
result = {}
manager={}
tables=soup.find_all("table")
tab = tables[1]
#print (tables[1])
i=0
#先用本辦法，解析表格，逐行逐單元格獲取淨值數據
for tr in tab.findAll('tr'):
#跳過表頭；獲取淨值、累計淨值和日收益率數據如果列數爲7，可以判斷爲一般基金。當然也可以通過標題或者基金類型參數來判斷，待後續優化
if tr.findAll('td') :#and len((tr.findAll('td')))==7 :
i=i+1
try:
result['fund_code']=fund_code
result['start_date']= tr.select('td:nth-of-type(1)')[0].getText().strip()
result['end_date']= tr.select('td:nth-of-type(2)')[0].getText().strip()
result['fund_managers']= tr.select('td:nth-of-type(3)')[0].getText().strip()
result['term']= tr.select('td:nth-of-type(4)')[0].getText().strip()
result['return_rate']= tr.select('td:nth-of-type(5)')[0].getText().strip('%')+'%'
result['created_date']=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
result['updated_date']=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
result['data_source']='eastmoney'

except Exception as e:
print (self.getCurrentTime(),'getFundManagers1', fund_code,fund_url,e )

try:
mySQL.insertData('fund_managers_chg', result)
print (self.getCurrentTime(),'fund_managers_chg:',result['fund_code'],i,result['start_date'],result['end_date'],result['fund_managers'],result['term'],result['return_rate'] )
except Exception as e:
print (self.getCurrentTime(),'getFundManagers2', fund_code,fund_url,e )

for a in tr.findAll('a'):
if a:
try:
manager['manager_id']=a['href'].strip('http://fund.eastmoney.com/manager/.html')
manager['url']=a['href']
manager['manager_name']=a.text
manager['created_date']=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
manager['updated_date']=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
manager['data_source']='eastmoney'
#print (self.getCurrentTime(),manager['manager_id'],manager['manager_name'],manager['url'])
except Exception as e:
print (self.getCurrentTime(),'getFundManagers3', fund_code,manager['manager_name'],manager['url'],fund_url,e )

try:
mySQL.insertData('fund_managers_info', manager)
print (self.getCurrentTime(),'fund_managers_info:',fund_code,manager['manager_name'],manager['url'],manager['manager_id'] )
except Exception as e:
print (self.getCurrentTime(),'getFundManagers4', fund_code,fund_url,e )
#print (self.getCurrentTime(),'getFundManagers',result['fund_code'],'共',str(i)+':','行數保存成功' )

return result

def getFundNav(self,fund_code):
'''
獲取基金淨值數據，因爲基金列表中是所有基金代碼，一般淨值型基金和貨幣基金數據稍有差異，下面根據數據表格長度判斷是一般基金還是貨幣基金，分別入庫
:param fund_code:
:return:
'''
try:
#http://fund.eastmoney.com/f10/F10DataApi.aspx?type=lsjz&code=000001&page=1&per=1
'''
#壽險獲取單個基金的第一頁數據，裏面返回的apidata 接口中包含了記錄數、分頁及數據文件等
#這裏暫按照字符串解析方式獲取，既然是標準API接口，應該可以通過更高效的方式批量獲取全部淨值數據，待後續研究。這裏傳入基金代碼、分頁頁碼和每頁的記錄數。先簡單查詢一次獲取總的記錄數，再一次性獲取所有歷史淨值
首次初始化完成後，如果後續每天更新或者定期更新，只要修改下每頁返回的記錄參數即可
'''
fund_url='http://fund.eastmoney.com/f10/F10DataApi.aspx?type=lsjz&code='+fund_code +'&page=1&per=1'
res = getURL(fund_url)
#獲取歷史淨值的總記錄數
records= (res.text.strip('var apidata=').strip('{;}').split(',')[1].strip('records:'))
#print(res.text.strip('var apidata=').strip('{;}').split(','))
#print (records)
except Exception as e:
print (self.getCurrentTime(),'getFundNav1', fund_code,fund_url,e )
try:
#根據基金代碼和總記錄數，一次返回所有歷史淨值
fund_nav='http://fund.eastmoney.com/f10/F10DataApi.aspx?type=lsjz&code='+fund_code +'&page=1&per='+records
res = getURL(fund_nav)
soup = BeautifulSoup(res.text, 'html.parser')
except Exception as e:
print (self.getCurrentTime(),'getFundNav2', fund_code,fund_url,e )

result={}
result['fund_code']=fund_code
tables = soup.findAll('table')
tab = tables[0]
i=0
#先用本辦法，解析表格，逐行逐單元格獲取淨值數據
for tr in tab.findAll('tr'):
#跳過表頭；獲取淨值、累計淨值和日收益率數據如果列數爲7，可以判斷爲一般基金。當然也可以通過標題或者基金類型參數來判斷，待後續優化
if tr.findAll('td') and len((tr.findAll('td')))==7 :
i=i+1
try:
result['the_date']= (tr.select('td:nth-of-type(1)')[0].getText().strip() )
result['nav']= (tr.select('td:nth-of-type(2)')[0].getText().strip() )
result['add_nav']= (tr.select('td:nth-of-type(3)')[0].getText().strip() )
result['nav_chg_rate']= (tr.select('td:nth-of-type(4)')[0].getText().strip() )
result['buy_state']= (tr.select('td:nth-of-type(5)')[0].getText().strip() )
result['sell_state']= tr.select('td:nth-of-type(6)')[0].getText().strip()
result['div_record']= tr.select('td:nth-of-type(7)')[0].getText().strip().strip('\'')
#print (self.getCurrentTime(),i,result['fund_code'],result['the_date'],result['nav'],result['add_nav'],result['nav_chg_rate'],result['buy_state'],result['sell_state'] )
except Exception as e:
print (self.getCurrentTime(),'getFundNav3', fund_code,fund_url,e )
try:
mySQL.insertData('fund_nav', result)
print (self.getCurrentTime(),'fund_nav',str(i)+'/'+str(records),result['fund_code'],result['the_date'],result['nav'],result['add_nav'],result['nav_chg_rate'],result['buy_state'],result['sell_state'],result['div_record'] )
except Exception as e:
print (self.getCurrentTime(),'getFundNav4', fund_code,fund_url,e )
#如果是貨幣基金，獲取萬份收益和7日年化利率
elif tr.findAll('td') and len((tr.findAll('td')))==6:
i=i+1
try:
result['the_date']= (tr.select('td:nth-of-type(1)')[0].getText().strip() )
#result['nav']=1
result['profit_per_units']= (tr.select('td:nth-of-type(2)')[0].getText().strip() )
result['profit_rate']= (tr.select('td:nth-of-type(3)')[0].getText().strip() )
result['buy_state']= (tr.select('td:nth-of-type(4)')[0].getText().strip() )
result['sell_state']= (tr.select('td:nth-of-type(5)')[0].getText().strip() )
result['div_record']= (tr.select('td:nth-of-type(6)')[0].getText().strip() )
#print (self.getCurrentTime(),i,result['fund_code'],result['the_date'],result['nav'],result['add_nav'],result['nav_chg_rate'],result['buy_state'],result['sell_state'] )
except Exception as e:
print (self.getCurrentTime(),'getFundNav5', fund_code,fund_url,e )
try:
mySQL.insertData('fund_nav_currency', result)
print (self.getCurrentTime(),'fund_nav_currency',str(i)+'/'+str(records),result['fund_code'],result['the_date'],result['profit_per_units'],result['profit_rate'],result['buy_state'],result['sell_state'] )
except Exception as e:
print (self.getCurrentTime(),'getFundNav6', fund_code,fund_url,e )
else :
pass
# if i>=1:
# break
print (self.getCurrentTime(),'getFundNav',result['fund_code'],'共',str(i)+'/'+str(records),'行數保存成功' )

return result

def main():
global mySQL, sleep_time, isproxy, proxy, header
mySQL = PyMySQL()
fundSpiders=FundSpiders()
mySQL._init_('127.0.0.1', 'root', '123456', 'invest')
isproxy = 0 # 如需要使用代理，改爲1，並設置代理IP參數 proxy
proxy = {"http": "http://proxy.zte.com.cn:80", "https": "http://proxy.zte.com.cn:80"}#這裏需要替換成可用的代理IP
header = randHeader()
sleep_time = 0.1
#fundSpiders.getFundJbgk('000001')
funds=fundSpiders.getFundCodesFromCsv()
#fundSpiders.getFundManagers('000001')
for fund in funds:
try:
fund1=str(fund)
fund1=fund1.zfill(6)
fundSpiders.getFundInfo(fund1)
print("this is ------------ begin")
print(fund)
print("this is ------------end ")
fundSpiders.getFundManagers(fund1)
fundSpiders.getFundNav(fund1)
except Exception as e:
print (getCurrentTime(),'main', fund1,e )

if __name__ == "__main__":
main()

基金代碼保存的問題：

24-5-18 X

基金爬蟲

12306搶票軟件的一些學習

mysql數據庫的批量文件導入

python量化投資研究

基金研究

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結