import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.ningze.db.model.T003ProductInfo;
import com.ningze.rest.xiaobu.XiaobuAPI;
/**
* **銀行理財數據爬取
*
* @author ROCK
*
*/
public class KunlunBank {
public static void main(String[] args) throws ClientProtocolException, IOException, ParseException {
System.out.println(KunlunBank.getData());
//getData();
}
public static List<T003ProductInfo> getData() throws ClientProtocolException, IOException, ParseException {
RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.DEFAULT).build();
CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(globalConfig).build();
RequestConfig localConfig = RequestConfig.copy(globalConfig).setCookieSpec(CookieSpecs.STANDARD_STRICT).build();
String url_1 = "https://www.eklb.cn";
//創建HttpGet實例
HttpGet httpGet = new HttpGet(url_1);
httpGet.setConfig(localConfig);
//創建響應處理器處理服務器響應內容
ResponseHandler<String> veriCodeHandler = handler();
//二次請求獲取cookie覆蓋第一次請求，證明是同一個瀏覽器訪問，真正獲取請求
String response = httpClient.execute(httpGet, veriCodeHandler);//使用給定的上下文執行HTTP請求。
//System.out.println(response);
String getDataURL = "https://www.eklb.cn/directbank/financeQueryList.do";
int offSet = 0 , QueryNum = 5;//分頁參數
//這個代碼多處用於Java像url發送Post請求。在發送post請求時用該list來存放參數。
//定義了一個數組集合對象，通過泛型聲明瞭數組裏的元素類型。
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("chanlNo", "DB"));
formparams.add(new BasicNameValuePair("bySort", ""));
formparams.add(new BasicNameValuePair("descAsc", "desc"));
formparams.add(new BasicNameValuePair("tranType", "actualYield"));
formparams.add(new BasicNameValuePair("page", String.valueOf(offSet)));
formparams.add(new BasicNameValuePair("pageSize", String.valueOf(QueryNum)));
UrlEncodedFormEntity requestFormEntity = new UrlEncodedFormEntity(formparams, Consts.UTF_8);
HttpPost httpPost = new HttpPost(getDataURL);
httpPost.setEntity(requestFormEntity);
httpPost.setConfig(localConfig);
ResponseHandler<String> getDataHandler = handler();
response = httpClient.execute(httpPost, getDataHandler);
List<T003ProductInfo> list= new ArrayList<>();
if (null != response) {
JSONObject firstPageJson = JSONObject.parseObject(response);
//System.out.println(firstPageJson);
String resCode = firstPageJson.getString("errorCode");
if ("000000".equals(resCode)) {
int pageTotalNum = Integer.valueOf(firstPageJson.getString("pageTotalNum"));
int page = Integer.valueOf(firstPageJson.getString("page"));
JSONArray records = firstPageJson.getJSONArray("financeList");
//System.out.println("-------------------------"+records);
records.forEach(oneRecord -> {
// 每一個產品的解析
JSONObject oneRecordJson = (JSONObject) oneRecord;
try {
list.add(getFormatData(oneRecordJson));
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
});
// 開始分頁請求
int pages = pageTotalNum / QueryNum;
if (pageTotalNum % QueryNum > 0) {
pages += 1;
}
for (int i = 2; i <= page; i++) {
formparams = new ArrayList<NameValuePair>();
formparams.add(new BasicNameValuePair("chanlNo", "DB"));
formparams.add(new BasicNameValuePair("bySort", ""));
formparams.add(new BasicNameValuePair("descAsc", "desc"));
formparams.add(new BasicNameValuePair("tranType", "actualYield"));
//formparams.add(new BasicNameValuePair("page", String.valueOf(offSet)));
formparams.add(new BasicNameValuePair("OffSet", String.valueOf((i - 1) * 4 + 1)));
formparams.add(new BasicNameValuePair("pageSize", String.valueOf(QueryNum)));
requestFormEntity = new UrlEncodedFormEntity(formparams, Consts.UTF_8);
httpPost = new HttpPost(getDataURL);
httpPost.setEntity(requestFormEntity);
httpPost.setConfig(localConfig);
getDataHandler = handler();
response = httpClient.execute(httpPost, getDataHandler);
if (null != response) {
if ("000000".equals(resCode)) {
records = firstPageJson.getJSONArray("financeList");
records.forEach(oneRecord -> {
// 每一個產品的解析
JSONObject oneRecordJson = (JSONObject) oneRecord;
try {
list.add(getFormatData(oneRecordJson));
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
});
}
}
}
}
}
return list;
}
private static T003ProductInfo getFormatData(JSONObject oneJsonRecord) throws ParseException {
T003ProductInfo oneProductInfo = new T003ProductInfo();
//銀行名稱
oneProductInfo.setBankName("**銀行");
//產品ID
oneProductInfo.setPrdId(oneJsonRecord.getString("productId"));
//產品名稱
oneProductInfo.setPrdName(oneJsonRecord.getString("productName"));
//年化收益
oneProductInfo.setIncomeRate(oneJsonRecord.getString("actualYield"));
//投資時長
oneProductInfo.setInvestDuration(oneJsonRecord.getString("productTerm"));
//時長單位 1:天;2:月;3:年
oneProductInfo.setDurationUnit("天");
// 計算最少起投金額
oneProductInfo.setFloor(oneJsonRecord.getString("raiseMinAmt"));
// 計算剩餘額度
oneProductInfo.setTotalAmount(oneJsonRecord.getString("maxAccRaiseAmt"));
oneProductInfo.setLeftAmount(String.valueOf(Double.valueOf(oneJsonRecord.getString("maxAccRaiseAmt"))
- Double.valueOf(oneJsonRecord.getString("maxRaiseAmt"))));
//產品類型碼（理財、基金、票據、P2P、各種寶）
oneProductInfo.setPrdType("理財");
//理財類型(0:封閉型、1:開放型)
oneProductInfo.setFinancingType("0");
// 募集開始日募集結束日,"projectBeginTime": "20171117093000","jMBeginTime":"20171119160000", // 募集開始日
oneProductInfo.setStartDay(oneJsonRecord.getString("raiseBegDate"));
// 募集結束日
oneProductInfo.setEndDay(oneJsonRecord.getString("raiseEndDate"));
// 起息日
oneProductInfo.setActiveDay(getStartDate(oneJsonRecord.getString("raiseEndDate")));
// 到期日
oneProductInfo.setOverdueDay(oneJsonRecord.getString("expireDate"));
return oneProductInfo;
}
/**
* @return
*/
private static <T> ResponseHandler<T> handler() {
// TODO Auto-generated method stub
ResponseHandler<T> loginHandler = new ResponseHandler<T>() {
@SuppressWarnings("unchecked")
@Override
public T handleResponse(final HttpResponse response) throws IOException {
StatusLine statusLine = response.getStatusLine();
HttpEntity entity = response.getEntity();
if (statusLine.getStatusCode() >= 300) {
throw new HttpResponseException(statusLine.getStatusCode(), statusLine.getReasonPhrase());
}
if (entity == null) {
throw new ClientProtocolException("Response contains no content");
}
entity = new BufferedHttpEntity(entity);
String responseAsString = EntityUtils.toString(entity, "UTF-8");
return (T) responseAsString;
}
};
return loginHandler;
}
/*
* raiseEndDate 20180109(募集結束日期)
*
* 起息日 20180110 ( raiseEndDate + 1 )
*/
public static String getStartDate(String endDay) throws ParseException {
SimpleDateFormat sf = new SimpleDateFormat("yyyyMMdd");
Calendar ca = Calendar.getInstance();
Date d = sf.parse(endDay);
ca.setTime(d);
ca.add(Calendar.DAY_OF_YEAR,1);
return sf.format(ca.getTime());
}
}
}
![]()
![]()
![]()
![]()
