proquest的採集程序demo測試截圖
程序C#.net開發，程序使用了 Castle Activerecord, Nhibernate ,Spring.net, Memcached分佈式數據排重,
自動分頁，錯誤自動重試，超時處理等自能糾錯技術，，
配置 部分
</configSections>
<appSettings>
<add key="startId" value="592" />
<add key="endId" value="600" />
<add key="startPId" value="0" />
<add key="Cookies" value="" />
<add key="DBID" value="10767" />
<add key="SINDEX" value="10" />
<add key="EINDEX" value="20" />
<add key="PM" value="53153" />
<add key="PC" value="6490441" />
<add key="D" value="437140591" />
<add key="backupData" value="53153,6490441,437140541,2315" />
</appSettings>
部分代碼
rtn = Zfr.Common.Helper.GetPage(url, cookies, Encoding.GetEncoding("iso-8859-1"), url, null);
if (rtn.IndexOf("LOGIN ERROR MESSAGE") != -1) {
Zfr.Common.ConsoleProgram.ShowError("線程->LOGIN ERROR,自動登錄");
System.Threading.Thread.Sleep(3000);
GetCookies(rtn);
goto p;
}
if (rtn.IndexOf("The remote name could not be resolved") != -1 ||
rtn.IndexOf("has timed out") != -1 ||
rtn.IndexOf("Bad Request") != -1)
{
error++;
if (error < 4)
{
Zfr.Common.ConsoleProgram.ShowError("線程->錯誤,暫停4秒,自動重試");
System.Threading.Thread.Sleep(4000);
goto p;
}
else
{
Zfr.Common.ConsoleProgram.ShowError("線程->錯誤,超過3次,回車跳過,繼續"); Console.ReadLine();
}
---------------------------------
if (MemcachedBench.KeyExists("" + obj.HASH))
{
Zfr.Common.ConsoleProgram.ShowError("KEY->重複:" + obj.HASH);
}
else
{
--------------------------------------------------------------------
if (m.Success)
{
obj.magLink = m.Groups[0].Value;
}
DB<ProQuestItem>.Create(obj);
Zfr.Common.ConsoleProgram.ShowSuccess("DATA->保存成功 ID:" + obj.ID);
MemcachedBench.Set("" + obj.HASH, null); successCount++;
SetConfig("backupData", string.Format("{0},{1},{2},{3}", pmid, pcid, did, obj.ID));
----------------------------------------------------------
void ShowSpeed()
{
TimeSpan ts = DateTime.Now - startTime;
double s = ts.TotalSeconds;
Zfr.Common.ConsoleProgram.Show(string.Format("INFO->{11}|{8} {9} {10}|{5} {6} {7}|C:{4} SP:{1} {2} {3} SPD:{0}", 24 * 60 * 60 * successCount / s, ts.Hours, ts.Minutes, ts.Seconds,successCount, pmid, pcid, did,sIndex,eIndex,cIndex,dbid));
}
------------------------------------------------
測試程序demo截圖pro-qq1163551688.jpg
