C# 抓取网页Html源码 ，网络爬虫

privatevoidSearch(stringurl)

{

stringrl;

WebRequest Request = WebRequest.Create(url.Trim());

WebResponse Response = Request.GetResponse();

Stream resStream = Response.GetResponseStream();

StreamReader sr =newStreamReader(resStream, Encoding.Default);

StringBuilder sb =newStringBuilder();

while((rl = sr.ReadLine()) !=null)

{

sb.Append(rl);

}

stringstr = sb.ToString().ToLower();

stringstr_get = mid(str,"<ul class=\"post_list\">","</ul>");

intstart = 0;

while(true)

{

if(str_get ==null)

break;

stringstrResult = mid(str_get,"href=\"","\"",outstart);

if(strResult ==null)

break;

else

{

lab[url] += strResult;

str_get = str_get.Substring(start);

}

}

}

privatestringmid(stringistr,stringstartString,stringendString)

{

intiBodyStart = istr.IndexOf(startString, 0);//开始位置

if(iBodyStart == -1)

returnnull;

iBodyStart += startString.Length;//第一次字符位置起的长度

intiBodyEnd = istr.IndexOf(endString, iBodyStart);//第二次字符在第一次字符位置起的首次位置

if(iBodyEnd == -1)

returnnull;

iBodyEnd += endString.Length;//第二次字符位置起的长度

stringstrResult = istr.Substring(iBodyStart, iBodyEnd - iBodyStart - 1);

returnstrResult;

}

privatestringmid(stringistr,stringstartString,stringendString,outintiBodyEnd)

{

//初始化out参数,否则不能return

iBodyEnd = 0;

intiBodyStart = istr.IndexOf(startString, 0);//开始位置

if(iBodyStart == -1)

returnnull;

iBodyStart += startString.Length;//第一次字符位置起的长度

iBodyEnd = istr.IndexOf(endString, iBodyStart);//第二次字符在第一次字符位置起的首次位置

if(iBodyEnd == -1)

returnnull;

iBodyEnd += endString.Length;//第二次字符位置起的长度

stringstrResult = istr.Substring(iBodyStart, iBodyEnd - iBodyStart - 1);

returnstrResult;

}