C# 新浪微博滚动抓取 WeiboGrab

应该先说，本来相对网页加载的程序段进行规范的，但是，当再次编写的时候发现，还是不能很好的掌握网页加载的具体规则，导致获取页面的代码还是很繁杂。其他部分改的差不多了，还有就是当微博中的字符含有{}等时，会提示字符串格式错误，这个也该需要改进的，，还没改进，程序还需要一个挂空线程的功能，保留现场，让程序可以继续爬取，而不是从头再爬。

各种类

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using HtmlAgilityPack;

namespace WeiBoGrab
{
    class WeiBoGrabClass
    {
    }
    
    public class GetPage
    {
        //加载初始页面
        public string GetLoginPage(WebBrowser browser)
        {
            while (browser.ReadyState != WebBrowserReadyState.Complete)
            {
                Application.DoEvents();
            }
            while (browser.Document.GetElementById("pl_login_form").InnerHtml == null)
            {
                Application.DoEvents();
            }
            return "加载登陆页面完成。";
        }
       //加载用户主页
        public string GetMainPage(WebBrowser browser)
        {
            while (browser.DocumentTitle != "我的首页 新浪微博-随时随地分享身边的新鲜事儿")
            {
                Application.DoEvents();
            }
             //确保加载完所需内容
            while (browser.Document.GetElementById("pl_rightmod_myinfo")!=null&&
                browser.Document.GetElementById("pl_rightmod_myinfo").Children.Count < 2)
            {
                Application.DoEvents();
            }
            
            return "加载个人主页完成。";
        }
       //加载用户关注对象的第一页
        public string GetFollowsPage(WebBrowser browser)
        {
            while (browser.DocumentTitle != "我关注的人 新浪微博-随时随地分享身边的新鲜事儿")
            {
                Application.DoEvents();
            }
            while (browser.Document.GetElementById("pl_relation_myfollow") == null)
            {
                Application.DoEvents();
            }
            while (browser.Document.GetElementById("pl_relation_myfollow").Children.Count < 3)
            {
                Application.DoEvents();
            }
            return "关注对象页面第一页加载完成。";
        }
        //加载用户关注对象的下一页
        public string GetFollowsNextPage(WebBrowser browser)
        {
            //将原页面的关注对象列表清空（关注对象列表为children[2].children[1]）
            //加载新页面3=browser.Document.GetElementById("pl_relation_myfollow").Children[2].Children.Count
            //不明白，孩子个数显示明明是3，但是述操作却正确。。。 
            //browser.Document.GetElementById("pl_relation_myfollow").Children[2].Children.Count < 4
            //<!--  -->此类标签有时会被当做标签计数或提取，需要实际分析

            while (browser.Document.GetElementById("pl_relation_myfollow").Children.Count < 3||
                   browser.Document.GetElementById("pl_relation_myfollow").Children[2].Children.Count < 4)
            {
                Application.DoEvents();
            }
            //当上述条件满足后，再加载，便是新生成的内容
            return "关注对象下一页加载完成。";
        }
        //加载关注对象的主页的第一页
        public string GetFollowMainPage(WebBrowser browser)
        {
            while (browser.ReadyState != WebBrowserReadyState.Complete)
            {
                Application.DoEvents();
            }

            //当微博是杂志、新闻类时
            if (browser.Document.GetElementById("epfeedlist") != null)
            {
                while (browser.Document.GetElementById("feed_list") == null)
                {
                    Application.DoEvents();
                }
                return "关注对象主页第一页加载完成。";
            }
            //当微博是个人、媒体类时
            if (browser.Document.GetElementById("pl_content_hisFeed") == null)
            {
                while (browser.Document.GetElementById("profileFeed").InnerHtml == null)
                {
                    Application.DoEvents();
                }
            }
            while (browser.Document.GetElementById("pl_content_hisFeed").InnerHtml == null)
            {
                Application.DoEvents();
            }
            //找到feed
            HtmlElementCollection ps = browser.Document.GetElementById("pl_content_hisFeed").Children;
            int feed_postion = 0;
            //有的微博页面需要此步骤
            while (browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].InnerText == "正在加载，请稍候..." ||
                browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].InnerText == "正在加载中，请稍候...")
            {
                Application.DoEvents();
            }
            //pl_content_hisFeed加载不全
            while (browser.Document.GetElementById("pl_content_hisFeed").Children.Count < 2)
            {
                Application.DoEvents();
            }
            foreach (HtmlElement p in ps)
            {
                if (p.GetAttribute("node-type") != null && p.GetAttribute("node-type") == "feed_list")
                {
                    break;
                }
                else
                    feed_postion++;
            }
            //非第一页加载时，有此等待
            while (browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[0].InnerText == "正在加载中，请稍候..."
                   || browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[0].InnerText == "正在加载，请稍候...")
            {
                Application.DoEvents();
            }
            //微博数量及等待加载模块所在位置表示
            int hisFeed_count = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children.Count - 1;
            //表示正在加载
            bool loading = true;
            //找出加载模块位置
            HtmlElement load = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[hisFeed_count];
            int i;
            for (i = 1; (i < 10) && (hisFeed_count - i >= 0); i++)
            {
                if (load.InnerText == "正在加载中，请稍候..." || load.InnerText == "正在加载，请稍候...")
                    break;
                load = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[hisFeed_count - i];
            }
            while (loading)
            {
                loading = false;
                load.ScrollIntoView(false);
                while (load.InnerText == "正在加载中，请稍候..." || load.InnerText == "正在加载，请稍候...")
                {
                    load.ScrollIntoView(false);
                    Application.DoEvents();
                    load = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[hisFeed_count - i];
                }
                //微博加载
                //限制次数，limit有待商榷，过小会使有的微博可能会加载失败
                int Limit = 100;
                int L = 0;
                while ((browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children.Count < hisFeed_count + 2)&&
                    (L < Limit ))
                {
                    L++;//防止无限加载的等待
                    Application.DoEvents();
                }
                //更新加载模块位置
                hisFeed_count = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children.Count - 1;
                //更新加载模块
                load = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[hisFeed_count];
                for (int j = 1; (j < 10) && (hisFeed_count - j >= 0); j++)//假设无效的标签数不超过10个
                {
                    if (load.InnerText == "正在加载中，请稍候..." || load.InnerText == "正在加载，请稍候...")
                        break;
                    load = browser.Document.GetElementById("pl_content_hisFeed").Children[feed_postion].Children[hisFeed_count - j];
                }
                if (load != null && (load.InnerText == "正在加载中，请稍候..." || load.InnerText == "正在加载，请稍候..."))
                {
                    loading = true;
                    load.ScrollIntoView(false);
                }
            }
            return "加载关注对象主页第一页面完成。";

        }
        //加载关注对象的的主页的下一页
        public string GetFollowMainNextPage(WebBrowser browser)
        {
            Application.DoEvents();
            while (browser.ReadyState != WebBrowserReadyState.Complete)
            {
                Application.DoEvents();
            }
            GetFollowMainPage(browser);
            //针对杂志、新闻类微博
            if (browser.Document.GetElementById("epfeedlist") == null)
                Application.DoEvents();
            return "加载关注对象后续页面完成。";
        }
    }
    //用户登陆类
    public class LoginSubmit
    {
        private string username;
        private string password;
        //初始化登陆对象
        public LoginSubmit(string username, string password)
        {
            this.username = username;
            this.password = password;
        }
        //点击登陆
        public void LoginClick(WebBrowser browser)
        {
            //登陆页面的登陆模块
            HtmlElement pl_login_form = browser.Document.GetElementById("pl_login_form");
            //登陆模块中的用户名_INPUT
            HtmlElement pl_login_form_username = pl_login_form.GetElementsByTagName("INPUT")[0];
            //让用户名输入框获取焦点(目的清空输入框)
            pl_login_form_username.InvokeMember("click");
            pl_login_form_username.SetAttribute("value",username);

            //登陆模块的密码_INPUT
            HtmlElement pl_login_form_password = pl_login_form.GetElementsByTagName("INPUT")[1];
            //让密码输入框获取焦点(目的清空输入框)
            pl_login_form_password.InvokeMember("click");
            pl_login_form_password.SetAttribute("value",password);

            //找到登陆按钮并点击
            HtmlElementCollection IsClick = pl_login_form.GetElementsByTagName("span");
            foreach (HtmlElement Click in IsClick)
            {
                if (Click.GetAttribute("node-type") != null && Click.GetAttribute("node-type") == "submitStates")
                {
                    Click.InvokeMember("click");
                    break;
                }
            }
        }
    }
    //将关注对象设为一类
    public class Follow
    {
        //获取关注对象（点击用户关注对象的超链接）
        public void GetFollows(WebBrowser browser)
        {
            //获取用户的信息模块
            HtmlElement pl_rightmod_myinfo = browser.Document.GetElementById("pl_rightmod_myinfo");
            //获取关注对象子模块
            HtmlElement my_info_follow = pl_rightmod_myinfo.GetElementsByTagName("strong")[0];
            if (my_info_follow.GetAttribute("node-type") == "follow")
            {
                //判断用户是否有关注对象
                if (my_info_follow.InnerText == "0")
                    return;
                my_info_follow.InvokeMember("click");
                GetPage getfollowpage = new GetPage();
                getfollowpage.GetFollowsPage(browser);
            }
        }
        //获取关注对象的url,并写到txt中
        public void GetFollowsUrl(WebBrowser browser, StreamWriter sw)
        {
            //是否还有下一页
            bool Next = true;
            int UrlCount = 0;
            while (Next)
            {
                //默认没有下一页
                Next = false;

                HtmlElement FollowLinks = browser.Document.GetElementById("pl_relation_myfollow");
                HtmlElementCollection Links = FollowLinks.GetElementsByTagName("div");

                foreach (HtmlElement Link in Links)
                {
                    if (Link.GetAttribute("action-type") == "ignore_list")
                    {
                        HtmlNode href = HtmlNode.CreateNode(Link.InnerHtml);

                        string url = href.Attributes["href"].Value;
                        string followname = href.FirstChild.Attributes["alt"].Value;

                        sw.WriteLine("No.{0}|{1}|{2}", ++UrlCount, followname, url);
                    }
                }
                HtmlElementCollection pages = FollowLinks.GetElementsByTagName("span");

                //判断是否有下一页
                foreach (HtmlElement page in pages)
                {
                    if (page.InnerText == "下一页")
                    {
                        Next = true;
                        page.InvokeMember("click");
                        
                        //Console.WriteLine("这个标签是："+browser.Document.GetElementById("pl_relation_myfollow").Children[2].Children[2].OuterHtml);
                        browser.Document.GetElementById("pl_relation_myfollow").Children[2].Children[2].OuterHtml = null;
                        
                        //载入关注对象页面的下一页
                        GetPage GetNext = new GetPage();
                        GetNext.GetFollowsNextPage(browser);
                        break;
                    }
                }
            }
            sw.Close();
        }
    }
    //将微博设为一类
    public class WeiBo
    {
        private string FollowName;
        private string FollowUrl;

        public WeiBo(string FollowName, string FollowUrl)
        {
            this.FollowName = FollowName;
            this.FollowUrl = FollowUrl;
        }
        public void GetWeiBo(WebBrowser browser)
        {
            StreamWriter sw = File.CreateText("D:\\weibo\\" + FollowName + ".txt");
            bool Next = true;
            int WeiBoCount = 0;
            browser.Navigate(new Uri(@FollowUrl));
            GetPage GetNext = new GetPage();
            GetNext.GetFollowMainPage(browser);
            //默认还没登记此类微博
            string Kind = "N";

            HtmlElement epfeedlist = browser.Document.GetElementById("epfeedlist");
            HtmlElement pl_content_hisFeed = browser.Document.GetElementById("pl_content_hisFeed");
            if (pl_content_hisFeed != null)
            {
                //媒体类微博的pl_content_hisFeed.Children[1].Children[0].TagName = "dl"
                
                //个人微博的pl_content_hisFeed.Children[1].OuterHtml =<!-- /高级搜索 -->
                if (pl_content_hisFeed.Children[1].Children.Count != 0)
                        //媒体（小）微博
                        Kind = "M";

                    //个人微博                
                else
                     Kind = "P";
            }
            if (epfeedlist != null)
                //杂志，新闻等微博
                Kind = "J";
            while (Next)
            {
                Next = false;
                switch (Kind)
                {
                    case "P": 
                        {
                            //爬取各条微博
                          HtmlElementCollection divs = browser.Document.GetElementById("pl_content_hisFeed").GetElementsByTagName("div");
                          foreach (HtmlElement div in divs)
                           {
                            if (div.GetAttribute("node-type") == "feed_list_content")
                                sw.WriteLine("第{0}条|" + div.InnerText, ++WeiBoCount);
                           }
                            //判断是否还有下一页
                          HtmlElementCollection spans = browser.Document.GetElementById("pl_content_hisFeed").GetElementsByTagName("span");
                          foreach (HtmlElement span in spans)
                          {
                              if (span.InnerText == "下一页")
                              {
                                  span.InvokeMember("click");
                                  Next = true;
                                  GetNext.GetFollowMainNextPage(browser);
                                  break;
                              }
                          }
                        }break;
                    case "J":
                        {
                            //爬取各条微博
                            int count_li = browser.Document.GetElementById("feed_list").Children.Count;
                            for (int i = 0; i < count_li; i++)
                            {
                                sw.WriteLine("第{0}条|" + browser.Document.GetElementById("feed_list").Children[i].GetElementsByTagName("p")[0].InnerText, ++WeiBoCount);
                            }
                            //判断是否还有下一页
                            HtmlElementCollection ems = browser.Document.GetElementById("feed_list").NextSibling.GetElementsByTagName("em");
                            int end = ems.Count;
                            if (ems[end - 1].InnerText == "下一页")
                            {
                                ems[end - 1].InvokeMember("click");
                                browser.Document.GetElementById("feed_list").OuterHtml = null;
                                GetNext.GetFollowMainNextPage(browser);
                                Next = true;
                            }
                        }break;
                    case "M":
                        {
                            HtmlElementCollection ps = browser.Document.GetElementById("pl_content_hisFeed").GetElementsByTagName("p");
                            foreach (HtmlElement p in ps)
                            {
                                if (p.GetAttribute("node-type") == "feed_list_content")
                                    sw.WriteLine("第{0}条|" + p.InnerText, ++WeiBoCount);
                            }
                            //判断是否还有下一页
                            HtmlElementCollection spans = browser.Document.GetElementById("pl_content_hisFeed").GetElementsByTagName("span");
                            foreach (HtmlElement span in spans)
                            {
                                if (span.InnerText == "下一页")
                                {
                                    span.InvokeMember("click");
                                    Next = true;
                                    GetNext.GetFollowMainNextPage(browser);
                                    break;
                                }
                            }

                        }break;
                    default: return;//还没记录的微博
                }
            }
            sw.Close();
        }
    }
}

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;

namespace WeiBoGrab
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            string username = textBox1.Text.ToString();
            string password = textBox2.Text.ToString();
            string url = "http://weibo.com/";
            GetPage getpage = new GetPage();
            StreamWriter sw = File.CreateText("FollowUrl.txt");
            WebBrowser browser = webBrowser1;

            browser.Navigate(new Uri(@url));
            //加载登陆页面
            textBox3.Text += getpage.GetLoginPage(browser);
            //登陆操作
            LoginSubmit loginsubmit = new LoginSubmit(username, password);
            loginsubmit.LoginClick(browser);
            //加载个人主页
            textBox3.Text += getpage.GetMainPage(browser);
            //获取关注对象
            Follow follow = new Follow();
            follow.GetFollows(browser);                     
            follow.GetFollowsUrl(browser,sw);

            FileStream fs = new FileStream("FollowUrl.txt",FileMode.Open);
            StreamReader sr = new StreamReader(fs);
            string s;
            while ((s = sr.ReadLine()) != null)
            {
                string[] arry = s.Split('|');
                string name = arry[1];
                string user_url = arry[2];
                WeiBo feed = new WeiBo(name, user_url);
                feed.GetWeiBo(browser);
            }
            sr.Close();
        }
    }
}