C#通过正则表达式抓取网页信息的类

概述C#通过正则表达式抓取网页信息的类

下面是内存溢出 jb51.cc 通过网络收集整理的代码片段。

内存溢出小编现在分享给大家，也给大家做个参考。

using System;using System.Data;using System.Configuration;using System.Net;using System.IO;using System.Text;using System.Collections.Generic;using System.Text.RegularExpressions;using System.Threading;using System.Web;using System.Web.UI.MobileControls;    /// <summary>    /// 网页类    /// </summary>    public class WebPage    {        #region 私有成员        private Uri m_uri;   //url        private List<link> m_links;    //此网页上的链接        private string m_Title;        //标题        private string m_HTML;         //HTML代码        private string m_outstr;       //网页可输出的纯文本        private bool m_good;           //网页是否可用        private int m_pagesize;       //网页的大小        private static Dictionary<string,cookieContainer> webcookies = new Dictionary<string,cookieContainer>();//存放所有网页的cookie               #endregion        #region 属性        /// <summary>        /// 通过此属性可获得本网页的网址，只读        /// </summary>        public string URL        {            get            {                return m_uri.absoluteUri;            }        }        /// <summary>        /// 通过此属性可获得本网页的标题，只读        /// </summary>        public string Title        {            get            {                if (m_Title == "")                {                    Regex reg = new Regex(@"(?m)<Title[^>]*>(?<Title>(?:\w|\W)*?)</Title[^>]*>",RegexOptions.Multiline | RegexOptions.IgnoreCase);                    Match mc = reg.Match(m_HTML);                    if (mc.Success)                        m_Title = mc.Groups["Title"].Value.Trim();                }                return m_Title;            }        }        public string M_HTML        {            get            {                if (m_HTML == null)                {                    m_HTML = "";                }                return m_HTML;            }        }        /// <summary>        /// 此属性获得本网页的所有链接信息，只读        /// </summary>        public List<link> links        {            get            {                if (m_links.Count == 0) getlinks();                return m_links;            }        }        /// <summary>        /// 此属性返回本网页的全部纯文本信息，只读        /// </summary>        public string Context        {            get            {                if (m_outstr == "") getContext(Int16.MaxValue);                return m_outstr;            }        }        /// <summary>        /// 此属性获得本网页的大小        /// </summary>        public int PageSize        {            get            {                return m_pagesize;            }        }        /// <summary>        /// 此属性获得本网页的所有站内链接        /// </summary>        public List<link> Insitelinks        {            get            {                return getSpeciallinksByUrl("^http://" + m_uri.Host,Int16.MaxValue);            }        }        /// <summary>        /// 此属性表示本网页是否可用        /// </summary>        public bool IsGood        {            get            {                return m_good;            }        }        /// <summary>        /// 此属性表示网页的所在的网站        /// </summary>        public string Host        {            get            {                return m_uri.Host;            }        }        #endregion        /// <summary>        /// 从HTML代码中分析出链接信息        /// </summary>        /// <returns>List<link></returns>        private List<link> getlinks()        {            if (m_links.Count == 0)            {                Regex[] regex = new Regex[2];                regex[0] = new Regex(@"<a\shref\s*=""(?<URL>[^""]*).*?>(?<Title>[^<]*)</a>",RegexOptions.IgnoreCase | RegexOptions.Singleline);                regex[1] = new Regex("<[i]*frame[^><]+src=(\"|')?(?<url>([^>\"'\s)])+)(\"|')?[^>]*>",RegexOptions.IgnoreCase);                for (int i = 0; i < 2; i++)                {                    Match match = regex[i].Match(m_HTML);                    while (match.Success)                    {                        try                        {                            string url = httpUtility.UrlDecode(new Uri(m_uri,match.Groups["URL"].Value).absoluteUri);                            string text = "";                            if (i == 0) text = new Regex("(<[^>]+>)|(\s)|(&nbsp;)|&|\"",RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(match.Groups["text"].Value,"");                            link link = new link();                            link.Text = text;                            link.NavigateUrl = url;                            m_links.Add(link);                        }                        catch (Exception ex) { Console.Writeline(ex.Message); };                        match = match.Nextmatch();                    }                }            }            return m_links;        }        /// <summary>        /// 此私有方法从一段HTML文本中提取出一定字数的纯文本        /// </summary>        /// <param name="instr">HTML代码</param>        /// <param name="firstN">提取从头数多少个字</param>        /// <param name="withlink">是否要链接里面的字</param>        /// <returns>纯文本</returns>        private string getFirstNchar(string instr,int firstN,bool withlink)        {            if (m_outstr == "")            {                m_outstr = instr.Clone() as string;                m_outstr = new Regex(@"(?m)<script[^>]*>(\w|\W)*?</script[^>]*>",RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(m_outstr,"");                m_outstr = new Regex(@"(?m)<style[^>]*>(\w|\W)*?</style[^>]*>","");                m_outstr = new Regex(@"(?m)<select[^>]*>(\w|\W)*?</select[^>]*>","");                if (!withlink) m_outstr = new Regex(@"(?m)<a[^>]*>(\w|\W)*?</a[^>]*>","");                Regex objReg = new System.Text.RegularExpressions.Regex("(<[^>]+?>)|&nbsp;",RegexOptions.Multiline | RegexOptions.IgnoreCase);                m_outstr = objReg.Replace(m_outstr,"");                Regex objReg2 = new System.Text.RegularExpressions.Regex("(\s)+",RegexOptions.Multiline | RegexOptions.IgnoreCase);                m_outstr = objReg2.Replace(m_outstr," ");            }            return m_outstr.Length > firstN ? m_outstr.Substring(0,firstN) : m_outstr;        }        #region 公有文法        /// <summary>        /// 此公有方法提取网页中一定字数的纯文本，包括链接文字        /// </summary>        /// <param name="firstN">字数</param>        /// <returns></returns>        public string getContext(int firstN)        {            return getFirstNchar(m_HTML,firstN,true);        }        /// <summary>        /// 此公有方法从本网页的链接中提取一定数量的链接，该链接的URL满足某正则式        /// </summary>        /// <param name="pattern">正则式</param>        /// <param name="count">返回的链接的个数</param>        /// <returns>List<link></returns>        public List<link> getSpeciallinksByUrl(string pattern,int count)        {            if (m_links.Count == 0) getlinks();            List<link> Speciallinks = new List<link>();            List<link>.Enumerator i;            i = m_links.GetEnumerator();            int cnt = 0;            while (i.MoveNext() && cnt < count)            {                if (new Regex(pattern,RegexOptions.Multiline | RegexOptions.IgnoreCase).Match(i.Current.NavigateUrl).Success)                {                    Speciallinks.Add(i.Current);                    cnt++;                }            }            return Speciallinks;        }        /// <summary>        /// 此公有方法从本网页的链接中提取一定数量的链接，该链接的文字满足某正则式        /// </summary>        /// <param name="pattern">正则式</param>        /// <param name="count">返回的链接的个数</param>        /// <returns>List<link></returns>        public List<link> getSpeciallinksByText(string pattern,RegexOptions.Multiline | RegexOptions.IgnoreCase).Match(i.Current.Text).Success)                {                    Speciallinks.Add(i.Current);                    cnt++;                }            }            return Speciallinks;        }        /// <summary>        /// 这公有方法提取本网页的纯文本中满足某正则式的文字        /// </summary>        /// <param name="pattern">正则式</param>        /// <returns>返回文字</returns>        public string getSpecialWords(string pattern)        {            if (m_outstr == "") getContext(Int16.MaxValue);            Regex regex = new Regex(pattern,RegexOptions.Multiline | RegexOptions.IgnoreCase);            Match mc = regex.Match(m_outstr);            if (mc.Success)                return mc.Groups[1].Value;            return string.Empty;        }        #endregion        #region 构造函数        private voID Init(string _url)        {            try            {                m_uri = new Uri(_url);                m_links = new List<link>();                m_HTML = "";                m_outstr = "";                m_Title = "";                m_good = true;                if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith(".msi"))                {                    m_good = false;                    return;                }                httpWebRequest rqst = (httpWebRequest)WebRequest.Create(m_uri);                rqst.AllowautoRedirect = true;                rqst.MaximumautomaticRedirections = 3;                rqst.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; windows NT 5.0)";                rqst.KeepAlive = true;                rqst.Timeout = 10000;                lock (WebPage.webcookies)                {                    if (WebPage.webcookies.ContainsKey(m_uri.Host))                        rqst.cookieContainer = WebPage.webcookies[m_uri.Host];                    else                    {                        cookieContainer cc = new cookieContainer();                        WebPage.webcookies[m_uri.Host] = cc;                        rqst.cookieContainer = cc;                    }                }                httpWebResponse rsps = (httpWebResponse)rqst.GetResponse();                Stream sm = rsps.GetResponseStream();                if (!rsps.ContentType.Tolower().StartsWith("text/") || rsps.ContentLength > 1 << 22)                {                    rsps.Close();                    m_good = false;                    return;                }                EnCoding cding = System.Text.EnCoding.Default;                string ContentType = rsps.ContentType.Tolower();                int ix = ContentType.IndexOf("charset=");                if (ix != -1)                {                    try                    {                        cding = System.Text.EnCoding.GetEnCoding(rsps.ContentType.Substring(ix + "charset".Length + 1));                    }                    catch                    {                        cding = EnCoding.Default;                    }                                       //该处视情况而定 有的需要解码                    //m_HTML = httpUtility.HTMLDecode(new StreamReader(sm,cding).ReadToEnd());                    m_HTML = new StreamReader(sm,cding).ReadToEnd();                                    }                else                {                  //该处视情况而定 有的需要解码                   //m_HTML = httpUtility.HTMLDecode(new StreamReader(sm,cding).ReadToEnd());                                        m_HTML = new StreamReader(sm,cding).ReadToEnd();                    Regex regex = new Regex("charset=(?<cding>[^=]+)?\"",RegexOptions.IgnoreCase);                    string strcding = regex.Match(m_HTML).Groups["cding"].Value;                    try                    {                        cding = EnCoding.GetEnCoding(strcding);                    }                    catch                    {                        cding = EnCoding.Default;                    }                    byte[] bytes = EnCoding.Default.GetBytes(m_HTML.tochararray());                    m_HTML = cding.GetString(bytes);                    if (m_HTML.Split('?').Length > 100)                    {                        m_HTML = EnCoding.Default.GetString(bytes);                    }                }                m_pagesize = m_HTML.Length;                m_uri = rsps.ResponseUri;                rsps.Close();            }            catch (Exception ex)            {                           }        }        public WebPage(string _url)        {            string uurl = "";            try            {                uurl = Uri.UnescapeDataString(_url);                _url = uurl;            }            catch { };            Init(_url);        }        #endregion    }

调用演示

WebPage webInfo = new WebPage("网址"); webInfo.Context;//不包含HTML标签的所有内容 webInfo.M_HTML;//包含HTML标签的内容

以上是内存溢出(jb51.cc)为你收集整理的全部代码内容，希望文章能够帮你解决所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。

总结

以上是内存溢出为你收集整理的C#通过正则表达式抓取网页信息的类全部内容，希望文章能够帮你解决C#通过正则表达式抓取网页信息的类所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。