python爬虫代码示例分享

概述python爬虫代码示例分享 这篇文章主要介绍了三个Python爬虫项目实例代码，使用了urllib2库，文中示例代码非常详细，对大家的学习或者工作具有一定的参考学习价值，需要的朋友可以参考下。

Python爬虫代码示例分享

一、爬取故事段子：推荐学习：Python视频教程

注：部分代码无法正常运行，但仍有一定的参考价值。

#enCoding=utf-8import urllib2 import re  class neihanba():  def spIDer(self):    '''    爬虫的主调度器    '''    isflow=True#判断是否进行下一页    page=1    while isflow:      url="http://www.neihanpa.com/article/List_5_"+str(page)+".HTML"      HTML=self.load(url)      self.deal(HTML,page)      panduan=raw_input("是否继续(y/n)!")      if panduan=="y":        isflow=True        page+=1      else:        isflow=False  def load(self,url):    '''    针对URL地址进行全部爬去    :param url: URL地址    :return: 返回爬去的内容    '''    header = {      "User-Agent": " Mozilla/5.0 (windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request = urllib2.Request(url, headers=header)    response = urllib2.urlopen(request)    HTML = response.read()    return HTML  def deal(self,HTML,page):    '''    对之前爬去的内容进行正则匹配，匹配出标题和正文内容    :param HTML:之前爬去的内容    :param page: 正在爬去的页码    '''    parrten=re.compile('<li class="picList\d+">(.*?)</li>',re.S)    TitleList=parrten.findall(HTML)    for Title in TitleList:      parrten1=re.compile('<a href="/article/\d+.HTML" rel="external nofollow" >(.*)</a>')      ti1=parrten1.findall(Title)      parrten2=re.compile('<div class="f18 mb20">(.*?)</div>',re.S)      til2=parrten2.findall(Title)      for t in ti1:        tr=t.replace("<b>","").replace("</b>","")        self.writeData(tr,page)      for t in til2:        tr=t.replace("<p>","").replace("</p>","").replace("<br>","").replace("<br />","").replace("&ldquo","\"").replace("&rdquo","\"")        self.writeData(tr,page)  def writeData(self,context,page):    '''    将最终爬去的内容写入文件中    :param context: 匹配好的内容    :param page: 当前爬去的页码数    '''    filename = "di" + str(page) + "yeHTML.txt"    with open(filename, "a") as file:      file.writelines(context + "\n")if __name__ == '__main__':  n=neihanba()  n.spIDer()

二、爬取智联：

#enCoding=utf-8import urllibimport urllib2 import re  class zhilian():  def spIDer(self,position,workPlace):    '''    爬虫的主调度器    :param position: 职位    :param workPlace: 工作地点    '''    url="http://sou.zhaopin.com/jobs/searchresult.ashx?"    url+=urllib.urlencode({"jl":workPlace})    url+="&"    url+=urllib.urlencode({"kw":position})    isflow=True#是否进行下一页的爬去    page=1    while isflow:      url+="&"+str(page)      HTML=self.load(url)      self.deal1(HTML,page)      panduan = raw_input("是否继续爬虫下一页(y/n)!")      if panduan == "y":        isflow = True        page += 1      else:        isflow = False  def load(self,url):    '''    针对URL地址进行全部爬去    :param url: URL地址    :return: 返回爬去的内容    '''    header = {      "User-Agent": " Mozilla/5.0 (windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request = urllib2.Request(url, headers=header)    response = urllib2.urlopen(request)    HTML = response.read()    return HTML  def deal1(self,HTML,page):    '''     对之前爬去的内容进行正则匹配，匹配职位所对应的链接    :param HTML:之前爬去的内容    :param page: 正在爬去的页码    '''    parrten=re.compile('<a\s+style="@R_301_6837@-weight:\s+bold"\s+par="ssIDkey=y&ss=\d+&ff=\d+&sg=\w+&so=\d+"\s+href="(.*?)" rel="external nofollow" target="_blank">.*?</a>',re.S)    til=parrten.findall(HTML)#爬去链接    for t in til:      self.deal2(t,page)  def deal2(self,t,page):    '''    进行二次爬虫，然后在新的页面中对公司、薪资、工作经验进行匹配    :param t: URL地址    :param page: 当前匹配的页数    '''    HTML=self.load(t)#返回二次爬虫的内容    parrten1=re.compile('<a\s+onclick=".*?"\s+href=".*?" rel="external nofollow" \s+target="_blank">(.*?)\s+.*?<img\s+class=".*?"\s+src=".*?"\s+border="\d+"\s+vinfo=".*?"></a>',re.S)    parrten2=re.compile('<li><span>职位月薪：</span><strong>(.*?) <a.*?>.*?</a></strong></li>',re.S)    parrent3=re.compile('<li><span>工作经验：</span><strong>(.*?)</strong></li>',re.S)    til1=parrten1.findall(HTML)    til2=parrten2.findall(HTML)    til3=parrent3.findall(HTML)    str=""    for t in til1:      t=t.replace('<img Title="专属页面" src="//img03.zhaopin.cn/2012/img/jobs/icon.png" border="0" />',"")      str+=t      str+="\t"    for t in til2:      str+=t      str += "\t"    for t in til3:      str+=t    self.writeData(str,page)  def writeData(self,context,page):    '''    将最终爬去的内容写入文件中    :param context: 匹配好的内容     :param page: 当前爬去的页码数    '''    filename = "di" + str(page) + "yeHTML.txt"    with open(filename, "a") as file:      file.writelines(context + "\n")if __name__ == '__main__':  position=raw_input("请输入职位：")  workPlace=raw_input("请输入工作地点：")  z=zhilian()  z.spIDer(position,workPlace)

三、爬取贴吧：

#enCoding=utf-8import urllibimport urllib2 import re  class teiba():  def spIDer(self,name,startPage,endPage):    url="http://tIEba.baIDu.com/f?IE=utf-8&"    url+=urllib.urlencode({"kw":name})    for page in range(startPage,endPage+1):      pn=50*(page-1)      urlFull=url+"&"+urllib.urlencode({"pn":pn})      HTML=self.loadPage(url)      self.dealPage(HTML,page)   def loadPage(self,url):    header={      "User-Agent":" Mozilla/5.0 (windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request=urllib2.Request(url,headers=header)    response=urllib2.urlopen(request)    HTML=response.read()    return HTML  def dealPage(self,HTML,page):    partten=re.compile(r'<a\s+rel="noreferrer"\s+href="/p/\d+" rel="external nofollow" \s+Title=".*?"\s+target="_blank" class="j_th_tit\s+">(.*?)</a>',re.S)    TitleList=partten.findall(HTML)    rstr=r'<span\s+class="topic-tag"\s+data-name=".*?">#(.*?)#</span>'    for Title in TitleList:      Title=re.sub(rstr,"",Title)      self.writePage(Title,page)  def writePage(self,context,page):    filename="di"+str(page)+"yeHTML.txt"    with open(filename,"a") as file:      file.writelines(context+"\n")if __name__ == '__main__':  name=raw_input("请输入贴吧名：")  startPage=raw_input("请输入起始页：")  endPage=raw_input("请输入终止页：")  t=teiba()  t.spIDer(name,int(startPage),int(endPage))

更多相关教程，请关注Python教程栏目。 总结

以上是内存溢出为你收集整理的python爬虫代码示例分享全部内容，希望文章能够帮你解决python爬虫代码示例分享所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。