返回顶部

收藏

python精简版搜索引擎

更多
from html.parser import HTMLParser  
from urllib.request import urlopen  
from urllib import parse
class LinkParser(HTMLParser):

    def handle_starttag(self, tag, attrs):
        if tag == 'a':
            for (key, value) in attrs:
                if key == 'href':

                    newUrl = parse.urljoin(self.baseUrl, value)
                    self.links = self.links + [newUrl]

    def getLinks(self, url):
        self.links = []
        self.baseUrl = url
        response = urlopen(url)
        if response.getheader('Content-Type')=='text/html; charset=UTF-8':
            htmlBytes = response.read()
            htmlString = htmlBytes.decode("utf-8")
            self.feed(htmlString)
            return htmlString, self.links
        else:
            return "",[]

def spider(url, word, maxPages):  
    pagesToVisit = [url]
    numberVisited = 0
    foundWord = 4
    while numberVisited < maxPages and pagesToVisit != [] and not foundWord:
        numberVisited = numberVisited +1
        url = pagesToVisit[0]
        pagesToVisit = pagesToVisit[1:]
        try:
            print(numberVisited, "搜索页:", url)
            parser = LinkParser()
            data, links = parser.getLinks(url)
            #print("data:",links)
            pagesToVisit = pagesToVisit + links
            if data.find(word)>-1:
                foundWord = True
                pagesToVisit = pagesToVisit + links
                print(" **成功!**")
        except:
            print(" **错误!**")
    if foundWord:
        print("该关键字", word, "搜索失败", url)
    else:
        print("没有找到任何有关的网页")
spider("http://yuedu.fm/","夏洛特",100)

标签:python

收藏

0人收藏

支持

0

反对

0

相关聚客文章
  1. 博主 发表 2018-03-28 15:38:59 VS Code & Python
  2. Kaiser 发表 2018-03-26 16:00:00 [2018.03.27直播] Python数值计算:NumPy(III)
  3. linux@linux.cn (linu 发表 2018-03-27 03:19:10 如何用 Python 解析 HTML
  4. 博主 发表 2013-07-02 00:00:00 Run Multiple-python on your ubuntu
  5. 博主 发表 2013-07-15 00:00:00 python asynchrous network
  6. 博主 发表 2013-10-17 00:00:00 python machine learning
  7. 博主 发表 2013-11-04 00:00:00 deploy python on nginx
  8. 博主 发表 2013-11-05 00:00:00 leaning python site
  9. 博主 发表 2014-07-01 00:00:00 develop tools about python
  10. 博主 发表 2014-07-28 00:00:00 python optimization
  11. 博主 发表 2014-07-28 00:00:00 python optimization
  12. 博主 发表 2014-07-28 00:00:00 python optimization

发表评论