python爬虫爬取网站小说

概述#加载模块importrequestsfrombs4importBeautifulSoup#定义所有章节和链接函数defget_novel_chapters():url="https://www.89wxw.comead/1037/"r=requests.get(url)main_page=BeautifulSoup(r.text,"html.parser")list1=[]

#加载模块import requestsfrom bs4 import BeautifulSoup# 定义所有章节和链接函数def get_novel_chapters():    url = "https://www.89wxw.com/read/1037/"    r = requests.get(url)    main_page = BeautifulSoup(r.text, "HTML.parser")    List1 = []    for dd in (main_page.find_all("dd")):        link = dd.find("a")        if not link:            continue        List1.append(("https://www.89wxw.com" + (link["href"]), (link.get_text())))    return List1# 定义获取所有章节内容def get_chapters_content(url):    r = requests.get(url)    main_page = BeautifulSoup(r.text, "HTML.parser")    data = main_page.find("div", ID="content").get_text()    return data#循环调用两个自定义函数将鸟啼和内容写入文本文档j = 0f = open("都市超级医圣.txt", "w")aList = get_novel_chapters()for i in aList:    j+=1    url,Title=i    print("循环爬取第{}次，写入文本文档，标题是：{}".format(j,Title))    f.write(Title+"\n"+get_chapters_content(url))f.close()

总结

以上是内存溢出为你收集整理的python爬虫爬取网站小说全部内容，希望文章能够帮你解决python爬虫爬取网站小说所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。