python入门 爬虫笔记

python入门 爬虫笔记

先将要用到的包导入

import requests
import time
import random
import pandas as pd
import re
import lxml.html

​​​​​​行情中心：国内快捷全面的股票、基金、期货、美股、港股、外汇、黄金、债券行情系统_东方财富网d​​​​​​

到目标网站找到url获取单页的数据并构造headers

def gethtml(cmd,page):
      
    headers = {
        'Accept':'*/*',
        'Accept-Encoding':'gzip, deflate',
        'Accept-Language':'zh-CN,zh;q=0.9',
        'Connection':'keep-alive',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
    }
    url="http://quote.eastmoney.com/center/gridlist.html#hs_a_board"
    r = requests.get(url)
    pat = "data:[(.*?)]"
    data = re.compile(pat,re.S).findall(r.text)
    return data

以上只进行了单页的爬取数据，通过for循环进行翻页 *** 作

def getstock(cmd,page):
    data = gethtml(cmd,page)
    datas = data[0].split('","')
    stocks = []
    for i in range(len(datas)):
            stock = datas[i].replace('"',"").split(",")
            stocks.append(stock)
            return stocks

为获取不同的股票数据，构造一个字典将不同板块对数据进行提取，并以Excel文件保存在本地def main():

    fs = {"上证":"m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23","深圳":"m:1+t:2,m:1+t:23"}
    for i in fs.keys():
        page = 1
        stocks = getstock(fs[i],page)
        while True:
            page +=1
            if gethtml(fs[i],page)!=gethtml(fs[i],page-1):
                stocks.extend(getstock(fs[i],page))
            else:
                break
        df = pd.Dataframe(stocks)
        columns={1:"代码",2:"名称",3:"最新价格",4:"跌涨额",5:"跌涨幅",6:"成交量",7:"成交额",8:"振幅",9:"最高",10:"最低",11:"今开",12:"昨收",13:"量比",14:"时间"}
        df.rename(columns = columns,inplace=True)
        df.to_excel("股票/"+i+".xls")
        print("已保存"+i+".xls")
main()