python爬取千万淘宝商品的脚本

概述python爬取千万淘宝商品的脚本

下面是内存溢出 jb51.cc 通过网络收集整理的代码片段。

内存溢出小编现在分享给大家，也给大家做个参考。

import timeimport leveldbfrom urllib.parse import quote_plus import reimport Jsonimport itertoolsimport sysimport requestsfrom queue import Queuefrom threading import ThreadURL_BASE = 'http://s.m.taobao.com/search?q={}&n=200&m=API4h5&style=List&page={}'def url_get(url):    # print('GET ' + url)    @R_404_5548@ = dict()    @R_404_5548@['Accept'] = 'text/HTML,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'    @R_404_5548@['Accept-EnCoding'] = 'gzip,deflate,sdch'    @R_404_5548@['Accept-Language'] = 'en-US,en;q=0.8'    @R_404_5548@['Connection'] = 'keep-alive'    @R_404_5548@['DNT'] = '1'    #@R_404_5548@['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/28.0.1500.71 Safari/537.36'    @R_404_5548@['User-Agent'] = 'Mozilla/12.0 (compatible; MSIE 8.0; windows NT)'    return requests.get(url,timeout = 5,@R_404_5548@s = @R_404_5548@).textdef item_thread(cate_queue,db_cate,db_item):    while True:        try:            cate = cate_queue.get()            post_exist = True            try:                state = db_cate.Get(cate.encode('utf-8'))                if state != b'OK': post_exist = False            except:                post_exist = False            if post_exist == True:                print('cate-{}: {} already exists ... Ignore'.format(cate,Title))                continue            db_cate.Put(cate.encode('utf-8'),b'crawling')            for item_page in itertools.count(1):                url = URL_BASE.format(quote_plus(cate),item_page)                for tr in range(5):                    try:                        items_obj = Json.loads(url_get(url))                        break                    except KeyboardInterrupt:                        quit()                    except Exception as e:                        if tr == 4: raise e                if len(items_obj['ListItem']) == 0: break                                        for item in items_obj['ListItem']:                    item_obj = dict(                        _ID = int(item['itemNumID']),name = item['name'],price = float(item['price']),query = cate,category = int(item['category']) if item['category'] != '' else 0,nick = item['nick'],area = item['area'])                    db_item.Put(str(item_obj['_ID']).encode('utf-8'),Json.dumps(item_obj,ensure_ascii = False).encode('utf-8'))                print('Get {} items from {}: {}'.format(len(items_obj['ListItem']),cate,item_page))                if 'nav' in items_obj:                    for na in items_obj['nav']['navCatList']:                        try:                            db_cate.Get(na['name'].encode('utf-8'))                        except:                            db_cate.Put(na['name'].encode('utf-8'),b'waiting')            db_cate.Put(cate.encode('utf-8'),b'OK')            print(cate,'OK')        except KeyboardInterrupt:            break        except Exception as e:            print('An {} exception occured'.format(e))def cate_thread(cate_queue,db_cate):    while True:        try:            for key,value in db_cate.RangeIter():                if value != b'OK':                    print('CateThread: put {} into queue'.format(key.decode('utf-8')))                    cate_queue.put(key.decode('utf-8'))            time.sleep(10)        except KeyboardInterrupt:            break        except Exception as e:            print('CateThread: {}'.format(e))if __name__ == '__main__':    db_cate = leveldb.LevelDB('./taobao-cate')    db_item = leveldb.LevelDB('./taobao-item')    orig_cate = '正装'    try:        db_cate.Get(orig_cate.encode('utf-8'))    except:        db_cate.Put(orig_cate.encode('utf-8'),b'waiting')    cate_queue = Queue(maxsize = 1000)    cate_th = Thread(target = cate_thread,args = (cate_queue,db_cate))    cate_th.start()    item_th = [Thread(target = item_thread,db_item)) for _ in range(5)]    for item_t in item_th:        item_t.start()    cate_th.join()

以上是内存溢出(jb51.cc)为你收集整理的全部代码内容，希望文章能够帮你解决所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。

总结

以上是内存溢出为你收集整理的python爬取千万淘宝商品的脚本全部内容，希望文章能够帮你解决python爬取千万淘宝商品的脚本所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。