Python搜索引擎安装教程

运行环境
开始之前你需要安装

•PHP 5.3.7 +
•MySQL
•Python 2.7 ~
•xunsearch 搜索引擎
获取源码
ssh 方式:

git clone git@github.com:k1995/BaiduyunSpider.git
https 方式:

git clone https://github.com/k1995/BaiduyunSpider
或手动下载

https://github.com/k1995/BaiduyunSpider/archive/master.zip
下载完毕后，项目的目录结构大致是这样的

— indexer/  #索引
— spider/   #爬虫
— sql/
— web/      #网站
— application/
— config/ # 配置相关
— config.php
— database.php # 数据库配置
…
…
— static/ # 存放静态资源，css|js|font
— system/
— index.php
…
开始部署
创建数据库
创建名为pan的数据库，编码设为utf-8。然后导入sql，完成表的创建。

网站部署
支持nginx，apache 服务器。

apache 需要开启 mod_rewrite 。

nginx  配置如下

location /
{
index index.php;
try_files $uri $uri/ /index.php/$uri;
}

location ~ [^/]\.php(/|$)
{
fastcgi_pass  127.0.0.1:9000;
fastcgi_index index.php;
include fastcgi.conf;
include pathinfo.conf;
}
配置文件修改
config.php 文件修改网站标题，描述等信息

database.php 修改数据库账号，密码等信息

网站是基于CodeIgniter 框架开发的，如安装，部署，或二次开发有问题，请参考官网文档
启动爬虫
进入 spider/目录，修改spider.py 中数据库信息。

如果你是第一次部署，需运行下面命令，完成做种

python spider.py –seed-user
上面其实就是抓取百度云热门分享用户的相关信息，然后从他们开始入手爬取数据

然后运行

python spider.py
此时爬虫已经开始工作了

安装xunsearch
目前使用xunsearch作为搜索引擎，后面会更换为elasticsearch。

安装过程请参考（不需要安装，PHP SDK，我已经整合到web里了）

http://xunsearch.com/doc/php/guide/start.installation

索引数据
上面我们完成了爬虫的数据抓取，网站的搭建，但还不能搜索，下面开始最后一步，索引的建立。

进入 indexer/目录，在indexer.php中将$prefix，替换为你web的根路径

require ‘$prefix/application/helpers/xs/lib/XS.php’;
并修改数据库账号密码

然后运行

./indexer.php

 

1. # -*- coding: utf-8 -*-
2. import urllib2,re,argparse,json,time
3. import MySQLdb as mdb
4. import metautils,traceback,Queue,socket
5. import random
6. “””
7. /*
8.  *——————————————–   
9.  *
10.  *  
11.  *    
12.  *  Github 仓库: https://github.com/k1995/BaiduyunSpider
13.  * 
14.  *  演示：http://www.11bt.net/ *
15.  *   
16.  * —————————————-*/
17. “””
19. DB_HOST=‘127.0.0.1’
20. DB_PORT=‘3306’
21. DB_USER=‘root’
22. # MySQL密码
23. DB_PASS=‘123123’
24. # 数据库名称
25. DB_NAME=‘pan’
26. SPIDER_INTERVAL=1
28. ERR_NO=0#正常
29. ERR_REFUSE=1#爬虫爬取速度过快，被拒绝
30. ERR_EX=2#未知错误
33. proxy_list = [
34.                {‘http’:“x.x.x.x:8080”},
35.                {‘http’:“x.x.x.x:8081”},
36.                {‘http’:“x.x.x.x:8082”},
37.                {‘http’:“x.x.x.x:8083”},
38.                {‘http’:“x.x.x.x:8084”},
39.                {‘http’:“x.x.x.x:8085”},
40.                {‘http’:“x.x.x.x:8086”},
41.                {‘http’:“x.x.x.x:8087”},
42.                {‘http’:“x.x.x.x:8088”},
43.                {‘http’:“x.x.x.x:8089”}
44.                 ]
49. def getHtml(url,ref=None,reget=5):
50.  try:
52.   uas = [
53.     “Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9b4) Gecko/2008030317 Firefox/3.0b4”,
54.     “Mozilla/5.0 (Windows; U; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; BIDUBrowser 7.6)”,
55.     “Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko”,
56.     “Mozilla/5.0 (Windows NT 6.3; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0”,
57.     “Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.99 Safari/537.36”,
58.     “Mozilla/5.0 (Windows NT 6.3; Win64; x64; Trident/7.0; Touch; LCJB; rv:11.0) like Gecko”,
59.     ]
60.   proxy_ip =random.choice(proxy_list)
62.   ua=random.choice(uas)
64.   print proxy_ip
65.   print ua
66.   proxy_support = urllib2.ProxyHandler(proxy_ip)
67.   opener = urllib2.build_opener(proxy_support,urllib2.HTTPHandler)
68.   urllib2.install_opener(opener)
70.   request = urllib2.Request(url)
71.   time.sleep(5)
72.   request.add_header(‘User-Agent’, ua)
73.   if ref:
74.    request.add_header(‘Referer’,ref)
75.   page = urllib2.urlopen(request,timeout=30)
76.   html = page.read()
77.  except:
78.   if reget>=1:
79.    #如果getHtml失败，则再次尝试5次
81.    print ‘getHtml error,reget…%d’%(6-reget)
82.    time.sleep(20)
83.    return getHtml(url,ref,reget-1)
84.   else:
85.    print ‘request url:’+url
86.    print ‘failed to fetch html’
87.    exit()
88.  else:
89.   return html
91. class Db(object):
92.  def __init__(self):
93.   self.dbconn=None
94.   self.dbcurr=None
96.  def check_conn(self):
97.   try:
98.    self.dbconn.ping()
99.   except:
100.    return False
101.   else:
102.    return True
104.  def conn(self):
105.   self.dbconn=mdb.connect(DB_HOST, DB_USER, DB_PASS,DB_NAME, charset=‘utf8’)
106.   self.dbconn.autocommit(False)
107.   self.dbcurr = self.dbconn.cursor()
109.  def fetchone(self):
110.   return self.dbcurr.fetchone()
112.  def fetchall(self):
113.   return self.dbcurr.fetchall()
115.  def execute(self, sql, args=None,falg=False):
116.   if not self.dbconn:
117.    #第一次链接数据库
118.    self.conn()
119.   try:
120.    if args:
121.     rs=self.dbcurr.execute(sql,args)
122.    else:
123.     rs=self.dbcurr.execute(sql)
124.    return rs
125.   except Exception, e:
126.    if self.check_conn():
127.     print ‘execute error’
128.     traceback.print_exc()
129.    else:
130.     print ‘reconnect mysql’
131.     self.conn()
132.           if args:
133.               rs=self.dbcurr.execute(sql,args)
134.           else:
135.               rs=self.dbcurr.execute(sql)
136.           return rs
138.  def commit(self):
139.   self.dbconn.commit()
141.  def rollback(self):
142.   self.dbconn.rollback()
144.  def close(self):
145.   self.dbconn.close()
146.   self.dbcurr.close()
147.  def last_row_id(self):
148.   return self.dbcurr.lastrowid
150. class BaiduPanSpider(object):
151.  def __init__(self):
152.   self.db=Db()
153.   self.files=[]
154.   self.got_files_count=0
155.   self.got_follow_count=0
156.   self.while_count=0
157.   self.spider_queue=Queue.Queue(maxsize=20)
158.   self.status=‘stop’
159.   self.errno=ERR_NO
160.   self.file_type_t={‘video’:0,‘image’:1,‘document’:2,‘music’:3,‘package’:4,‘software’:5,‘torrent’:6,‘other’:-1}
162.  def getShareUser(self,uk):
163.   url=‘http://pan.baidu.com/share/count?uk=%d&channel=chunlei&clienttype=0&web=1’%uk
164.   follows_json=json.loads(getHtml(url,uk))
165.   if follows_json[‘errno’]!=0:
166.    if follows_json[‘errno’]==-55:
167.     self.errno=ERR_REFUSE
168.    else:
169.     self.errno=ERR_EX
170.    return False
171.   return {
172.    ‘pubshare_cnt’:follows_json[‘pubshare_cnt’],
173.    ‘fans’:follows_json[‘fans’],
174.    ‘follow’:follows_json[‘follow’],
175.    ‘album’:follows_json[‘follows_json’]
176.   }
178.  def getHotUser(self):
179.   url=‘http://pan.baidu.com/pcloud/friend/gethotuserlist?type=1&from=feed&start=0&limit=24&channel=chunlei&clienttype=0&web=1’
180.   follows_json=json.loads(getHtml(url))
181.   if follows_json[‘errno’]!=0:
182.    print u‘failed to fetch hot users’
183.    return False
184.   returns=[]
185.   count=0
187.   for item in follows_json[‘hotuser_list’]:
188.    count=count+1
189.    hot_uname=item[‘hot_uname’].encode(‘utf-8’)
190.    hot_uk=item[‘hot_uk’]
191.    avatar_url=item[‘avatar_url’].encode(‘utf-8’)
192.    intro=item[‘intro’].encode(‘utf-8’)
193.    follow_count=item[‘follow_count’]
194.    fans_count=item[‘fans_count’]
195.    pubshare_count=item[‘pubshare_count’]
196.    album_count=item[‘album_count’]
197.    returns.append({‘hot_uname’:hot_uname,‘hot_uk’:hot_uk,‘avatar_url’:avatar_url,‘intro’:intro,‘follow_count’:follow_count,‘fans_count’:fans_count,‘pubshare_count’:pubshare_count,‘album_count’:album_count})
199.   if count==0:
200.    print “got no hot users”
201.    return False
202.   else:
203.    print “success to fetched hot users: %d”%count
204.   return returns
206.  def getFans(self,uk,start=0,limit=24):
207.   #query_uk:用户ID
208.   #limit:每一页最多显示数量
209.   #start:当前页数
210.   follows_url=‘http://pan.baidu.com/pcloud/friend/getfanslist?query_uk=%d&limit=%d&start=%d’%(uk,limit,start)
211.   follows_json=json.loads(getHtml(follows_url,uk))
212.   if follows_json[‘errno’]!=0:
213.    print u‘failed to fetch fens’
214.    return False
215.   total_count=follows_json[‘total_count’]
216.   returns=[]
217.   count=0
219.   for item in follows_json[‘fans_list’]:
220.    count=count+1
221.    fans_uname=item[‘fans_uname’].encode(‘utf-8’)
222.    fans_uk=item[‘fans_uk’]
223.    avatar_url=item[‘avatar_url’].encode(‘utf-8’)
224.    intro=item[‘intro’].encode(‘utf-8’)
225.    follow_count=item[‘follow_count’]
226.    fans_count=item[‘fans_count’]
227.    pubshare_count=item[‘pubshare_count’]
228.    album_count=item[‘album_count’]
229.    returns.append({‘fans_uname’:fans_uname,‘fans_uk’:fans_uk,‘avatar_url’:avatar_url,‘intro’:intro,‘follow_count’:follow_count,‘fans_count’:fans_count,‘pubshare_count’:pubshare_count,‘album_count’:album_count})
231.   return (total_count,count,returns)
233.  def getFollows(self,uk,start=0,limit=24):
234.   follows_url=‘http://pan.baidu.com/pcloud/friend/getfollowlist?query_uk=%d&limit=%d&start=%d&bdstoken=d82467db8b1f5741daf1d965d1509181&channel=chunlei&clienttype=0&web=1’%(uk,limit,start)
235.   ref=‘http://pan.baidu.com/pcloud/friendpage?type=follow&uk=%d&self=1’%uk
236.   follows_json=json.loads(getHtml(follows_url,ref))
237.   if follows_json[‘errno’]!=0:
238.    print ‘getFollows errno:%d’%follows_json[‘errno’]
239.    print ‘request_url:’+follows_url
240.    if follows_json[‘errno’]==-55:
241.     self.errno=ERR_REFUSE
242.    else:
243.     self.errno=ERR_EX
244.    return False
245.   total_count=follows_json[‘total_count’]
246.   returns=[]
247.   count=0
248.   if(total_count>0):
249.    for item in follows_json[‘follow_list’]:
250.     count=count+1
251.     returns.append({
252.      ‘follow_uname’:item[‘follow_uname’].encode(‘utf-8’),
253.      ‘follow_uk’:item[‘follow_uk’],
254.      ‘avatar_url’:item[‘avatar_url’].encode(‘utf-8’),
255.      ‘intro’:item[‘intro’].encode(‘utf-8’),
256.      ‘follow_count’:item[‘follow_count’],
257.      ‘fans_count’:item[‘fans_count’],
258.      ‘pubshare_count’:item[‘pubshare_count’],
259.      ‘album_count’:item[‘album_count’]
260.     })
262.   return (total_count,count,returns)
264.  def getShareLists(self,uk,start=0,limit=60):
265.   sharelists_url=‘http://pan.baidu.com/pcloud/feed/getsharelist?category=0&auth_type=1&request_location=share_home&start=%d&limit=%d&query_uk=%d&channel=chunlei&clienttype=0&web=1’%(start,limit,uk)
266.   ref=‘http://pan.baidu.com/share/home?uk=%d&view=share’%uk
267.   listhtm=getHtml(sharelists_url,ref)
268.   print(sharelists_url)
269.   sharelists_json=json.loads(listhtm)
270.   if(sharelists_json[‘errno’]!=0):
271.    print ‘getShareLists errno:%d’%sharelists_json[‘errno’]
272.    print ‘request_url:’+sharelists_url
273.    if sharelists_json[‘errno’]==-55:
274.     self.errno=ERR_REFUSE
275.    else:
276.     self.errno=ERR_EX
277.    return False
278.   total_count=sharelists_json[‘total_count’]
279.   returns=[]
280.   count=0
281.   if total_count>0:
282.    for item in sharelists_json[‘records’]:
283.     count=count+1
284.     feed_type=item[‘feed_type’]
285.     isdir=0
286.     size=0
287.     md5=”
288.     album_id=”
289.     shorturl=”
290.     if feed_type==‘share’:
291.      if item[‘filecount’]==1:
292.       filelist=item[‘filelist’]
293.       isdir=filelist[0][‘isdir’]
294.       size=filelist[0][‘size’]
295.       md5=filelist[0][‘md5’]
296.      else:
297.       isdir=1
298.     elif feed_type==‘album’:
299.      album_id=item[‘album_id’]
300.      isdir=2
302.     if item.has_key(‘shorturl’):
303.      shorturl=item[‘shorturl’]
304.     if item.has_key(‘username’):
305.      username=item[‘username’].encode(‘utf-8’)
306.     if feed_type==‘share’ or feed_type==‘album’:
307.      returns.append({
308.       ‘title’:item[‘title’].encode(‘utf-8’),
309.       ‘username’:username,
310.       ‘shorturl’:shorturl,
311.       ‘shareid’:item[‘source_id’],
312.       ‘feed_time’:item[‘feed_time’]//1000,#分享时间
313.       ‘dCnt’:item[‘dCnt’],
314.       ‘isdir’:isdir,
315.       ‘size’:size,
316.       ‘md5’:md5,
317.       ‘uk’:uk,
318.       ‘feed_type’:feed_type
319.      })
320.   return (total_count,count,returns)
322.  def getAlbum(self,uk,start=0,limit=60):
323.   url=‘http://pan.baidu.com/pcloud/album/getlist?start=%d&limit=%d&query_uk=%d&channel=chunlei&clienttype=0&web=1&bdstoken=d82467db8b1f5741daf1d965d1509181’%(start,limit,uk)
324.   album_json=json.loads(getHtml(url,uk))
325.   total_count=album_json[‘count’]
326.   returns=[]
327.   count=0
329.   for item in album_json[‘album_list’]:
330.    count=count+1
331.    title=item[‘title’].encode(‘utf-8’)
332.    album_id=item[‘album_id’]
333.    create_time=item[‘create_time’]
334.    update_time=item[‘update_time’]
335.    filecount=item[‘filecount’]
336.    desc=item[‘desc’]
337.    returns.append({‘title’:title,‘album_id’:album_id,‘create_time’:create_time,‘desc’:desc,‘update_time’:update_time,‘filecount’:filecount,‘uk’:uk})
339.   if count==0:
340.    print “get nothing”
341.    return False
342.   else:
343.    print “success to fetched : %d”%count
345.   if (start+count)<total_count:
346.    start=start+limit
347.    returns=returns+self.getAlbum(uk,start)
348.   return returns
350.  def seedUsers(self):
351.   hot_usrs=self.getHotUser()
352.   if not hot_usrs:
353.    return
354.   try:
355.    for user in hot_usrs:
356.     time_stamp=int(time.time())
357.     if user[‘pubshare_count’]>0:
358.      self.db.execute(“INSERT INTO share_users (uk,user_name,avatar_url,intro,follow_count,album_count,\
359.       fens_count,pubshare_count,last_visited,create_time,weight) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)”,(
360.        user[‘hot_uk’],user[‘hot_uname’],user[‘avatar_url’],user[‘intro’],user[‘follow_count’],
361.        user[‘album_count’],user[‘fans_count’],user[‘pubshare_count’],time_stamp,time_stamp,5
362.       )
363.      )
364.      uid=self.db.last_row_id()
365.      self.db.execute(“INSERT INTO spider_list (uk,uid) VALUES(%s,%s)”,(user[‘hot_uk’],uid))
366.   except:
367.    traceback.print_exc()
368.    self.db.rollback()
369.   else:
370.    self.db.commit()
372.  def startSpider(self):
373.   if self.spider_queue.empty():
374.    fetched_users=self.db.execute(‘SELECT * from spider_list ORDER BY weight DESC limit 0,20’)
375.    if fetched_users<=0:
376.     print ‘nothing to spider,spider_list is empty’
377.     return False
378.    self.start=‘start’
379.    self.errno=ERR_NO
380.    fetchall=self.db.fetchall()
381.    #将数据库中取出的待爬取的分享者，加入爬取队列
382.    for item in fetchall:
383.     self.spider_queue.put({
384.     ‘sid’:item[0],
385.     ‘uk’:item[1],
386.     ‘file_fetched’:item[2],
387.     ‘follow_fetched’:item[3],
388.     ‘follow_done’:item[4],
389.     ‘file_done’:item[5],
390.     ‘weight’:item[6],
391.     ‘uid’:item[7]
392.    })
393.    self.got_follow_count=0
394.    self.got_files_count=0
395.    self.while_count=0
397.   while not self.spider_queue.empty():
398.    self.while_count+=1
399.    share_user=self.spider_queue.get()
400.    #爬取分享者的文件列表
401.    if not share_user[‘file_done’]:
402.     print ‘%d now spidering file ,%d  file fetched’%(share_user[‘uk’],share_user[‘file_fetched’])
403.     rs=self.getShareLists(share_user[‘uk’],share_user[‘file_fetched’])
404.     #print(rs)
405.     if not rs:
406.      print ‘uk:%d error to fetch files,try again later…’%share_user[‘uk’]
407.      return True
408.     total_count,fetched_count,file_list=rs
409.     total_fetched=share_user[‘file_fetched’]+fetched_count
410.     print ‘fetched_file_count:%d’%fetched_count
411.     if total_fetched>=total_count or total_count==0:
412.      share_user[‘file_done’]=1#该分享者所有文件爬取完成
413.     if total_count==0:
414.      self.db.execute(“UPDATE spider_list set file_done=%s WHERE sid=%s”,(1,share_user[‘sid’]))
415.      self.db.commit()
416.     else:
417.      try:
418.       files_count=0
419.       for file in file_list:
420.        files_count+=1
421.        ext=”
422.        file_type=”
423.        file_type_i=-1
424.        if file[‘isdir’]==0 and file[‘feed_type’]==‘share’:
425.         ext = metautils.get_extension(file[‘title’]).lower()
426.         file_type = metautils.get_category(ext)
427.         file_type_i=self.file_type_t[file_type]
428.        time_stamp=int(time.time())
429.        self.db.execute(“INSERT INTO share_file (title,uk,user_name,shareid,shorturl,isdir,size,md5,ext,feed_time,create_time,file_type,uid,feed_type) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)”,(file[‘title’],file[‘uk’],file[‘username’],file[‘shareid’],         file[‘shorturl’],file[‘isdir’],file[‘size’],file[‘md5’],ext,file[‘feed_time’],time_stamp,file_type_i,share_user[‘uid’],file[‘feed_type’])
430.        )
431.      except:
432.       share_user[‘file_done’]=0
433.       self.db.rollback()
434.       traceback.print_exc()
435.       return False
436.      else:
437.       self.db.execute(“UPDATE spider_list set file_fetched=%s,file_done=%s WHERE sid=%s”,(total_fetched,share_user[‘file_done’],share_user[‘sid’]))
438.       self.db.execute(“UPDATE share_users set fetched=%s WHERE uid=%s”,(total_fetched,share_user[‘uid’]))
439.       share_user[‘file_fetched’]=total_fetched
440.       self.got_files_count+=files_count
441.       self.db.commit()
443.    #爬取完文件后在爬取订阅列表
444.    if share_user[‘follow_done’]==0 and share_user[‘file_done’]==1:
445.     print ‘%d now spidering follow ,%d  follow fetched’%(share_user[‘uk’],share_user[‘follow_fetched’])
446.     rs=self.getFollows(share_user[‘uk’],share_user[‘follow_fetched’])
447.     if not rs:
448.      print ‘error to fetch follows,try again later…’
449.      return
450.     total_count,fetched_count,follow_list=rs
451.     total_fetched=share_user[‘follow_fetched’]+fetched_count
452.     print ‘fetched_follow_count:%d’%fetched_count
453.     if total_fetched>=total_count or total_count==0:
454.      share_user[‘follow_done’]=1
455.     if total_count==0:
456.      self.db.execute(“DELETE FROM spider_list WHERE sid=%s”,(share_user[‘sid’],))
457.      self.db.commit()
458.     else:
459.      try:
460.       follow_count=0
461.       for follow in follow_list:
462.        follow_count+=1
463.        #判断该用户是否已经在表中了
464.        if self.db.execute(‘SELECT * FROM share_users WHERE uk=%s’,(follow[‘follow_uk’],))>0:
465.         print ‘uk:%d has already in share_user table’%follow[‘follow_uk’]
466.         continue
467.        time_stamp=int(time.time())
468.        self.db.execute(“INSERT INTO share_users (uk,user_name,avatar_url,intro,follow_count,album_count,\
469.         fens_count,pubshare_count,last_visited,create_time,weight) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)”,(
470.          follow[‘follow_uk’],follow[‘follow_uname’],follow[‘avatar_url’],follow[‘intro’],follow[‘follow_count’],
471.          follow[‘album_count’],follow[‘fans_count’],follow[‘pubshare_count’],time_stamp,time_stamp,5
472.         )
473.        )
474.        #将获取的新分享者加入爬取列表
475.        self.db.execute(“INSERT INTO spider_list (uk,uid) VALUES(%s,%s)”,(follow[‘follow_uk’],self.db.last_row_id()))
476.      except:
477.       share_user[‘follow_done’]=0
478.       self.db.rollback()
479.       traceback.print_exc()
480.       return False
481.      else:
482.       if share_user[‘follow_done’]==1:
483.        #订阅者爬取完成，该分享者的任务完成，从待爬取列表中删除
484.        print ‘delete follow fetched sid:%d from spider_list’%share_user[‘sid’]
485.        self.db.execute(“DELETE FROM spider_list WHERE sid=%s”,(share_user[‘sid’],))
486.       else:
487.        self.db.execute(“UPDATE spider_list set follow_fetched=%s,follow_done=%s WHERE sid=%s”,(total_fetched,share_user[‘follow_done’],share_user[‘sid’]))
488.       share_user[‘follow_fetched’]=total_fetched
489.       self.got_follow_count+=follow_count
490.       self.db.commit()
491.    #只要分享者列表没完成，说明该分享者还未爬取完，则加入工作队列，继续爬取
492.    if share_user[‘follow_done’]==0:
493.     self.spider_queue.put(share_user)
494.    else:
495.     print ‘%d has done’%share_user[‘uk’]
496.     del share_user
497.    time.sleep(SPIDER_INTERVAL)
499.   print ‘—————–Done——————‘
500.   print ‘while_count:%d’%self.while_count
501.   print ‘got_follow_count:%d’%self.got_follow_count
502.   print ‘got_files_count:%d’%self.got_files_count
503.   return True
505.  def stop(self):
506.   pass
508. if __name__ == “__main__”:
509.  parser = argparse.ArgumentParser()
510.  parser.add_argument(“–seed-user”, help=“get seed user”, action=“store_true”)
511.  args = parser.parse_args()
513.  spider=BaiduPanSpider()
515.  # 做种
516.  if args.seed_user:
517.   spider.seedUsers()
518.  else:
519.   while(1):
520.    print ‘start spider…’
521.    result=spider.startSpider()
522.    if not result:
523.     print ‘The spider is refused,5 mins later try again auto…’
524.     time.sleep(60*5)
525.    else:
526.     print ‘one worker queue id done’
527.     time.sleep(1)