[关闭]
@StrGlee 2016-10-20T11:24:02.000000Z 字数 1653 阅读 586

pyspider

pyspider

requirements.txt

  1. Flask>=0.10
  2. Jinja2>=2.7
  3. chardet>=2.2
  4. cssselect>=0.9
  5. lxml
  6. pycurl
  7. pyquery
  8. requests>=2.2
  9. tornado>=3.2
  10. mysql-connector-python>=1.2.2
  11. pika>=0.9.14
  12. pymongo>=2.7.2
  13. unittest2>=0.5.1
  14. Flask-Login>=0.2.11
  15. u-msgpack-python>=1.6
  16. click>=3.3
  17. SQLAlchemy>=0.9.7
  18. six>=1.5.0
  19. amqp>=1.3.0,<2.0
  20. redis
  21. kombu
  22. psycopg2
  23. elasticsearch

config.json

  1. {
  2. "taskdb": "mysql+taskdb://username:password@host:port/taskdb",
  3. "projectdb": "mysql+projectdb://username:password@host:port/projectdb",
  4. "resultdb": "mysql+resultdb://username:password@host:port/resultdb",
  5. "message_queue": "amqp://username:password@host:port/%2F",
  6. "webui": {
  7. "username": "some_name",
  8. "password": "some_passwd",
  9. "need-auth": true
  10. }
  11. }

running

  1. # start **only one** scheduler instance
  2. pyspider -c config.json scheduler
  3. # phantomjs
  4. pyspider -c config.json phantomjs
  5. # start fetcher / processor / result_worker instances as many as your needs
  6. pyspider -c config.json --phantomjs-proxy="localhost:25555" fetcher
  7. pyspider -c config.json processor
  8. pyspider -c config.json result_worker
  9. # start webui, set `--scheduler-rpc` if scheduler is not running on the same host as webui
  10. pyspider -c config.json webui

supervisor

  1. [group:pyspider]
  2. programs=pyspider-fetcher,pyspider-processor,pyspider-phantomjs
  3. [program:pyspider-fetcher]
  4. command=/usr/local/bin/pyspider -c /pyspider/config.json fetcher
  5. autorestart=true
  6. user=pyspider
  7. group=pyspider
  8. directory=/pyspider
  9. stopasgroup=true
  10. [program:pyspider-processor]
  11. command=/usr/local/bin/pyspider -c /pyspider/config.json processor
  12. autorestart=true
  13. user=pyspider
  14. group=pyspider
  15. directory=/pyspider
  16. stopasgroup=true
  17. [program:pyspider-phantomjs]
  18. command=/usr/local/bin/pyspider -c /pyspider/config.json phantomjs
  19. autorestart=true
  20. user=pyspider
  21. group=pyspider
  22. directory=/pyspider
  23. stopasgroup=true
添加新批注
在作者公开此批注前,只有你和作者可见。
回复批注