正式环境支持多种方法运行server (#138)

* update dockerfile and build.yaml

* update run app mode in prod

- update readme

* fix alpine python install gevent

* fix #138

Author: staugur

Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.7-alpine AS build
 COPY requirements.txt .
 RUN apk update &&\
-    apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev &&\
+    apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev build-base musl-dev &&\
+    pip install -U pip &&\
     pip install --timeout 30 --user --no-cache-dir --no-warn-script-location -r requirements.txt
 
 FROM python:3.7-alpine

README.md

@@ -38,6 +38,8 @@ cd ProxyPool
 
 安装方法自行搜索即可。
 
+官方 Docker Hub 镜像：[germey/proxypool](https://hub.docker.com/r/germey/proxypool)
+
 ### 常规方式
 
 常规方式要求有 Python 环境、Redis 环境，具体要求如下：
@@ -100,25 +102,20 @@ docker-compose -f build.yml up
 设置 host、port、password，如果 password 为空可以设置为空字符串，示例如下：
 
 ```shell script
-export REDIS_HOST='localhost'
-export REDIS_PORT=6379
-export REDIS_PASSWORD=''
-export REDIS_DB=0
+export PROXYPOOL_REDIS_HOST='localhost'
+export PROXYPOOL_REDIS_PORT=6379
+export PROXYPOOL_REDIS_PASSWORD=''
+export PROXYPOOL_REDIS_DB=0
 ```
 
 或者只设置连接字符串：
 
 ```shell script
-export REDIS_CONNECTION_STRING='redis://[password]@host:port/db'
-```
-
-如果没有密码也要设置为：
-
-```shell script
-export REDIS_CONNECTION_STRING='redis://@host:port/db'
+export PROXYPOOL_REDIS_CONNECTION_STRING='redis://localhost'
 ```
 
-这里连接字符串的格式需要符合 `redis://[password]@host:port/db` 的格式，注意不要遗漏 `@`。
+这里连接字符串的格式需要符合 `redis://[:password@]host[:port][/database]` 的格式，
+中括号参数可以省略，port默认是6379，database默认是0，密码默认为空。
 
 以上两种设置任选其一即可。
 
@@ -233,6 +230,8 @@ get random proxy 116.196.115.209:8080
 
 - APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev
 - APP_DEBUG：调试模式，可以设置 true 或 false，默认 true
+- APP_PROD_METHOD: 正式环境启动应用方式，默认是`gevent`，
+  可选：`tornado`，`meinheld`（分别需要安装tornado或meinheld模块）
 
 ### Redis 连接
 

proxypool/processors/server.py

@@ -1,11 +1,13 @@
 from flask import Flask, g
 from proxypool.storages.redis import RedisClient
-from proxypool.setting import API_HOST, API_PORT, API_THREADED
+from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV
 
 
 __all__ = ['app']
 
 app = Flask(__name__)
+if IS_DEV:
+    app.debug = True
 
 
 def get_conn():
@@ -46,8 +48,9 @@ def get_proxy_all():
     conn = get_conn()
     proxies = conn.all()
     proxies_string = ''
-    for proxy in proxies:
-        proxies_string += str(proxy) + '\n'
+    if proxies:
+        for proxy in proxies:
+            proxies_string += str(proxy) + '\n'
 
     return proxies_string
 

proxypool/scheduler.py

@@ -3,7 +3,8 @@
 from proxypool.processors.server import app
 from proxypool.processors.getter import Getter
 from proxypool.processors.tester import Tester
-from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, API_THREADED, API_PORT, ENABLE_SERVER, \
+from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, \
+    API_THREADED, API_PORT, ENABLE_SERVER, IS_PROD, APP_PROD_METHOD, \
     ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS
 from loguru import logger
 
@@ -56,7 +57,42 @@ def run_server(self):
         if not ENABLE_SERVER:
             logger.info('server not enabled, exit')
             return
-        app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
+        if IS_PROD:
+            if APP_PROD_METHOD == 'gevent':
+                try:
+                    from gevent.pywsgi import WSGIServer
+                except ImportError as e:
+                    logger.exception(e)
+                else:
+                    http_server = WSGIServer((API_HOST, API_PORT), app)
+                    http_server.serve_forever()
+
+            elif APP_PROD_METHOD == 'tornado':
+                try:
+                    from tornado.wsgi import WSGIContainer
+                    from tornado.httpserver import HTTPServer
+                    from tornado.ioloop import IOLoop
+                except ImportError as e:
+                    logger.exception(e)
+                else:
+                    http_server = HTTPServer(WSGIContainer(app))
+                    http_server.listen(API_PORT)
+                    IOLoop.instance().start()
+
+            elif APP_PROD_METHOD == "meinheld":
+                try:
+                    import meinheld
+                except ImportError as e:
+                    logger.exception(e)
+                else:
+                    meinheld.listen((API_HOST, API_PORT))
+                    meinheld.run(app)
+
+            else:
+                logger.error("unsupported APP_PROD_METHOD")
+                return
+        else:
+            app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
 
     def run(self):
         global tester_process, getter_process, server_process
@@ -71,13 +107,13 @@ def run(self):
             if ENABLE_GETTER:
                 getter_process = multiprocessing.Process(
                     target=self.run_getter)
-                logger.info(f'starting getter, pid{getter_process.pid}...')
+                logger.info(f'starting getter, pid {getter_process.pid}...')
                 getter_process.start()
 
             if ENABLE_SERVER:
                 server_process = multiprocessing.Process(
                     target=self.run_server)
-                logger.info(f'starting server, pid{server_process.pid}...')
+                logger.info(f'starting server, pid {server_process.pid}...')
                 server_process.start()
 
             tester_process and tester_process.join()

proxypool/setting.py

@@ -22,6 +22,12 @@
 APP_PROD = IS_PROD = APP_ENV == PROD_MODE
 APP_TEST = IS_TEST = APP_ENV == TEST_MODE
 
+# Which WSGI container is used to run applications
+# - gevent: pip install gevent
+# - tornado: pip install tornado
+# - meinheld: pip install meinheld
+APP_PROD_METHOD = env.str('APP_PROD_METHOD', "gevent").lower()
+
 # redis host
 REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST',
                      env.str('REDIS_HOST', '127.0.0.1'))

requirements.txt

@@ -11,3 +11,6 @@ redis==3.5.3
 lxml==4.6.5
 fake_headers==1.0.2
 maxminddb_geolite2==2018.703
+gevent>=21.1.0
+tornado>=6.0
+meinheld>=1.0.0