update

Author: Germey

Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:alpine
+FROM alpine:3.7
 WORKDIR /app
 RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev \
     openssl-dev libxml2-dev libxslt-dev gcc musl-dev py3-pip && \

kubernetes/templates/proxypool-deployment.yaml

@@ -23,6 +23,14 @@ spec:
             - containerPort: 5555
               protocol: TCP
           imagePullPolicy: {{ .Values.deployment.imagePullPolicy }}
+          livenessProbe:
+            httpGet:
+              path: /random
+              port: 5555
+            initialDelaySeconds: 60
+            periodSeconds: 5
+            failureThreshold: 5
+            timeoutSeconds: 10
           resources:
             {{- toYaml .Values.deployment.resources | nindent 12 }}
           env:

proxypool/crawlers/base.py

@@ -4,23 +4,25 @@
 from proxypool.setting import GET_TIMEOUT
 from fake_headers import Headers
 import time
+
+
 class BaseCrawler(object):
     urls = []
-    
+
     @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000)
     def fetch(self, url, **kwargs):
         try:
             headers = Headers(headers=True).generate()
             kwargs.setdefault('timeout', GET_TIMEOUT)
             kwargs.setdefault('verify', False)
             kwargs.setdefault('headers', headers)
-            response = requests.get(url ,**kwargs)
+            response = requests.get(url, **kwargs)
             if response.status_code == 200:
                 response.encoding = 'utf-8'
                 return response.text
         except requests.ConnectionError:
             return
-    
+
     @logger.catch
     def crawl(self):
         """