add prometheus monitor

Author: ResolveWang

haipproxy/config/settings.py

@@ -60,7 +60,7 @@
 REDIS_HOST = '127.0.0.1'
 REDIS_PORT = 6379
 REDIS_PASSWORD = '123456'
-REDIS_DB = 3
+REDIS_DB = 0
 
 # scheduler settings
 TIMER_RECORDER = 'haipproxy:scheduler:task'
@@ -142,11 +142,10 @@
 # https://github.com/getsentry/sentry
 # disable it by setting USE_SENTRY = False
 USE_SENTRY = True
-SENTRY_DSN = 'http://009f7f5f50794deeb24791a39b86f254:56a9a5f5d3fc42a18bf87aa4341f8f3f@127.0.0.1:9000/2'
+SENTRY_DSN = 'http://82c130028fa942f29add1e0aa0ff9cbd:cffa174304d248b9aa2bdb385d3b01b8@127.0.0.1:9000/6'
 
 # prometheus for monitoring, for more information see
 # https://github.com/prometheus/prometheus
-# disable it by setting use_prom = False
-USE_PROM = False
+# you have to config prometheus first if you want to monitor haipproxy status
 EXPORTER_LISTEN_HOST = '0.0.0.0'
-EXPORTER_LISTEN_PORT = 8000
+EXPORTER_LISTEN_PORT = 7001

haipproxy/monitor/__init__.py

@@ -0,0 +1 @@
+from .exporter import exporter_start

haipproxy/monitor/exporter.py

@@ -0,0 +1,63 @@
+import time
+
+from prometheus_client import start_http_server
+from prometheus_client.core import (
+    CounterMetricFamily, GaugeMetricFamily,
+    REGISTRY
+)
+
+from haipproxy.config.settings import (
+    DATA_ALL, INIT_HTTP_QUEUE,
+    SPIDER_COMMON_TASK, SPIDER_AJAX_TASK,
+    SPIDER_GFW_TASK, SPIDER_AJAX_GFW_TASK,
+    TEMP_ZHIHU_QUEUE, VALIDATED_ZHIHU_QUEUE,
+    TTL_ZHIHU_QUEUE, SPEED_ZHIHU_QUEUE,
+    TTL_VALIDATED_RESOURCE, LOWEST_SCORE,
+    LONGEST_RESPONSE_TIME, EXPORTER_LISTEN_HOST,
+    EXPORTER_LISTEN_PORT,)
+from haipproxy.utils import get_redis_conn
+
+
+class CustomCollector:
+    def __init__(self):
+        self.conn = get_redis_conn()
+
+    def collect(self):
+        start_time = int(time.time()) - TTL_VALIDATED_RESOURCE * 60
+
+        pipe = self.conn.pipeline(False)
+        pipe.scard(DATA_ALL)
+        pipe.llen(INIT_HTTP_QUEUE)
+        pipe.scard(TEMP_ZHIHU_QUEUE)
+        pipe.zrevrangebyscore(VALIDATED_ZHIHU_QUEUE, '+inf', LOWEST_SCORE)
+        pipe.zrevrangebyscore(TTL_ZHIHU_QUEUE, '+inf', start_time)
+        pipe.zrangebyscore(SPEED_ZHIHU_QUEUE, 0, 1000*LONGEST_RESPONSE_TIME)
+        pipe.llen(SPIDER_COMMON_TASK)
+        pipe.llen(SPIDER_AJAX_TASK)
+        pipe.llen(SPIDER_GFW_TASK)
+        pipe.llen(SPIDER_AJAX_GFW_TASK)
+        r = pipe.execute()
+        available_proxies = len(set(r[3]) & set(r[4]) & set(r[5]))
+
+        yield CounterMetricFamily('total_proxies', 'total proxies', r[0])
+        yield GaugeMetricFamily('init_proxies', 'total init proxies', r[1])
+        yield GaugeMetricFamily('to_validated_proxies', 'total proxies to validate', r[2])
+        yield GaugeMetricFamily('scored_proxies', 'high score proxies', len(r[3]))
+        yield GaugeMetricFamily('alive_proxies', 'proxies validated recently', len(r[4]))
+        yield GaugeMetricFamily('speed_proxies', 'high speed proxies', len(r[5]))
+        yield GaugeMetricFamily('available_proxies', 'proxies of high quality', available_proxies)
+        yield GaugeMetricFamily('total_common_task', 'common task num', r[6])
+        yield GaugeMetricFamily('total_ajax_task', 'ajax task num', r[7])
+        yield GaugeMetricFamily('total_gfw_task', 'gfw task num', r[8])
+        yield GaugeMetricFamily('total_ajax_gfw_task', 'ajax gfw task num', r[9])
+
+
+def exporter_start():
+    print('starting server http://{}:{}/metrics'.format(
+        EXPORTER_LISTEN_HOST, EXPORTER_LISTEN_PORT))
+    REGISTRY.register(CustomCollector())
+    start_http_server(EXPORTER_LISTEN_PORT, addr=EXPORTER_LISTEN_HOST)
+    while True:
+        time.sleep(5)
+
+

monitor_booter.py

@@ -0,0 +1,11 @@
+"""
+This module is used to start prometheus exporter.
+
+If you have your own metrics to monitor, just custom them
+in haipproxy.monitor.exporter
+"""
+from haipproxy.monitor import exporter_start
+
+
+if __name__ == '__main__':
+    exporter_start()

requirements.txt

@@ -1,8 +1,66 @@
-Scrapy==1.5.0
-requests==2.18.4
-redis==2.10.6
+amqp==2.3.2
+asn1crypto==0.24.0
+attrs==17.4.0
+Automat==0.6.0
+Babel==2.6.0
+beautifulsoup4==4.6.0
+billiard==3.5.0.3
+bs4==0.0.1
+cached-property==1.4.0
+cachetools==2.0.1
+celery==4.1.0
+certifi==2018.1.18
+cffi==1.11.4
+chardet==3.0.4
+click==6.7
+constantly==15.1.0
+cryptography==2.1.4
+cssselect==1.0.3
+docker==2.7.0
+docker-compose==1.19.0
+docker-pycreds==0.2.2
+dockerpty==0.4.1
+docopt==0.6.2
+flower==0.9.1
+google-auth==1.4.1
+hyperlink==17.3.1
+idna==2.6
+incremental==17.5.0
+ipaddress==1.0.19
+jsonschema==2.6.0
+kombu==4.2.1
+kubernetes==5.0.0
+lxml==4.1.1
+oauthlib==2.0.7
+parsel==1.3.1
+prometheus-client==0.2.0
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pycparser==2.18
+PyDispatcher==2.0.5
+PyMySQL==0.7.11
+pyOpenSSL==17.5.0
+python-dateutil==2.7.2
+pytz==2018.4
+PyYAML==3.12
+queuelib==1.4.2
+raven==6.8.0
+redis==2.10.5
+requests==2.13.0
+requests-oauthlib==0.8.0
+rsa==3.4.2
 schedule==0.5.0
+Scrapy==1.5.0
 scrapy-splash==0.7.2
-click==6.7
+selenium==3.11.0
 service-identity==17.0.0
-raven==6.8.0
+six==1.11.0
+SQLAlchemy==1.1.15
+texttable==0.9.1
+tornado==4.2
+Twisted==17.9.0
+urllib3==1.22
+vine==1.1.4
+w3lib==1.19.0
+websocket-client==0.47.0
+zope.interface==4.4.3