IPProxy.py

# coding:utf-8
# coding:utf-8
import traceback

import gevent
import psutil
from gevent import monkey

from util.watcher import Watcher

monkey.patch_all()

from db.DataStore import store_data
from multiprocessing import Value, Queue, Process
from api.apiServer import start_api_server

from spider.ProxyCrawl import startProxyCrawl
from validator.Validator import validator, getMyIP
from loguru import logger

if __name__ == "__main__":
    #Watcher()

    my_current_ip = getMyIP()
    logger.info("My Current IP:{}", my_current_ip)

    DB_PROXY_NUM = Value('i', 0)
    q1 = Queue()
    q2 = Queue()
    # 内部处理ip/port验证的process 控制列表
    process_pool = {}

    p0 = Process(target=start_api_server)
    p1 = Process(target=startProxyCrawl, args=(q1, DB_PROXY_NUM))
    p2 = Process(target=validator, args=(q1, q2, my_current_ip, process_pool))
    p3 = Process(target=store_data, args=(q2, DB_PROXY_NUM))

    try:
        p0.start()
        p1.start()
        p2.start()
        p3.start()
        p0.join()
        p1.join()
        p2.join()
        p3.join()
    except KeyboardInterrupt as e:
        info = traceback.format_exc()
        logger.error("Ctrl+c is invoke to stop the application:{}", info)
    finally:
        p0.terminate()
        p1.terminate()
        p2.terminate()
        p3.terminate()
        logger.info("ready to terminate the process:{}", len(process_pool))

        for p in process_pool.values():
            p.terminate()