Repository: Happy-zyy/MyProject Branch: master Commit: d3187665a338 Files: 33 Total size: 67.4 KB Directory structure: gitextract_am3dz27m/ ├── HySAS/ │ ├── Vendor/ │ │ ├── DB/ │ │ │ └── DB.py │ │ └── WorkerManager/ │ │ ├── WorkerManager.py │ │ ├── __init__.py │ │ └── config.py │ ├── Worker/ │ │ ├── Monitor/ │ │ │ ├── Controller.py │ │ │ └── Monitor.py │ │ └── Process/ │ │ └── Process.py │ ├── config/ │ │ ├── event_mysql.json │ │ ├── raw_mysql.json │ │ └── redis.json │ ├── console.py │ ├── core/ │ │ ├── Functions.py │ │ ├── Vendor.py │ │ ├── Worker.py │ │ ├── log/ │ │ │ ├── DB/ │ │ │ │ ├── error.log │ │ │ │ └── info.log │ │ │ └── console/ │ │ │ ├── error.log │ │ │ └── info.log │ │ └── util.py │ ├── log/ │ │ ├── DB/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── Monitor/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── Process/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── WorkerManager/ │ │ │ ├── error.log │ │ │ └── info.log │ │ └── console/ │ │ ├── error.log │ │ └── info.log │ ├── main.py │ ├── test.py │ └── test1.py └── README.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: HySAS/Vendor/DB/DB.py ================================================ # -*- coding: utf-8 -*- """ # Created on # @author: # @contact: """ # 以下是自动生成的 # # --- 导入系统配置 import core.util as util from core.Vendor import Vendor # --- 导入自定义配置 # 以上是自动生成的 # import pymysql import redis class DB(Vendor): def __init__(self, **kwargs): super().__init__(**kwargs) def get_MySQLdb(self, config="raw_mysql.json", timeout=1500): import os if "/" in config: cfg = util.read_config(config) else: cfg = util.read_config( os.path.join(os.getcwd(), "config", config) ) host = cfg["host"] port = cfg["port"] user = cfg["user"] password = cfg["password"] db = cfg["db"] try: self.logger.info("Trying to connect to MySQL-db") db = pymysql.connect(host, user, password, db) cursor = db.cursor() cursor.execute("SELECT VERSION()") cursor.fetchone() self.logger.info("Successfully connected to MySQL") return db except: self.logger.warning( ">>>>>>>>>>>>>>>>>>Failed to connect to MySQL<<<<<<<<<<<<<<<<<<<") return False def get_redis(self, config="redis.json"): import os if "/" in config: cfg = util.read_config(config) else: cfg = util.read_config( os.path.join(os.getcwd(), "config", config) ) host = cfg["host"] port = cfg["port"] try: self.logger.info("Trying to connect to redis") self.redis = redis.StrictRedis( host=host, port=port ) self.redis.client_list() self.logger.info("Successfully connected to redis") return self.redis except: self.logger.warning("Failed to connect to redis") return False ================================================ FILE: HySAS/Vendor/WorkerManager/WorkerManager.py ================================================ # -*- coding: utf-8 -*- """ # Created on # @author: # @contact: """ # 以下是自动生成的 # # --- 导入系统配置 import core.util as util from core.Vendor import Vendor # --- 导入自定义配置 # 以上是自动生成的 # import time import pickle from core.Functions import get_vendor import copy class WorkerManager(Vendor): def __init__(self, **kwargs): super().__init__(**kwargs) self.redis = get_vendor("DB").get_redis() self.worker_names = list() self.worker_info = dict() self.updating = False if "auto_remove_terminated" in kwargs: self.auto_remove_terminated = kwargs["auto_remove_terminated"] else: self.auto_remove_terminated = -1 # -1: never; # otherwise: auto remove terminated workers # after (self.auto_remove_terminated) seconds def get_worker_names(self): worker_names = util.get_worker_names(logger=self.logger) return worker_names def get_workers_from_redis(self): keys = list(map(str,self.redis.keys("HySAS.Worker.*.*.Info"))) workers = dict() for i in range(len(keys)): channel = keys[i] parsed_channel = channel.split('.') if (len(parsed_channel) == 5) and (parsed_channel[4] == "Info") \ and (parsed_channel[0] == 'HySAS') \ and (parsed_channel[1] == 'Worker'): if parsed_channel[2] not in workers: workers[parsed_channel[2]] = dict() worker = self.redis.hgetall(keys[i]) workers[parsed_channel[2]][parsed_channel[3]] = worker return workers def update_workers(self): """ 更新Worker信息 :return: """ self.updating = True # 根据文件目录检索 self.worker_names = self.get_worker_names() # 从redis中获取worker信息并更新 worker_info = self.get_workers_from_redis() # 去除掉过期信息 for worker_name, workers in worker_info.items(): if worker_name not in self.worker_info: self.worker_info[worker_name] = dict() for nickname, worker_info in workers.items(): heart_beat_interval = 1 difference = float("inf") if "heart_beat_interval" in worker_info: heart_beat_interval = int(worker_info["heart_beat_interval"]) try: difference = time.time() - float(worker_info["heart_beat"]) except ValueError: self.logger.error("时间错误") if difference > heart_beat_interval+1: # 已经停止 if worker_info["status"] == "started": worker_info["status"] = "disappeared" self.logger.warning( "发现一个没有正常关闭的Worker:\n" "\tworker_name: {}\n" "\tnickname: {}" .format(worker_name, nickname) ) self.redis.hmset("HySAS.Worker."+worker_name+"."+nickname+".Info", worker_info) self.worker_info[worker_name][nickname] = copy.deepcopy(worker_info) if self.auto_remove_terminated > -1 and\ difference> self.auto_remove_terminated+1: if "token" in worker_info: token = worker_info["token"] else: token = None removed = self.remove_worker( worker_name=worker_name, nickname=nickname, token=token ) if removed == 1: self.logger.info( "从redis中自动移除过期Worker:\n" "\tworker_name:{}\n" "\tnickname:{}\n" "\ttoken:{}\n" .format(worker_name,nickname,token) ) else: self.logger.warning( "从redis中自动移除过期Worker:{}\n" "\tworker_name:{}\n" "\tnickname:{}\n" "\ttoken:{}\n" .format( removed, worker_name, nickname, token ) ) self.updating = False def remove_worker( self, worker_name, nickname, token ): key = "HySAS.Worker."+worker_name+"."+nickname+"."+"Info" worker_info = self.redis.hgetall(key) if "token" in worker_info: if worker_info["token"] == token: return self.redis.delete(key) else: if token is None: return self.redis.delete(key) return 0 ================================================ FILE: HySAS/Vendor/WorkerManager/__init__.py ================================================ ================================================ FILE: HySAS/Vendor/WorkerManager/config.py ================================================ ================================================ FILE: HySAS/Worker/Monitor/Controller.py ================================================ ================================================ FILE: HySAS/Worker/Monitor/Monitor.py ================================================ from core.Worker import Worker from core.Functions import get_vendor, get_worker_class import os import signal class Monitor(Worker): def __init__(self, **kwargs): super().__init__(**kwargs) def start_worker(self, worker_name, **kwargs): worker = get_worker_class(worker_name=worker_name, **kwargs) worker.start() def terminate_worker(self, nickname=None, pid=None): if pid is None: pid = self.get_pid_by_nickname(redis_cli=self.__redis__, ) os.kill(pid, signal.SIGTERM) def __producer__(self): """ 定时更新redis中各个worker的状态 :return: """ import time # wm: short for "worker_manager" self.wm = get_vendor( "WorkerManager", auto_remove_terminated=3600*24 ) while True: time.sleep(1) self.wm.update_workers() def get_workers_info( self, redis_cli=None, by="nickname", nickname=None, worker_name=None ): if redis_cli is None: redis_cli = self.__redis__ result = list() keys = list() if by == "nickname" and nickname is not None: keys = redis_cli.keys("dHydra.Worker.*." + nickname + ".Info") elif by == "worker_name" and worker_name is not None: keys = redis_cli.keys("dHydra.Worker." + worker_name + ".*.Info") for k in keys: result.append(redis_cli.hgetall(k)) return result def get_pid_by_nickname( self, redis_cli=None, nickname=None ): if redis_cli is None: redis_cli = self.__redis__ workers_info = self.get_workers_info( redis_cli=self.__redis__, nickname=nickname) if len(workers_info) == 1: return int(workers_info[0]["pid"].decode("utf-8")) else: self.logger.warning("Worker is not Unique.") return 0 ================================================ FILE: HySAS/Worker/Process/Process.py ================================================ """ 数据处理函数 """ from core.Worker import Worker from core.Functions import get_vendor from core import util import sys class Process(Worker): def __init__(self, **kwargs): super().__init__(**kwargs) def init_MySQLdb(self): self.raw_MySQL = get_vendor("DB").get_MySQLdb("raw_mysql.json") if self.raw_MySQL is False: self.logger.error("Cannot connect to raw_MySQLdb") return False self.event_MySQL = get_vendor("DB").get_MySQLdb("event_mysql.json") if self.event_MySQL is False: self.logger.error("Cannot connect to event_MySQLdb") return False def on_start(self): self.init_MySQLdb() def __producer__(self): #拥有数据库句柄 通过 self.db调用 """ 取数据 :return: msg """ msg = "data" self.__data_handler__(msg) pass def __data_handler__(self, msg): print(msg) print("======") pass ================================================ FILE: HySAS/config/event_mysql.json ================================================ { "host": "127.0.0.1", "port": 6379, "user": "", "password": "", "db": "" } ================================================ FILE: HySAS/config/raw_mysql.json ================================================ { "host": "127.0.0.1", "port": 3306, "user": "root", "password": "123", "db": "test" } ================================================ FILE: HySAS/config/redis.json ================================================ { "host": "127.0.0.1", "port": 6379 } ================================================ FILE: HySAS/console.py ================================================ # -*- coding: utf-8 -*- import logging from core.Functions import * import core.util as util import click import traceback import pickle def init_logger(): return util.get_logger(logger_name="console") @click.command() @click.argument('worker_name', nargs=1) @click.argument('nickname', nargs=1) def start(worker_name=None, nickname=None): """ 主要依靠nickname来指定配置文件，通过配置文件传入参数 :param worker_name: :param nickname: 如果nickname以.json结尾，那么程序会去"./config/"目录下寻找对应配置文件 如果不是以.json结尾，则认为传入的就是nickname，同时会去"./config/"目录下寻找 nickname.json :return: """ kwargs = dict() kwargs["worker_name"] = worker_name if nickname[-5:] == ".json": kwargs["config"] = nickname kwargs["nickname"] = nickname[0:-5] else: kwargs["config"] = nickname + ".json" kwargs["nickname"] = nickname config_file = os.path.join(os.getcwd(), "config", kwargs["config"]) if os.path.exists(config_file): config = util.read_config(config_file) else: # 命令指定了配置文件，如果不存在，就结束 print("未能找到配置文件：", config_file) print("默认无配置启动") config = {} for k in config.keys(): kwargs[k] = config[k] msg = { "type": "sys", "operation_name": "start_worker", "kwargs": kwargs } __redis__.publish("HySAS.Command", pickle.dumps(msg)) # @click.command() # @click.argument('nickname', nargs=1) def stop(nickname=None): msg = {"type": "sys", "operation_name": "terminate_worker", "kwargs": {"nickname": nickname} } if nickname is not None: msg["kwargs"]["nickname"] = nickname __redis__.publish("HySAS.Command", pickle.dumps(msg)) def shutdown(nickname=None): msg = {"type": "sys", "func": "shutdown" } __redis__.publish("HySAS.Command", pickle.dumps(msg)) def send_command( channel_name="HySAS.Command", command_type="sys", operation_name=None, token=None, kwargs={} ): if operation_name is not None: command = { "type": command_type, "operation_name": operation_name, "token": token, "kwargs": kwargs } __redis__.publish(channel_name, pickle.dumps(command)) logger = init_logger() __redis__ = get_vendor("DB").get_redis() ================================================ FILE: HySAS/core/Functions.py ================================================ """ 框架的全局方法，在主程序运行时会被引用 --- Created on 01/14/2019 @author: Happy-zyy @contact: 617532750@qq.com """ import importlib import sys import os import logging import traceback def get_workers(): candidates = set(os.listdir("./Worker")) | set(os.listdir( os.path.split(os.path.realpath(__file__))[0][:-4] + "Worker")) candidates = list(candidates) workers = list() for item in candidates: if (item[0] >= "A" and item[0] <= "Z"): workers.append(item) return workers """ 动态加载web controller """ def get_controller_method(class_name, method): logger = logging.getLogger("Functions") # get instance of controller if os.path.exists( os.getcwd() + "/Worker/" + class_name + "/" + "Controller.py" ): func = getattr( importlib.import_module("Worker." + class_name + ".Controller"), method ) return func else: try: func = getattr(importlib.import_module( "dHydra.Worker." + class_name + ".Controller"), method ) return func except Exception as e: return False def V(name, vendor_name=None, **kwargs): return get_vendor(name=name, vendor_name=None, **kwargs) def get_vendor(name, vendor_name=None, **kwargs): """ get_vendor方法，动态加载vendor类 """ logger = logging.getLogger('Functions') if vendor_name is None: vendor_name = "V-" + name class_name = name module_name = 'Vendor.' + name + '.' + class_name if os.path.exists( os.path.join(os.getcwd(), "Vendor", name, class_name + ".py") ): try: instance = getattr( __import__(module_name, globals(), locals(), [class_name], 0), class_name )(**kwargs) except ImportError: traceback.print_exc() else: try: instance = getattr( __import__( "HySAS." + module_name, globals(), locals(), [class_name], 0 ), class_name )(**kwargs) except ImportError: traceback.print_exc() return instance def get_worker_class(worker_name, **kwargs): logger = logging.getLogger('Functions') module_name = 'Worker.' + worker_name + '.' + worker_name if os.path.exists( os.getcwd() + "/Worker/" + worker_name + "/" + worker_name + ".py" ): try: if module_name in sys.modules: del sys.modules[module_name] return getattr( __import__(module_name, globals(), locals(), [worker_name], 0), worker_name )(**kwargs) except ImportError: traceback.print_exc() else: try: if "HySAS."+module_name in sys.modules: del sys.modules["HySAS."+module_name] return getattr( __import__( "HySAS." + module_name, globals(), locals(), [worker_name], 0 ), worker_name )(**kwargs) except ImportError: traceback.print_exc() def convert(data): if isinstance(data, bytes): return data.decode('ascii') if isinstance(data, dict): return dict(map(convert, data.items())) if isinstance(data, tuple): return map(convert, data) return data ================================================ FILE: HySAS/core/Vendor.py ================================================ # -*- coding: utf-8 -*- import logging import core.util as util class Vendor: def __init__( self, log_path="log", # console_log=True, # 屏幕打印日志开关，默认True console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO critical_log=False, # critica单独l写文件日志，默认关闭 error_log=True, # error级别单独写文件日志，默认开启 warning_log=False, # warning级别单独写日志，默认关闭 info_log=True, # info级别单独写日志，默认开启 debug_log=False, # debug级别日志，默认关闭 **kwargs ): self.logger = util.get_logger( log_path=log_path, # console_log=console_log, # 屏幕打印日志开关，默认True console_log_level=console_log_level, # 屏幕打印日志的级别，默认为INFO critical_log=critical_log, # critica单独l写文件日志，默认关闭 error_log=error_log, # error级别单独写文件日志，默认开启 warning_log=warning_log, # warning级别单独写日志，默认关闭 info_log=info_log, # info级别单独写日志，默认开启 debug_log=debug_log, # debug级别日志，默认关闭 logger_name=self.__class__.__name__, ) ================================================ FILE: HySAS/core/Worker.py ================================================ # -*- coding: utf-8 -*- """ Worker抽象类 Created on 01/14/2019 @author: Happy-zyy @contact: 617532750@qq.com """ import multiprocessing import threading import time import logging import traceback import redis import json import copy import core.util as util from console import * from datetime import datetime from datetime import timedelta from abc import ABCMeta import signal import sys import os import ast import pickle class Worker(multiprocessing.Process): __metaclass__ = ABCMeta def __init__( self, singleton=True, # 单例模式 nickname=None, # Worker的自定义名字 description="No Description", # 备注说明 heart_beat_interval=1, # 默认1秒心跳 log_path="log", # console_log=True, # 屏幕打印日志开关，默认True console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO critical_log=False, # critical单独写文件日志，默认关闭 error_log=True, # error级别单独写文件日志，默认开启 warning_log=False, # warning级别单独写日志，默认关闭 info_log=True, # info级别单独写日志，默认开启 debug_log=False, # debug级别日志，默认关闭 **kwargs ): super().__init__() # 记录日志配置 self.__log_path__ = log_path self.__console_log__ = console_log self.__console_log_level__ = console_log_level self.__critical_log__ = critical_log self.__error_log__ = error_log self.__warning_log__ = warning_log self.__info_log__ = info_log self.__debug_log__ = debug_log self.logger = util.get_logger( logger_name=self.__class__.__name__, log_path=self.__log_path__, console_log=self.__console_log__, # 屏幕打印日志开关，默认True console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO critical_log=self.__critical_log__, # critica单独l写文件日志，默认关闭 error_log=self.__error_log__, # error级别单独写文件日志，默认开启 warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭 info_log=self.__info_log__, # info级别单独写日志，默认开启 debug_log=self.__debug_log__, # debug级别日志，默认关闭 ) self.__token__ = util.generate_token() if nickname is None: self.__nickname__ = self.__class__.__name__ else: self.__nickname__ = nickname self.nickname = self.__nickname__ self.name = self.__nickname__ self.__singleton__ = singleton self.__description__ = description self.__heart_beat_interval__ = heart_beat_interval self.__threads__ = dict() # 被监控的线程 self.__data_feeder__ = set() # 本Worker订阅的内容 self.__follower__ = set() # Follower self.__error_msg__ = None # self.__stop_info__ = None # self.__stop_time__ = None # self.__status__ = "init" self.redis_key = "HySAS.Worker." + \ self.__class__.__name__ + "." + self.__nickname__ + "." self.channel_pub = self.redis_key + "Pub" """ self.__threads__ = { "nickname": { "description" : "该线程功能备注说明", "name" : "该线程的名字", "target" : "该线程的target" "restart_mode" : "重启模式，可以为 manual/auto/remove; manual则代表允许管理员发送命令手工重启线程, auto则一旦线程关闭立即自动开启， remove则代表一旦线程结束就从监控列表移除", "restart_func" : "自动/手动重启时调用的方法", }, } """ self.shutdown_signals = [ "SIGQUIT", # quit 信号 "SIGINT", # 键盘信号 "SIGHUP", # nohup 命令 "SIGTERM", # kill 命令 ] for s in self.shutdown_signals: # 捕获退出信号后的要调用的,唯一的 shutdown 接口 try: if hasattr(signal, s): signal.signal( getattr(signal, s, None), self.__on_termination__ ) except Exception as e: self.logger.info("绑定退出信号：{}失败，可能与windows系统有关。".format(s)) # 清空它，在run以后重新实例化 # 否则windows下会无法pickle del(self.logger) def __is_unique__(self): info = self.__redis__.hgetall(self.redis_key + "Info") if "token" in info: if info["token"] != self.__token__: if "heart_beat" in info: if (time.time() - float(info["heart_beat"])) < self.__heart_beat_interval__: return False return True def __auto_restart_thread__(self): # Worker内置的默认自动重启线程方法 pass def __command_handler__(self, msg_command): # cli is a dict with the following structure: """ msg_command = { "type" : "sys/customized", "operation" : "operation_name", "kwargs" : "suppose that the operation is a function, we need to pass some arguments", "token" : "the token is used to verify the authentication of the operation" } """ try: msg_command = pickle.loads(msg_command) if not isinstance(msg_command, dict): return except Exception as e: traceback.print_exc() print(e) if msg_command["type"] == "sys": if hasattr(self, msg_command["operation_name"]): func = getattr(self, msg_command["operation_name"]) try: print(msg_command["kwargs"]) result = func(**msg_command["kwargs"]) except Exception as e: traceback.print_exc() self.logger.error(e) def monitor_add_thread(self, thread, description="No Description", restart_mode="manual", restart_func=None): # 将该线程加入管理员监控范围 pass def monitor_remove_thread(self, thread): # 取消管理员对线程thread的监控 pass def init_redis(self): # 检测redis, sqldb连接 try: self.__redis__ = get_vendor("DB").get_redis() #self.__redis__.client_list() self.__listener__ = self.__redis__.pubsub() self.__listener__.subscribe(["HySAS"]) except redis.ConnectionError: self.logger.error("Cannot connect to redis") return False def check_prerequisites(self): """ 检查是否满足开启进程的条件 """ # 如果是单例，检测是否重复开启 return True def __listen_command__(self): # self.command_listener = self.__redis__.pubsub() channel_name = self.redis_key + "Command" self.command_listener.subscribe([channel_name]) while True: msg_command = self.command_listener.get_message() if msg_command: if (msg_command["type"] == "message") or \ (msg_command["type"] == "pmessage"): self.__command_handler__(msg_command["data"]) else: time.sleep(0.5) def __heart_beat__(self): # flush status infomation to redis status = dict() status["heart_beat"] = time.time() status["nickname"] = self.__nickname__ status["pid"] = self.pid status["token"] = self.__token__ status["heart_beat_interval"] = self.__heart_beat_interval__ if self.__error_msg__: status["error_msg"] = self.__error_msg__ if self.__stop_info__: status["stop_info"] = self.__stop_info__ if self.__stop_info__: status["stop_time"] = self.__stop_time__ if self.__status__: status["status"] = self.__status__ if self.__threads__: status["threads"] = copy.deepcopy(self.__threads__) if self.__data_feeder__: status["data_feeder"] = self.__data_feeder__ if self.__follower__: status["follower"] = self.__follower__ self.__redis__.hmset(self.redis_key + "Info", status) def __producer__(self): """ 在子类中被重写的用以作为生产者的线程 若不重写，线程启动后就结束了 """ pass def __consumer__(self): """ 默认的消费者线程 随着Worker进程的start而启动 """ pass # while True: # data = self.__listener__.get_message(timeout=10) # if data is not None: # self.__data_handler__(data) # time.sleep(1) # 需要在子类中重写的数据处理方法 def __data_handler__(self, msg): """ 需要在子类中被重写的用以处理数据的方法， 接受到的msg数据是原始的从Redis中监听到的数据 """ print("*****") pass def __before_termination__(self, sig): self.logger.info("收到了退出信号。进程类：{}，进程名：{},进程pid:{}" .format( self.__class__.__name__, self.__nickname__, self.pid ) ) def __on_termination__(self, sig, frame): self.__before_termination__(sig) self.__status__ = "terminated" self.__heart_beat__() # The last heart_beat, sad... sys.exit(0) def publish(self,data,channel_name=None): if channel_name is None: channel_name = self.channel_pub # publish data to redis try: self.__redis__.publish(channel_name, data) except Exception as e: self.logger.warning(e) def __on_start__(self): """ 进程开始运行时调用 :return: """ if self.check_prerequisites() is not True: sys.exit(0) self.init_redis() # 实例化self.logger self.logger = util.get_logger( logger_name=self.__class__.__name__, log_path=self.__log_path__, # console_log=self.__console_log__, # 屏幕打印日志开关，默认True console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO critical_log=self.__critical_log__, # critical写文件日志，默认关闭 error_log=self.__error_log__, # error级别单独写文件日志，默认开启 warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭 info_log=self.__info_log__, # info级别单独写日志，默认开启 debug_log=self.__debug_log__, # debug级别日志，默认关闭 ) def on_start(self): pass def run(self): """ 初始化Worker """ self.__on_start__() self.logger.info("初始化{}".format(self.__class__.__name__)) # 用户自定义的on_start self.on_start() # 首先检查是否已经有相同的进程被开启 if self.__is_unique__(): self.__status__ = "started" else: self.error_msg = "Duplicated Process" self.logger.warning(self.error_msg) sys.exit(0) # 开启监听命令线程 self.__thread_listen_command__ = threading.Thread( target=self.__listen_command__ ) self.__thread_listen_command__.setDaemon(True) self.monitor_add_thread( thread=self.__thread_listen_command__, description="Listening Command Channel", restart_mode="auto", restart_func=self.__auto_restart_thread__ ) self.__thread_listen_command__.start() # 检查初始化设置，按需开启 # PUB线程 self.__thread_pub__ = threading.Thread(target=self.__producer__) self.__thread_pub__.setDaemon(True) self.monitor_add_thread( thread=self.__thread_pub__, description="DATA PUBLISHER", restart_mode="auto", restart_func=self.__auto_restart_thread__ ) self.__thread_pub__.start() # LISTENER self.__thread_sub__ = threading.Thread(target=self.__consumer__) self.__thread_sub__.setDaemon(True) self.monitor_add_thread( thread=self.__thread_sub__, description="DATA CONSUMER", restart_mode="auto", restart_func=self.__auto_restart_thread__ ) self.__thread_sub__.start() while True: # heart beat self.__heart_beat__() time.sleep(self.__heart_beat_interval__) def subscribe(self,channel_name=None, worker_name=None, nickname=None): """ 订阅Worker 可以直接填入channel_name 也可以通过填入worker_name/nickname/worker_name+nickname来自动订阅对应的内容 """ if channel_name is not None: self.__listener__.subscribe(channel_name) return None if (worker_name is not None) and (nickname is None): # 订阅所有此类Worker self.__listener__.psubscribe( "HySAS.Worker." + worker_name + ".*.Pub" ) self.logger.info( "About to subscribe the Worker of worker_name: {}, pattern:{}" .format( worker_name, "HySAS.Worker." + worker_name + ".*.Pub" ) ) elif (worker_name is not None) and (nickname is not None): channel_name = "HySAS.Worker." \ + worker_name + "." + nickname + ".Pub" self.__listener__.subscribe(channel_name) self.logger.info( "Subscribed: {}".format(channel_name) ) elif (nickname is not None): # 订阅nickname self.__listener__.psubscribe( "HySAS.Worker.*." + nickname + ".Pub" ) self.logger.info( "About to subscribe the Worker of nickname: {}, pattern:{}" .format( nickname, "HySAS.Worker.*." + nickname + ".Pub" ) ) else: self.logger.warning("nickname/worker_name的输入方式不合理") def unsubscribe(self, worker_name=None, nickname=None): """ 退订Worker """ if (worker_name is not None) and (nickname is None): # 订阅所有此类Worker self.__listener__.punsubscribe( "HySAS.Worker." + worker_name + ".*.Pub") self.logger.info( "About to unsubscribe the Worker of worker_name: {}, \ pattern:{}" .format( nickname, "HySAS.Worker.*." + worker_name + ".Pub" ) ) pass elif (nickname is not None): # 订阅nickname self.__listener__.punsubscribe( "HySAS.Worker.*." + nickname + ".Pub" ) self.logger.info( "About to subscribe the Worker of nickname: {}, pattern:{}" .format( nickname, "HySAS.Worker.*." + nickname + ".Pub" ) ) else: self.logger.warning("nickname/worker_name的输入方式不合理") ================================================ FILE: HySAS/core/log/DB/error.log ================================================ ================================================ FILE: HySAS/core/log/DB/info.log ================================================ ================================================ FILE: HySAS/core/log/console/error.log ================================================ ================================================ FILE: HySAS/core/log/console/info.log ================================================ ================================================ FILE: HySAS/core/util.py ================================================ import logging import os import hashlib import time import json def get_worker_names(logger=None): """ 根据文件夹名字返回所有可能的worker_name :return: """ worker_names = [] # path = os.path.split(os.path.realpath(__file__))[0][:-6]+"/Worker" # worker_names.extend(os.listdir(path)) try: worker_names.extend(os.listdir(os.getcwd()+"/Worker")) except FileNotFoundError as e: if logger is None: print("HySAS运行目录下没有Worker文件夹") else: logger.warning("HySAS运行目录下没有Worker文件夹") return worker_names def get_logger( logger_name="main", log_path="log", # console_log=True, # 屏幕打印日志开关，默认True console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO critical_log=False, # critical单独写文件日志，默认关闭 error_log=True, # error级别单独写文件日志，默认开启 warning_log=False, # warning级别单独写日志，默认关闭 info_log=True, # info级别单独写日志，默认开启 debug_log=False, # debug级别日志，默认关闭 ): formatter = logging.Formatter( '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s' ) logger = logging.getLogger(logger_name) logger.setLevel(logging.DEBUG) if log_path: # 补全文件夹 if log_path[-1] != '/': log_path += '/' if not logger.handlers: # 屏幕日志打印设置 if console_log: console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) console_handler.setLevel(logging.INFO) logger.addHandler(console_handler) if not os.path.exists(log_path + logger_name): os.makedirs(log_path + logger_name) # 打开下面的输出到文件 if critical_log: log_handler = logging.FileHandler( log_path + logger_name + '/critical.log' ) log_handler.setLevel(logging.CRITICAL) log_handler.setFormatter(formatter) logger.addHandler(log_handler) if error_log: log_handler = logging.FileHandler( log_path + logger_name + '/error.log' ) log_handler.setLevel(logging.ERROR) log_handler.setFormatter(formatter) logger.addHandler(log_handler) if warning_log: log_handler = logging.FileHandler( log_path + logger_name + '/warning.log' ) log_handler.setLevel(logging.WARNING) log_handler.setFormatter(formatter) logger.addHandler(log_handler) if info_log: log_handler = logging.FileHandler( log_path + logger_name + '/info.log' ) log_handler.setLevel(logging.INFO) log_handler.setFormatter(formatter) logger.addHandler(log_handler) if debug_log: log_handler = logging.FileHandler( log_path + logger_name + '/debug.log' ) log_handler.setLevel(logging.DEBUG) log_handler.setFormatter(formatter) logger.addHandler(log_handler) return logger def generate_token(): token = hashlib.sha1() token.update(str(time.time()).encode()) token = token.hexdigest() return token def read_config(file_path): # 读取配置 try: f_config = open(file_path) cfg = json.load(f_config) except Exception as e: print("{}".format(e)) cfg = dict() print( "未能正确加载{}，请检查路径，json文档格式，或者忽略此警告" .format(file_path) ) return cfg ================================================ FILE: HySAS/log/DB/error.log ================================================ ================================================ FILE: HySAS/log/DB/info.log ================================================ 2019-01-15 10:38:23,552 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:38:25,562 - DB - 68 - WARNING - Failed to connect to redis 2019-01-15 10:49:10,014 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:55:46,379 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:55:49,677 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:56:19,580 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:56:45,196 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:57:28,660 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 10:57:31,169 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:12:30,375 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:12:38,731 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:17:00,225 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:17:03,378 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:19:08,291 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:19:14,068 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:20:30,782 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:20:50,996 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:27:14,317 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:27:17,802 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:28:06,379 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:33:25,918 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:33:28,955 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:34:53,982 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:34:56,690 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:35:04,770 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:35:24,233 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:36:02,853 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:36:11,119 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 11:36:17,741 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:55:08,689 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:55:30,131 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:55:58,190 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:56:45,528 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:57:11,346 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:57:57,146 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 14:58:13,734 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:00:27,200 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:01:49,462 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:01:49,480 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:02:13,434 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:10:44,988 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:10:45,020 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:36:17,193 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:36:17,237 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:36:17,704 - DB - 59 - INFO - Trying to connect to redis 2019-01-15 15:36:17,717 - DB - 59 - INFO - Trying to connect to redis 2019-01-16 11:23:18,509 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:18,517 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:18,668 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:19,312 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:19,318 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:19,326 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:44,698 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:44,706 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:44,735 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:45,256 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:45,263 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:45,270 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:23:45,272 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:25:34,629 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:40,779 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,228 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,235 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,271 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,794 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,801 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,808 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:25:59,809 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:26:41,487 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:41,494 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:41,548 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,183 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,190 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,197 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,199 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:26:42,256 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,263 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,292 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:26:42,294 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:59:43,416 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:43,423 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:43,453 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:44,014 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:44,022 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 11:59:44,035 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 12:00:42,046 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,053 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,082 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,633 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,637 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,640 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,644 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,648 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,650 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 12:00:42,651 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 12:00:42,652 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 12:00:42,653 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 12:00:42,655 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:05:27,277 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:05:40,119 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:07:35,467 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:08:00,707 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:08:57,073 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:09:44,100 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:12:09,237 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:14:16,598 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:16:39,681 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:17:03,198 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:19:56,705 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:20:22,377 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:21:44,117 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:22:49,185 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:24:28,167 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:37:33,920 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:53,691 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:53,701 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:53,748 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,371 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,378 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,385 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,386 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:39:54,392 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:39:54,409 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,417 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,426 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:39:54,428 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:39:54,431 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:44:17,721 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:17,729 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:17,765 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,326 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,333 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,334 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:44:18,335 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:45:06,377 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,384 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,419 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,964 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,965 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,980 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,981 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:45:06,982 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:45:06,983 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:45:06,984 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:45:06,986 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:53:47,231 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,239 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,271 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,836 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:53:47,852 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:53:47,854 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:54:42,455 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:54:56,807 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:20,724 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:20,731 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:20,770 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,360 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,367 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,368 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,374 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,375 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:56:21,376 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:56:21,379 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:56:21,402 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:47,471 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:47,478 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:47,507 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,083 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,090 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,097 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,099 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:58:48,101 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 14:58:48,104 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,109 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,112 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,120 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 14:58:48,122 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 14:58:48,124 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 15:14:27,070 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,077 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,112 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,672 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,679 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 15:14:27,691 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 15:14:27,698 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,140 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,148 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,187 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,771 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:22:55,788 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 15:22:55,798 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:52:49,239 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:53:33,871 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:53:48,123 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:53:55,740 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:54:53,430 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:55:31,092 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:58:03,617 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:58:33,713 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 15:58:50,782 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:01:24,014 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:01:43,708 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:01:56,358 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:02:52,853 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:03:15,355 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:41,913 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:41,920 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:41,950 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,580 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,587 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,590 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,594 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,596 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:06:42,597 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,602 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:06:42,605 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:06:42,607 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:06:42,609 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:06:42,618 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:09,805 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:09,811 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:09,841 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,416 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:07:10,426 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:58,732 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:58,739 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:58,768 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:59,340 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:59,347 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:15:59,359 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:15:59,379 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,013 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,020 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,051 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,610 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,617 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,618 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,626 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:30:05,629 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:30:05,632 - DB - 61 - INFO - Trying to connect to redis 2019-01-16 16:30:05,635 - DB - 37 - INFO - ӵMySQL-db 2019-01-16 16:30:05,637 - DB - 42 - INFO - ѾɹӵMySQL 2019-01-16 16:30:05,643 - DB - 61 - INFO - Trying to connect to redis ================================================ FILE: HySAS/log/Monitor/error.log ================================================ ================================================ FILE: HySAS/log/Monitor/info.log ================================================ 2019-01-16 11:23:45,365 - Monitor - 326 - INFO - ʼWorker 2019-01-16 11:25:59,817 - Monitor - 326 - INFO - ʼWorker 2019-01-16 11:26:42,206 - Monitor - 326 - INFO - ʼWorker 2019-01-16 11:59:44,056 - Monitor - 326 - INFO - ʼWorker 2019-01-16 12:00:42,652 - Monitor - 326 - INFO - ʼWorker 2019-01-16 14:39:54,393 - Monitor - 320 - INFO - ʼWorker 2019-01-16 14:44:18,339 - Monitor - 320 - INFO - ʼWorker 2019-01-16 14:45:06,985 - Monitor - 320 - INFO - ʼWorker 2019-01-16 14:53:47,855 - Monitor - 320 - INFO - ʼWorker 2019-01-16 14:56:21,380 - Monitor - 320 - INFO - ʼWorker 2019-01-16 14:58:48,102 - Monitor - 320 - INFO - ʼWorker 2019-01-16 15:14:27,692 - Monitor - 320 - INFO - ʼWorker 2019-01-16 15:22:55,791 - Monitor - 327 - INFO - ʼWorker 2019-01-16 16:06:42,603 - Monitor - 327 - INFO - ʼWorker 2019-01-16 16:07:10,421 - Monitor - 327 - INFO - ʼWorker 2019-01-16 16:15:59,359 - Monitor - 327 - INFO - ʼWorker 2019-01-16 16:30:05,630 - Monitor - 327 - INFO - ʼWorker ================================================ FILE: HySAS/log/Process/error.log ================================================ ================================================ FILE: HySAS/log/Process/info.log ================================================ 2019-01-16 11:26:42,301 - Process - 326 - INFO - ʼWorker 2019-01-16 11:59:44,056 - Process - 326 - INFO - ʼWorker 2019-01-16 12:00:42,656 - Process - 326 - INFO - ʼWorker 2019-01-16 14:39:54,432 - Process - 320 - INFO - ʼWorker 2019-01-16 14:44:18,338 - Process - 320 - INFO - ʼWorker 2019-01-16 14:45:06,986 - Process - 320 - INFO - ʼWorker 2019-01-16 14:53:47,855 - Process - 320 - INFO - ʼWorker 2019-01-16 14:56:21,380 - Process - 320 - INFO - ʼWorker 2019-01-16 14:58:48,125 - Process - 320 - INFO - ʼWorker 2019-01-16 15:14:27,692 - Process - 320 - INFO - ʼWorker 2019-01-16 15:22:55,791 - Process - 327 - INFO - ʼWorker 2019-01-16 16:06:42,610 - Process - 327 - INFO - ʼWorker 2019-01-16 16:07:10,421 - Process - 327 - INFO - ʼWorker 2019-01-16 16:15:59,359 - Process - 327 - INFO - ʼWorker 2019-01-16 16:30:05,638 - Process - 327 - INFO - ʼWorker ================================================ FILE: HySAS/log/WorkerManager/error.log ================================================ ================================================ FILE: HySAS/log/WorkerManager/info.log ================================================ ================================================ FILE: HySAS/log/console/error.log ================================================ 2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' 2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' 2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' ================================================ FILE: HySAS/log/console/info.log ================================================ 2019-01-15 15:02:10,563 - console - 86 - INFO - Monitor has started 2019-01-15 15:10:45,019 - console - 86 - INFO - Monitor has started 2019-01-15 15:36:17,236 - console - 86 - INFO - Monitor has started 2019-01-16 11:23:18,665 - console - 139 - INFO - Monitor has started 2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' 2019-01-16 11:23:44,734 - console - 139 - INFO - Monitor has started 2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' 2019-01-16 11:25:59,270 - console - 139 - INFO - Monitor has started 2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname' 2019-01-16 11:26:41,524 - console - 139 - INFO - Monitor has started 2019-01-16 11:59:43,452 - console - 139 - INFO - Monitor has started 2019-01-16 12:00:42,081 - console - 139 - INFO - Monitor has started 2019-01-16 14:39:53,748 - console - 139 - INFO - Monitor has started 2019-01-16 14:44:17,764 - console - 139 - INFO - Monitor has started 2019-01-16 14:45:06,418 - console - 139 - INFO - Monitor has started 2019-01-16 14:53:47,270 - console - 139 - INFO - Monitor has started 2019-01-16 14:56:20,769 - console - 139 - INFO - Monitor has started 2019-01-16 14:58:47,506 - console - 139 - INFO - Monitor has started 2019-01-16 15:14:27,111 - console - 145 - INFO - Monitor has started 2019-01-16 15:22:55,187 - console - 145 - INFO - Monitor has started 2019-01-16 16:06:41,950 - console - 145 - INFO - Monitor has started 2019-01-16 16:07:09,840 - console - 145 - INFO - Monitor has started 2019-01-16 16:15:58,767 - console - 145 - INFO - Monitor has started 2019-01-16 16:30:05,050 - console - 145 - INFO - Monitor has started ================================================ FILE: HySAS/main.py ================================================ from console import logger from core.Functions import * import time import signal import click import pickle __redis__ = get_vendor("DB").get_redis() worker_dict = dict() def __on_termination__(sig, frame): logger.info("The HySAS Server is about to terminate, pid:{}" .format(os.getpid()) ) sys.exit(0) def bind_quit_signals(): import signal shutdown_signals = [ "SIGQUIT", # quit 信号 "SIGINT", # 键盘信号 "SIGHUP", # nohup 命令 "SIGTERM", # kill 命令 ] for s in shutdown_signals: if hasattr(signal, s): signal.signal( getattr(signal, s, None), __on_termination__ ) def start_worker(worker_name, **kwargs): worker = get_worker_class(worker_name=worker_name, **kwargs) worker_dict[worker.nickname] = worker worker.start() def terminate_worker(nickname=None, pid=None): logger.info("{}".format(worker_dict)) if pid is None: pid = get_pid_by_nickname(redis_cli=__redis__, nickname=nickname) os.kill(pid, signal.SIGTERM) i = 0 while worker_dict[nickname]._popen is None and i < 30: time.sleep(0.1) i += 1 worker_dict[nickname]._popen.wait(1) worker_dict.pop(nickname) def get_workers_info(redis_cli=None, by="nickname", nickname=None, worker_name=None): if redis_cli is None: redis_cli = __redis__ result = list() keys = list() if by == "nickname" and nickname is not None: keys = redis_cli.keys("HySAS.Worker.*." + nickname + ".Info") elif by == "worker_name" and worker_name is not None: keys = redis_cli.keys("HySAS.Worker." + worker_name + ".*.Info") for k in keys: result.append(redis_cli.hgetall(k)) return result def get_pid_by_nickname(redis_cli=None, nickname=None): if redis_cli is None: redis_cli = __redis__ workers_info = get_workers_info(redis_cli=__redis__, nickname=nickname) workers_info = list(map(convert, workers_info)) if len(workers_info) == 1: return int(workers_info[0]["pid"]) else: logger.warning("Worker is not Unique.") return 0 def __command_handler__(msg_command): # msg_command is a dict with the following structure: """ msg_command = { "type" : "sys/user/data", "operation" : "operation_name", "kwargs" : "suppose that the operation is a function, we need to pass some arguments", "token" : "the token is used to verify the authentication of the operation" } """ import sys try: msg_command = pickle.loads(msg_command) if not isinstance(msg_command, dict): return except Exception as e: traceback.print_exc() print("msg_command is not dict \n",e) if msg_command["type"] == "sys": if "func" in msg_command and msg_command["func"] == "shutdown": signal_handler() elif hasattr( sys.modules["HySAS.main"], msg_command["operation_name"] ): func = getattr( sys.modules["HySAS.main"], msg_command["operation_name"] ) try: print(msg_command["kwargs"]) result = func(**msg_command["kwargs"]) except Exception as e: traceback.print_exc() logger.error(e) elif msg_command["type"] == "user": """ 用于接受前端用户数据 """ pass elif msg_command["type"] == "data": """ 用于接受关联分析时可能向外传输的交互信号 """ pass def signal_handler(): for work_name in list(worker_dict.keys()): terminate_worker(work_name) print('Shutdown HySAS!') sys.exit(0) # @click.command() # @click.argument('what', nargs=-1) def main(what=None): try: if what: if what[0] != "HySAS": print("Please input HySAS for starting") exit(0) else: print( "Welcome to HySAS! Following is the awesome!!!" ) logo = "◆　　　　◆　　　　　　　　　　　　　　　　　　　　　　 ◆\n"\ "◆　　　　◆　　　　　　　　　　　　　◆◆◆◆◆　　　　　◆◆　　　　　　◆◆◆◆◆\n"\ "◆　　　　◆　　　　　　　　　　　　◆◆◆◆◆ 　　　　 ◆◆◆　　　　　◆◆◆\n"\ "◆　　　　◆　　　◆◆　　◆◆　　　◆◆◆◆　　　　　　 ◆◆◆◆　　　　　　◆◆◆　　　　\n"\ "◆◆◆◆◆◆　　　　◆◆　◆◆　　　　◆◆◆◆◆　　　　◆◆　◆◆　　　　　　◆◆◆◆◆　　\n"\ "◆　　　　◆　　　　◆◆◆◆　　　　　　　　◆◆◆　　　◆◆◆◆◆◆　　　　　　　　◆◆◆　\n"\ "◆　　　　◆　　　　　◆◆◆　　　◆◆◆　◆◆◆　　　 ◆◆　　◆◆◆　　　◆◆◆ ◆◆◆　　\n"\ "◆　　　　◆　　　　　◆◆　　　　◆◆◆◆◆◆　　　　◆◆　　　◆◆◆　　　◆◆◆◆◆◆　　\n"\ "　　　　　　　　　　　◆◆　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　\n"\ "　　　　　　　　　　　◆◆　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　\n" print(logo) # open a thread for the Worker of Monitor start_worker(worker_name="Monitor",nickname="Monitor") logger.info("Monitor has started") # 开启Web if len(what) == 1: # 没指定http端口，不开启Tornado pass else: port = int(what[1]) start_worker(worker_name="Web", nickname="Tornado") # 绑定退出信号 bind_quit_signals() redis_conn = get_vendor("DB").get_redis() command_listener = redis_conn.pubsub() channel_name = "HySAS.Command" command_listener.subscribe([channel_name]) # 数据关联 态势分析 start_worker(worker_name="Process",nickname="Process") for item in command_listener.listen(): # 阻塞式接收 if isinstance(item['data'], bytes): __command_handler__(item["data"]) else: print("HySAS What?") except Exception as e: traceback.print_exc() logger.error("{}".format(e)) print("Hail error?") if __name__ == '__main__': main(what = ["HySAS"]) ================================================ FILE: HySAS/test.py ================================================ from multiprocessing import Process import threading import time import os import redis import pickle import logging from HySAS.core.Functions import * def __command_handler__(msg_command): print(msg_command) redis_conn = get_vendor("DB").get_redis() command_listener = redis_conn.pubsub() channel_name = "dHydra.Command" command_listener.subscribe([channel_name]) # for item in command_listener.listen(): # if item["type"] == "message": # print(pickle.loads(item['data'])) from datetime import datetime # status = dict() # status["heart_beat"] = time.time() # status["nickname"] = "zyyz" # redis_key = "HySAS" # # redis_conn.hmset(redis_key + "Info", status) # info = redis_conn.hgetall(redis_key + "Info") worker_names = [] # path = os.path.join(os.path.split(os.path.realpath(__file__))[0]+"/Worker") # worker_names.extend(os.listdir(path)) try: worker_names.extend(os.listdir(os.getcwd()+"/Worker")) except FileNotFoundError as e: print("HySAS运行目录下没有Worker文件夹") print(worker_names) ================================================ FILE: HySAS/test1.py ================================================ import time import os import redis import pickle from HySAS.core.Functions import * msg = { "type": "sys", "operation_name": "start_worker", "kwargs":"haha" } print(pickle.dumps(msg)) __redis__ = get_vendor("DB").get_redis() __redis__.publish("dHydra.Command", pickle.dumps(msg)) ================================================ FILE: README.md ================================================ # HySAS简介 HySAS(Hydrogen Situation Awareness System )是一个安全态势感知系统的开发框架，用于实现多进程多数据源的实时计算。不管是在开发过程还是运行过程中，都拥有极高的并行性和扩展性。 **特点** + 采用Redis作为消息队列，实现进程间通信 + 每个Worker都可以被独立工作，必要时可以相互配合 + 支持动态联编，灵活加载功能模块 # 运行环境 - python 3.5以上 (开发环境windows 10, python 3.6)，**不对python2.7提供支持，多版本虚拟环境安装请参考安装HySAS文档** - Mysql 8.0.13 - Redis X64 3.2 # Quick Start 1. Step 1:安装Redis（略） 2. Step 2:安装Mysql（略） # 目录介绍 # 核心对象 ## main.py 主程序入口 main程序订阅了`HySAS.Command`频道，通过向这个频道发送命令可以开启或关闭响应的服务 `worker_dict` : 对象字典`worker_dict[worker_name.nickname]：worker对象`用于记录系统中正在运行的worker对象 ## console.py 允许通过命令行的形式，以第三方的身份动态加载Worker对象。旨在解决如果主程序中有部分Worker died之后，无需重启主程序，便可重启部分Worker，保证数据流的畅通性。 **用命令行启动/关闭** ``` start Demo DemoName ``` 这样做就会开启一个叫`DemoName`的`Demo`进程 这里的`DemoName`是`nickname`，`nickname`是全局唯一的，可能有多个不同的进程都是Demo类，但是每个进程都有唯一的昵称。 以Worker为例： > - 自己订阅了自己的频道 > - 每秒向Redis中的`HySAS.Worker.Demo.DemoName.Pub`频道发送一个数字 > - 将自己订阅收到的内容打印到屏幕 > 原理：之所以可以这样做是因为运行了`pip install --editable .`，它在你当前环境下认识了start命令，它会调用HySAS.console下的start方法，这个方法向Redis中的"HySAS.Command"频道发送了一条指令，HySAS Server监听到此指令以后就会实例化Demo目录下的DemoName类对象 ``` stop DemoName ``` > 运行stop DemoName后，我们会向"HySAS.Command"频道发送一条指令，由HySAS Server去执行这个关闭Demo的任务，会捕捉到终止信号，并且执行用户自定义的`__before_termination__` ## 1.Worker Worker类是HySAS的任务处理单元的基类，其继承于`multiprocessing.Process`。用于并行处理各个子任务，其派生类包括`Monitor`、`Process`。 ###工作流程 `__init__（）`：初始化成员变量。比较重要的成员：`self.__data_feeder__ = set()`用于存储本Worker订阅的内容。 `run（）`： `__on_start__()`：用于连接redis（所有Worker共同订阅了`HySAS`频道）、Mysql，并实例化logger `on_start()`：自定义启动项，用于拓展接口 `__is_unique__()`：首先检查是否已经有相同的进程被开启，通过worker启动的时间，每个worker会生产一个唯一的`token` `__thread_listen_command__`：监听命令线程（守护进程），订阅了`self.redis_key.Command`频道，其中`self.redis_key`由创建worker时传入的参数决定。 `__thread_pub__`：生产者进程（按需开启） `__thread_sub__`：消费者进程（按需开启），获取了订阅`HySAS`频道的redis句柄，并实时接受数据进行处理 `__command_handler__`：上述的数据处理函数 `__heart_beat__()`：为该worker注册心跳包，用于检验该worker是否被多重开启、存活性等。通过在`self.redis_key.Info`频道发布自身状态信息 ，心跳包内容如下： ```python status = dict() #以下必带 status["heart_beat"] = time.time() status["nickname"] = self.__nickname__ status["pid"] = self.pid status["token"] = self.__token__ status["heart_beat_interval"] = self.__heart_beat_interval__ #以下选带 if self.__error_msg__: status["error_msg"] = self.__error_msg__ if self.__stop_info__: status["stop_info"] = self.__stop_info__ if self.__stop_info__: status["stop_time"] = self.__stop_time__ if self.__status__: status["status"] = self.__status__ if self.__threads__: status["threads"] = copy.deepcopy(self.__threads__) if self.__data_feeder__: status["data_feeder"] = self.__data_feeder__ if self.__follower__: status["follower"] = self.__follower__ self.__redis__.hmset(self.redis_key + "Info", status) ``` ## 2.Monitor 继承于Worker，主要用于派生WorkerManger对象监视各个Worker的运行状态 ## 3.WorkerManger WorkerManger继承于Vendor，由Woker类的派生类对象Monitor重写了`run（）`中由`__thread_pub__`调用的`__producer__`生产。 ### 工作流程 `__init__（）`：初始化成员变量，获取redis句柄。 `update_workers（）`：更新Worker信息。从redis获取所有对象的心跳包，以`dict[workerName][nickName] ：心跳包`的字典格式存入`work_info`。如果发现当前时间间隔大于worker的`heart_beat_interval`，即心跳停止，则放出warning，并从redis中删除该Worker信息。