[ { "path": "HySAS/Vendor/DB/DB.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\n# Created on\n# @author:\n# @contact:\n\"\"\"\n# 以下是自动生成的 #\n# --- 导入系统配置\nimport core.util as util\nfrom core.Vendor import Vendor\n# --- 导入自定义配置\n# 以上是自动生成的 #\nimport pymysql\n\nimport redis\n\n\nclass DB(Vendor):\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n def get_MySQLdb(self, config=\"raw_mysql.json\", timeout=1500):\n import os\n if \"/\" in config:\n cfg = util.read_config(config)\n else:\n cfg = util.read_config(\n os.path.join(os.getcwd(), \"config\", config)\n )\n host = cfg[\"host\"]\n port = cfg[\"port\"]\n user = cfg[\"user\"]\n password = cfg[\"password\"]\n db = cfg[\"db\"]\n try:\n self.logger.info(\"Trying to connect to MySQL-db\")\n db = pymysql.connect(host, user, password, db)\n cursor = db.cursor()\n cursor.execute(\"SELECT VERSION()\")\n cursor.fetchone()\n self.logger.info(\"Successfully connected to MySQL\")\n return db\n except:\n self.logger.warning(\n \">>>>>>>>>>>>>>>>>>Failed to connect to MySQL<<<<<<<<<<<<<<<<<<<\")\n return False\n\n def get_redis(self, config=\"redis.json\"):\n import os\n if \"/\" in config:\n cfg = util.read_config(config)\n else:\n cfg = util.read_config(\n os.path.join(os.getcwd(), \"config\", config)\n )\n host = cfg[\"host\"]\n port = cfg[\"port\"]\n\n try:\n self.logger.info(\"Trying to connect to redis\")\n self.redis = redis.StrictRedis(\n host=host,\n port=port\n )\n self.redis.client_list()\n self.logger.info(\"Successfully connected to redis\")\n return self.redis\n except:\n self.logger.warning(\"Failed to connect to redis\")\n return False\n" }, { "path": "HySAS/Vendor/WorkerManager/WorkerManager.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\n# Created on\n# @author:\n# @contact:\n\"\"\"\n# 以下是自动生成的 #\n# --- 导入系统配置\nimport core.util as util\nfrom core.Vendor import Vendor\n# --- 导入自定义配置\n# 以上是自动生成的 #\nimport time\nimport pickle\nfrom core.Functions import get_vendor\nimport copy\n\n\nclass WorkerManager(Vendor):\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.redis = get_vendor(\"DB\").get_redis()\n self.worker_names = list()\n self.worker_info = dict()\n self.updating = False\n if \"auto_remove_terminated\" in kwargs:\n self.auto_remove_terminated = kwargs[\"auto_remove_terminated\"]\n else:\n self.auto_remove_terminated = -1\n # -1: never;\n # otherwise: auto remove terminated workers\n # after (self.auto_remove_terminated) seconds\n\n def get_worker_names(self):\n worker_names = util.get_worker_names(logger=self.logger)\n return worker_names\n\n def get_workers_from_redis(self):\n keys = list(map(str,self.redis.keys(\"HySAS.Worker.*.*.Info\")))\n workers = dict()\n for i in range(len(keys)):\n channel = keys[i]\n parsed_channel = channel.split('.')\n if (len(parsed_channel) == 5) and (parsed_channel[4] == \"Info\") \\\n and (parsed_channel[0] == 'HySAS') \\\n and (parsed_channel[1] == 'Worker'):\n if parsed_channel[2] not in workers:\n workers[parsed_channel[2]] = dict()\n worker = self.redis.hgetall(keys[i])\n workers[parsed_channel[2]][parsed_channel[3]] = worker\n return workers\n\n def update_workers(self):\n \"\"\"\n 更新Worker信息\n :return:\n \"\"\"\n self.updating = True\n # 根据文件目录检索\n self.worker_names = self.get_worker_names()\n\n # 从redis中获取worker信息并更新\n worker_info = self.get_workers_from_redis()\n\n # 去除掉过期信息\n for worker_name, workers in worker_info.items():\n if worker_name not in self.worker_info:\n self.worker_info[worker_name] = dict()\n for nickname, worker_info in workers.items():\n heart_beat_interval = 1\n difference = float(\"inf\")\n if \"heart_beat_interval\" in worker_info:\n heart_beat_interval = int(worker_info[\"heart_beat_interval\"])\n try:\n difference = time.time() - float(worker_info[\"heart_beat\"])\n except ValueError:\n self.logger.error(\"时间错误\")\n\n if difference > heart_beat_interval+1:\n # 已经停止\n if worker_info[\"status\"] == \"started\":\n worker_info[\"status\"] = \"disappeared\"\n self.logger.warning(\n \"发现一个没有正常关闭的Worker:\\n\"\n \"\\tworker_name: {}\\n\"\n \"\\tnickname: {}\"\n .format(worker_name, nickname)\n )\n self.redis.hmset(\"HySAS.Worker.\"+worker_name+\".\"+nickname+\".Info\", worker_info)\n self.worker_info[worker_name][nickname] = copy.deepcopy(worker_info)\n if self.auto_remove_terminated > -1 and\\\n difference> self.auto_remove_terminated+1:\n if \"token\" in worker_info:\n token = worker_info[\"token\"]\n else:\n token = None\n removed = self.remove_worker(\n worker_name=worker_name,\n nickname=nickname,\n token=token\n )\n if removed == 1:\n self.logger.info(\n \"从redis中自动移除过期Worker:\\n\"\n \"\\tworker_name:{}\\n\"\n \"\\tnickname:{}\\n\"\n \"\\ttoken:{}\\n\"\n .format(worker_name,nickname,token)\n )\n else:\n self.logger.warning(\n \"从redis中自动移除过期Worker:{}\\n\"\n \"\\tworker_name:{}\\n\"\n \"\\tnickname:{}\\n\"\n \"\\ttoken:{}\\n\"\n .format(\n removed,\n worker_name,\n nickname,\n token\n )\n )\n\n self.updating = False\n\n def remove_worker(\n self,\n worker_name,\n nickname,\n token\n ):\n key = \"HySAS.Worker.\"+worker_name+\".\"+nickname+\".\"+\"Info\"\n worker_info = self.redis.hgetall(key)\n if \"token\" in worker_info:\n if worker_info[\"token\"] == token:\n return self.redis.delete(key)\n else:\n if token is None:\n return self.redis.delete(key)\n return 0" }, { "path": "HySAS/Vendor/WorkerManager/__init__.py", "content": "" }, { "path": "HySAS/Vendor/WorkerManager/config.py", "content": "" }, { "path": "HySAS/Worker/Monitor/Controller.py", "content": "" }, { "path": "HySAS/Worker/Monitor/Monitor.py", "content": "from core.Worker import Worker\nfrom core.Functions import get_vendor, get_worker_class\nimport os\nimport signal\n\n\nclass Monitor(Worker):\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n def start_worker(self, worker_name, **kwargs):\n worker = get_worker_class(worker_name=worker_name, **kwargs)\n worker.start()\n\n def terminate_worker(self, nickname=None, pid=None):\n if pid is None:\n pid = self.get_pid_by_nickname(redis_cli=self.__redis__, )\n os.kill(pid, signal.SIGTERM)\n\n def __producer__(self):\n \"\"\"\n 定时更新redis中各个worker的状态\n :return:\n \"\"\"\n import time\n # wm: short for \"worker_manager\"\n self.wm = get_vendor(\n \"WorkerManager\",\n auto_remove_terminated=3600*24\n )\n while True:\n time.sleep(1)\n self.wm.update_workers()\n\n def get_workers_info(\n self,\n redis_cli=None,\n by=\"nickname\",\n nickname=None,\n worker_name=None\n ):\n if redis_cli is None:\n redis_cli = self.__redis__\n result = list()\n keys = list()\n if by == \"nickname\" and nickname is not None:\n keys = redis_cli.keys(\"dHydra.Worker.*.\" + nickname + \".Info\")\n elif by == \"worker_name\" and worker_name is not None:\n keys = redis_cli.keys(\"dHydra.Worker.\" + worker_name + \".*.Info\")\n for k in keys:\n result.append(redis_cli.hgetall(k))\n return result\n\n def get_pid_by_nickname(\n self,\n redis_cli=None,\n nickname=None\n ):\n if redis_cli is None:\n redis_cli = self.__redis__\n workers_info = self.get_workers_info(\n redis_cli=self.__redis__, nickname=nickname)\n if len(workers_info) == 1:\n return int(workers_info[0][\"pid\"].decode(\"utf-8\"))\n else:\n self.logger.warning(\"Worker is not Unique.\")\n return 0\n" }, { "path": "HySAS/Worker/Process/Process.py", "content": "\"\"\"\n数据处理函数\n\"\"\"\nfrom core.Worker import Worker\nfrom core.Functions import get_vendor\nfrom core import util\nimport sys\n\nclass Process(Worker):\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n\n def init_MySQLdb(self):\n self.raw_MySQL = get_vendor(\"DB\").get_MySQLdb(\"raw_mysql.json\")\n if self.raw_MySQL is False:\n self.logger.error(\"Cannot connect to raw_MySQLdb\")\n return False\n self.event_MySQL = get_vendor(\"DB\").get_MySQLdb(\"event_mysql.json\")\n if self.event_MySQL is False:\n self.logger.error(\"Cannot connect to event_MySQLdb\")\n return False\n\n def on_start(self):\n self.init_MySQLdb()\n\n def __producer__(self): #拥有数据库句柄 通过 self.db调用\n \"\"\"\n 取数据\n :return: msg\n \"\"\"\n msg = \"data\"\n self.__data_handler__(msg)\n pass\n\n def __data_handler__(self, msg):\n print(msg)\n print(\"======\")\n pass\n" }, { "path": "HySAS/config/event_mysql.json", "content": "{\n \"host\": \"127.0.0.1\",\n \"port\": 6379,\n \"user\": \"\",\n \"password\": \"\",\n \"db\": \"\"\n}\n" }, { "path": "HySAS/config/raw_mysql.json", "content": "{\n \"host\": \"127.0.0.1\",\n \"port\": 3306,\n \"user\": \"root\",\n \"password\": \"123\",\n \"db\": \"test\"\n}\n" }, { "path": "HySAS/config/redis.json", "content": "{\n \"host\": \"127.0.0.1\",\n \"port\": 6379\n}\n" }, { "path": "HySAS/console.py", "content": "# -*- coding: utf-8 -*-\nimport logging\nfrom core.Functions import *\nimport core.util as util\nimport click\nimport traceback\nimport pickle\n\n\ndef init_logger():\n return util.get_logger(logger_name=\"console\")\n\n\n@click.command()\n@click.argument('worker_name', nargs=1)\n@click.argument('nickname', nargs=1)\ndef start(worker_name=None, nickname=None):\n \"\"\"\n 主要依靠nickname来指定配置文件，通过配置文件传入参数\n\n :param worker_name:\n :param nickname:\n 如果nickname以.json结尾，那么程序会去\"./config/\"目录下寻找对应配置文件\n 如果不是以.json结尾，则认为传入的就是nickname，同时会去\"./config/\"目录下寻找\n nickname.json\n :return:\n \"\"\"\n kwargs = dict()\n kwargs[\"worker_name\"] = worker_name\n if nickname[-5:] == \".json\":\n kwargs[\"config\"] = nickname\n kwargs[\"nickname\"] = nickname[0:-5]\n else:\n kwargs[\"config\"] = nickname + \".json\"\n kwargs[\"nickname\"] = nickname\n config_file = os.path.join(os.getcwd(), \"config\", kwargs[\"config\"])\n if os.path.exists(config_file):\n config = util.read_config(config_file)\n else:\n # 命令指定了配置文件，如果不存在，就结束\n print(\"未能找到配置文件：\", config_file)\n print(\"默认无配置启动\")\n config = {}\n\n for k in config.keys():\n kwargs[k] = config[k]\n\n msg = {\n \"type\": \"sys\",\n \"operation_name\": \"start_worker\",\n \"kwargs\": kwargs\n }\n __redis__.publish(\"HySAS.Command\", pickle.dumps(msg))\n\n\n# @click.command()\n# @click.argument('nickname', nargs=1)\ndef stop(nickname=None):\n msg = {\"type\": \"sys\",\n \"operation_name\": \"terminate_worker\",\n \"kwargs\": {\"nickname\": nickname}\n }\n if nickname is not None:\n msg[\"kwargs\"][\"nickname\"] = nickname\n __redis__.publish(\"HySAS.Command\", pickle.dumps(msg))\n\n\ndef shutdown(nickname=None):\n msg = {\"type\": \"sys\",\n \"func\": \"shutdown\"\n }\n __redis__.publish(\"HySAS.Command\", pickle.dumps(msg))\n\n\ndef send_command(\n channel_name=\"HySAS.Command\",\n command_type=\"sys\",\n operation_name=None,\n token=None,\n kwargs={}\n):\n if operation_name is not None:\n command = {\n \"type\": command_type,\n \"operation_name\": operation_name,\n \"token\": token,\n \"kwargs\": kwargs\n }\n __redis__.publish(channel_name, pickle.dumps(command))\n\nlogger = init_logger()\n__redis__ = get_vendor(\"DB\").get_redis()\n\n" }, { "path": "HySAS/core/Functions.py", "content": "\"\"\"\n框架的全局方法，在主程序运行时会被引用\n---\nCreated on 01/14/2019\n@author: Happy-zyy\n@contact: 617532750@qq.com\n\"\"\"\nimport importlib\nimport sys\nimport os\nimport logging\nimport traceback\n\ndef get_workers():\n candidates = set(os.listdir(\"./Worker\")) | set(os.listdir(\n os.path.split(os.path.realpath(__file__))[0][:-4] + \"Worker\"))\n candidates = list(candidates)\n workers = list()\n for item in candidates:\n if (item[0] >= \"A\" and item[0] <= \"Z\"):\n workers.append(item)\n return workers\n\n\"\"\"\n动态加载web controller\n\"\"\"\n\n\ndef get_controller_method(class_name, method):\n logger = logging.getLogger(\"Functions\")\n # get instance of controller\n if os.path.exists(\n os.getcwd() + \"/Worker/\" +\n class_name + \"/\" + \"Controller.py\"\n ):\n func = getattr(\n importlib.import_module(\"Worker.\" + class_name + \".Controller\"),\n method\n )\n return func\n else:\n try:\n func = getattr(importlib.import_module(\n \"dHydra.Worker.\" + class_name + \".Controller\"),\n method\n )\n return func\n except Exception as e:\n return False\n\n\ndef V(name, vendor_name=None, **kwargs):\n return get_vendor(name=name, vendor_name=None, **kwargs)\n\n\ndef get_vendor(name, vendor_name=None, **kwargs):\n \"\"\"\n get_vendor方法，动态加载vendor类\n \"\"\"\n logger = logging.getLogger('Functions')\n if vendor_name is None:\n vendor_name = \"V-\" + name\n class_name = name\n module_name = 'Vendor.' + name + '.' + class_name\n if os.path.exists(\n os.path.join(os.getcwd(), \"Vendor\", name, class_name + \".py\")\n ):\n try:\n instance = getattr(\n __import__(module_name, globals(), locals(), [class_name], 0),\n class_name\n )(**kwargs)\n except ImportError:\n traceback.print_exc()\n else:\n try:\n instance = getattr(\n __import__(\n \"HySAS.\" + module_name,\n globals(),\n locals(),\n [class_name],\n 0\n ),\n class_name\n )(**kwargs)\n except ImportError:\n traceback.print_exc()\n return instance\n\n\ndef get_worker_class(worker_name, **kwargs):\n logger = logging.getLogger('Functions')\n module_name = 'Worker.' + worker_name + '.' + worker_name\n if os.path.exists(\n os.getcwd() + \"/Worker/\" +\n worker_name + \"/\" + worker_name + \".py\"\n ):\n try:\n if module_name in sys.modules:\n del sys.modules[module_name]\n return getattr(\n __import__(module_name, globals(), locals(), [worker_name], 0),\n worker_name\n )(**kwargs)\n except ImportError:\n traceback.print_exc()\n else:\n try:\n if \"HySAS.\"+module_name in sys.modules:\n del sys.modules[\"HySAS.\"+module_name]\n return getattr(\n __import__(\n \"HySAS.\" +\n module_name,\n globals(),\n locals(),\n [worker_name],\n 0\n ),\n worker_name\n )(**kwargs)\n except ImportError:\n traceback.print_exc()\n\ndef convert(data):\n if isinstance(data, bytes): return data.decode('ascii')\n if isinstance(data, dict): return dict(map(convert, data.items()))\n if isinstance(data, tuple): return map(convert, data)\n return data" }, { "path": "HySAS/core/Vendor.py", "content": "# -*- coding: utf-8 -*-\nimport logging\nimport core.util as util\n\n\nclass Vendor:\n\n def __init__(\n self,\n log_path=\"log\", #\n console_log=True, # 屏幕打印日志开关，默认True\n console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO\n critical_log=False, # critica单独l写文件日志，默认关闭\n error_log=True, # error级别单独写文件日志，默认开启\n warning_log=False, # warning级别单独写日志，默认关闭\n info_log=True, # info级别单独写日志，默认开启\n debug_log=False, # debug级别日志，默认关闭\n **kwargs\n ):\n self.logger = util.get_logger(\n log_path=log_path, #\n console_log=console_log, # 屏幕打印日志开关，默认True\n console_log_level=console_log_level, # 屏幕打印日志的级别，默认为INFO\n critical_log=critical_log, # critica单独l写文件日志，默认关闭\n error_log=error_log, # error级别单独写文件日志，默认开启\n warning_log=warning_log, # warning级别单独写日志，默认关闭\n info_log=info_log, # info级别单独写日志，默认开启\n debug_log=debug_log, # debug级别日志，默认关闭\n logger_name=self.__class__.__name__,\n )\n" }, { "path": "HySAS/core/Worker.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nWorker抽象类\nCreated on 01/14/2019\n@author: Happy-zyy\n@contact: 617532750@qq.com\n\"\"\"\nimport multiprocessing\nimport threading\nimport time\nimport logging\nimport traceback\nimport redis\nimport json\nimport copy\nimport core.util as util\nfrom console import *\nfrom datetime import datetime\nfrom datetime import timedelta\nfrom abc import ABCMeta\nimport signal\nimport sys\nimport os\nimport ast\nimport pickle\n\nclass Worker(multiprocessing.Process):\n __metaclass__ = ABCMeta\n\n def __init__(\n self,\n singleton=True, # 单例模式\n nickname=None, # Worker的自定义名字\n description=\"No Description\", # 备注说明\n heart_beat_interval=1, # 默认1秒心跳\n log_path=\"log\", #\n console_log=True, # 屏幕打印日志开关，默认True\n console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO\n critical_log=False, # critical单独写文件日志，默认关闭\n error_log=True, # error级别单独写文件日志，默认开启\n warning_log=False, # warning级别单独写日志，默认关闭\n info_log=True, # info级别单独写日志，默认开启\n debug_log=False, # debug级别日志，默认关闭\n **kwargs\n ):\n super().__init__()\n # 记录日志配置\n self.__log_path__ = log_path\n self.__console_log__ = console_log\n self.__console_log_level__ = console_log_level\n self.__critical_log__ = critical_log\n self.__error_log__ = error_log\n self.__warning_log__ = warning_log\n self.__info_log__ = info_log\n self.__debug_log__ = debug_log\n\n self.logger = util.get_logger(\n logger_name=self.__class__.__name__,\n log_path=self.__log_path__,\n console_log=self.__console_log__, # 屏幕打印日志开关，默认True\n console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO\n critical_log=self.__critical_log__, # critica单独l写文件日志，默认关闭\n error_log=self.__error_log__, # error级别单独写文件日志，默认开启\n warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭\n info_log=self.__info_log__, # info级别单独写日志，默认开启\n debug_log=self.__debug_log__, # debug级别日志，默认关闭\n )\n\n self.__token__ = util.generate_token()\n if nickname is None:\n self.__nickname__ = self.__class__.__name__\n else:\n self.__nickname__ = nickname\n self.nickname = self.__nickname__\n self.name = self.__nickname__\n self.__singleton__ = singleton\n self.__description__ = description\n self.__heart_beat_interval__ = heart_beat_interval\n self.__threads__ = dict() # 被监控的线程\n self.__data_feeder__ = set() # 本Worker订阅的内容\n self.__follower__ = set() # Follower\n self.__error_msg__ = None #\n self.__stop_info__ = None #\n self.__stop_time__ = None #\n self.__status__ = \"init\"\n\n self.redis_key = \"HySAS.Worker.\" + \\\n self.__class__.__name__ + \".\" + self.__nickname__ + \".\"\n self.channel_pub = self.redis_key + \"Pub\"\n \"\"\"\n self.__threads__ = {\n \"nickname\": {\n \"description\"\t: \"该线程功能备注说明\",\n \"name\"\t\t\t: \"该线程的名字\",\n \"target\"\t\t: \"该线程的target\"\n \"restart_mode\"\t: \"重启模式，可以为 manual/auto/remove;\n manual则代表允许管理员发送命令手工重启线程,\n auto则一旦线程关闭立即自动开启，\n remove则代表一旦线程结束就从监控列表移除\",\n \"restart_func\"\t: \"自动/手动重启时调用的方法\",\n },\n }\n \"\"\"\n\n self.shutdown_signals = [\n \"SIGQUIT\", # quit 信号\n \"SIGINT\", # 键盘信号\n \"SIGHUP\", # nohup 命令\n \"SIGTERM\", # kill 命令\n ]\n for s in self.shutdown_signals:\n # 捕获退出信号后的要调用的,唯一的 shutdown 接口\n try:\n if hasattr(signal, s):\n signal.signal(\n getattr(signal, s, None),\n self.__on_termination__\n )\n except Exception as e:\n self.logger.info(\"绑定退出信号：{}失败，可能与windows系统有关。\".format(s))\n\n # 清空它，在run以后重新实例化\n # 否则windows下会无法pickle\n del(self.logger)\n\n def __is_unique__(self):\n info = self.__redis__.hgetall(self.redis_key + \"Info\")\n if \"token\" in info:\n if info[\"token\"] != self.__token__:\n if \"heart_beat\" in info:\n if (time.time() - float(info[\"heart_beat\"])) < self.__heart_beat_interval__:\n return False\n return True\n\n def __auto_restart_thread__(self):\n # Worker内置的默认自动重启线程方法\n pass\n\n def __command_handler__(self, msg_command):\n # cli is a dict with the following structure:\n \"\"\"\n msg_command = {\n \"type\"\t:\t\t\"sys/customized\",\n \"operation\"\t:\t\"operation_name\",\n \"kwargs\"\t:\t\"suppose that the operation is a function,\n we need to pass some arguments\",\n \"token\"\t\t:\t\"the token is used to verify the authentication\n of the operation\"\n }\n \"\"\"\n try:\n msg_command = pickle.loads(msg_command)\n if not isinstance(msg_command, dict):\n return\n except Exception as e:\n traceback.print_exc()\n print(e)\n if msg_command[\"type\"] == \"sys\":\n if hasattr(self, msg_command[\"operation_name\"]):\n func = getattr(self, msg_command[\"operation_name\"])\n try:\n print(msg_command[\"kwargs\"])\n result = func(**msg_command[\"kwargs\"])\n except Exception as e:\n traceback.print_exc()\n self.logger.error(e)\n\n def monitor_add_thread(self,\n thread,\n description=\"No Description\",\n restart_mode=\"manual\",\n restart_func=None):\n # 将该线程加入管理员监控范围\n pass\n\n def monitor_remove_thread(self, thread):\n # 取消管理员对线程thread的监控\n pass\n\n def init_redis(self):\n # 检测redis, sqldb连接\n try:\n self.__redis__ = get_vendor(\"DB\").get_redis()\n #self.__redis__.client_list()\n self.__listener__ = self.__redis__.pubsub()\n self.__listener__.subscribe([\"HySAS\"])\n except redis.ConnectionError:\n self.logger.error(\"Cannot connect to redis\")\n return False\n\n\n def check_prerequisites(self):\n \"\"\"\n 检查是否满足开启进程的条件\n \"\"\"\n # 如果是单例，检测是否重复开启\n return True\n\n def __listen_command__(self):\n #\n self.command_listener = self.__redis__.pubsub()\n channel_name = self.redis_key + \"Command\"\n self.command_listener.subscribe([channel_name])\n while True:\n msg_command = self.command_listener.get_message()\n if msg_command:\n if (msg_command[\"type\"] == \"message\") or \\\n (msg_command[\"type\"] == \"pmessage\"):\n self.__command_handler__(msg_command[\"data\"])\n else:\n time.sleep(0.5)\n\n def __heart_beat__(self):\n # flush status infomation to redis\n status = dict()\n status[\"heart_beat\"] = time.time()\n status[\"nickname\"] = self.__nickname__\n status[\"pid\"] = self.pid\n status[\"token\"] = self.__token__\n status[\"heart_beat_interval\"] = self.__heart_beat_interval__\n if self.__error_msg__:\n status[\"error_msg\"] = self.__error_msg__\n if self.__stop_info__:\n status[\"stop_info\"] = self.__stop_info__\n if self.__stop_info__:\n status[\"stop_time\"] = self.__stop_time__\n if self.__status__:\n status[\"status\"] = self.__status__\n if self.__threads__:\n status[\"threads\"] = copy.deepcopy(self.__threads__)\n if self.__data_feeder__:\n status[\"data_feeder\"] = self.__data_feeder__\n if self.__follower__:\n status[\"follower\"] = self.__follower__\n self.__redis__.hmset(self.redis_key + \"Info\", status)\n\n def __producer__(self):\n \"\"\"\n 在子类中被重写的用以作为生产者的线程\n 若不重写，线程启动后就结束了\n \"\"\"\n pass\n\n def __consumer__(self):\n \"\"\"\n 默认的消费者线程\n 随着Worker进程的start而启动\n \"\"\"\n pass\n # while True:\n # data = self.__listener__.get_message(timeout=10)\n # if data is not None:\n # self.__data_handler__(data)\n # time.sleep(1)\n\n # 需要在子类中重写的数据处理方法\n def __data_handler__(self, msg):\n \"\"\"\n 需要在子类中被重写的用以处理数据的方法，\n 接受到的msg数据是原始的从Redis中监听到的数据\n \"\"\"\n print(\"*****\")\n pass\n\n def __before_termination__(self, sig):\n self.logger.info(\"收到了退出信号。进程类：{}，进程名：{},进程pid:{}\"\n .format(\n self.__class__.__name__,\n self.__nickname__,\n self.pid\n )\n )\n\n def __on_termination__(self, sig, frame):\n self.__before_termination__(sig)\n self.__status__ = \"terminated\"\n self.__heart_beat__() # The last heart_beat, sad...\n sys.exit(0)\n\n def publish(self,data,channel_name=None):\n if channel_name is None:\n channel_name = self.channel_pub\n # publish data to redis\n try:\n self.__redis__.publish(channel_name, data)\n except Exception as e:\n self.logger.warning(e)\n\n def __on_start__(self):\n \"\"\"\n 进程开始运行时调用\n :return:\n \"\"\"\n if self.check_prerequisites() is not True:\n sys.exit(0)\n\n self.init_redis()\n\n # 实例化self.logger\n self.logger = util.get_logger(\n logger_name=self.__class__.__name__,\n log_path=self.__log_path__, #\n console_log=self.__console_log__, # 屏幕打印日志开关，默认True\n console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO\n critical_log=self.__critical_log__, # critical写文件日志，默认关闭\n error_log=self.__error_log__, # error级别单独写文件日志，默认开启\n warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭\n info_log=self.__info_log__, # info级别单独写日志，默认开启\n debug_log=self.__debug_log__, # debug级别日志，默认关闭\n )\n\n def on_start(self):\n pass\n\n def run(self):\n \"\"\"\n 初始化Worker\n \"\"\"\n self.__on_start__()\n self.logger.info(\"初始化{}\".format(self.__class__.__name__))\n # 用户自定义的on_start\n self.on_start()\n\n # 首先检查是否已经有相同的进程被开启\n if self.__is_unique__():\n self.__status__ = \"started\"\n else:\n self.error_msg = \"Duplicated Process\"\n self.logger.warning(self.error_msg)\n sys.exit(0)\n\n # 开启监听命令线程\n self.__thread_listen_command__ = threading.Thread(\n target=self.__listen_command__\n )\n self.__thread_listen_command__.setDaemon(True)\n self.monitor_add_thread(\n thread=self.__thread_listen_command__,\n description=\"Listening Command Channel\",\n restart_mode=\"auto\",\n restart_func=self.__auto_restart_thread__\n )\n self.__thread_listen_command__.start()\n\n # 检查初始化设置，按需开启\n # PUB线程\n self.__thread_pub__ = threading.Thread(target=self.__producer__)\n self.__thread_pub__.setDaemon(True)\n self.monitor_add_thread(\n thread=self.__thread_pub__,\n description=\"DATA PUBLISHER\",\n restart_mode=\"auto\",\n restart_func=self.__auto_restart_thread__\n )\n self.__thread_pub__.start()\n\n # LISTENER\n self.__thread_sub__ = threading.Thread(target=self.__consumer__)\n self.__thread_sub__.setDaemon(True)\n self.monitor_add_thread(\n thread=self.__thread_sub__,\n description=\"DATA CONSUMER\",\n restart_mode=\"auto\",\n restart_func=self.__auto_restart_thread__\n )\n self.__thread_sub__.start()\n\n while True:\n # heart beat\n self.__heart_beat__()\n time.sleep(self.__heart_beat_interval__)\n\n\n def subscribe(self,channel_name=None, worker_name=None, nickname=None):\n \"\"\"\n 订阅Worker\n 可以直接填入channel_name\n 也可以通过填入worker_name/nickname/worker_name+nickname来自动订阅对应的内容\n \"\"\"\n if channel_name is not None:\n self.__listener__.subscribe(channel_name)\n return None\n if (worker_name is not None) and (nickname is None):\n # 订阅所有此类Worker\n self.__listener__.psubscribe(\n \"HySAS.Worker.\" + worker_name + \".*.Pub\"\n )\n self.logger.info(\n \"About to subscribe the Worker of worker_name: {}, pattern:{}\"\n .format(\n worker_name,\n \"HySAS.Worker.\" + worker_name + \".*.Pub\"\n )\n )\n elif (worker_name is not None) and (nickname is not None):\n channel_name = \"HySAS.Worker.\" \\\n + worker_name + \".\" + nickname + \".Pub\"\n self.__listener__.subscribe(channel_name)\n self.logger.info(\n \"Subscribed: {}\".format(channel_name)\n )\n elif (nickname is not None):\n # 订阅nickname\n self.__listener__.psubscribe(\n \"HySAS.Worker.*.\" +\n nickname + \".Pub\"\n )\n self.logger.info(\n \"About to subscribe the Worker of nickname: {}, pattern:{}\"\n .format(\n nickname,\n \"HySAS.Worker.*.\" +\n nickname +\n \".Pub\"\n )\n )\n else:\n self.logger.warning(\"nickname/worker_name的输入方式不合理\")\n\n def unsubscribe(self, worker_name=None, nickname=None):\n \"\"\"\n 退订Worker\n \"\"\"\n if (worker_name is not None) and (nickname is None):\n # 订阅所有此类Worker\n self.__listener__.punsubscribe(\n \"HySAS.Worker.\" + worker_name + \".*.Pub\")\n self.logger.info(\n \"About to unsubscribe the Worker of worker_name: {}, \\\n pattern:{}\"\n .format(\n nickname,\n \"HySAS.Worker.*.\" + worker_name + \".Pub\"\n )\n )\n pass\n elif (nickname is not None):\n # 订阅nickname\n self.__listener__.punsubscribe(\n \"HySAS.Worker.*.\" + nickname + \".Pub\"\n )\n self.logger.info(\n \"About to subscribe the Worker of nickname: {}, pattern:{}\"\n .format(\n nickname,\n \"HySAS.Worker.*.\" + nickname + \".Pub\"\n )\n )\n else:\n self.logger.warning(\"nickname/worker_name的输入方式不合理\")\n\n" }, { "path": "HySAS/core/log/DB/error.log", "content": "" }, { "path": "HySAS/core/log/DB/info.log", "content": "" }, { "path": "HySAS/core/log/console/error.log", "content": "" }, { "path": "HySAS/core/log/console/info.log", "content": "" }, { "path": "HySAS/core/util.py", "content": "import logging\nimport os\nimport hashlib\nimport time\nimport json\n\ndef get_worker_names(logger=None):\n \"\"\"\n 根据文件夹名字返回所有可能的worker_name\n :return:\n \"\"\"\n worker_names = []\n # path = os.path.split(os.path.realpath(__file__))[0][:-6]+\"/Worker\"\n # worker_names.extend(os.listdir(path))\n try:\n worker_names.extend(os.listdir(os.getcwd()+\"/Worker\"))\n except FileNotFoundError as e:\n if logger is None:\n print(\"HySAS运行目录下没有Worker文件夹\")\n else:\n logger.warning(\"HySAS运行目录下没有Worker文件夹\")\n\n return worker_names\n\ndef get_logger(\n logger_name=\"main\",\n log_path=\"log\", #\n console_log=True, # 屏幕打印日志开关，默认True\n console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO\n critical_log=False, # critical单独写文件日志，默认关闭\n error_log=True, # error级别单独写文件日志，默认开启\n warning_log=False, # warning级别单独写日志，默认关闭\n info_log=True, # info级别单独写日志，默认开启\n debug_log=False, # debug级别日志，默认关闭\n):\n formatter = logging.Formatter(\n '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s'\n )\n logger = logging.getLogger(logger_name)\n logger.setLevel(logging.DEBUG)\n\n if log_path:\n # 补全文件夹\n if log_path[-1] != '/':\n log_path += '/'\n\n if not logger.handlers:\n # 屏幕日志打印设置\n if console_log:\n console_handler = logging.StreamHandler()\n console_handler.setFormatter(formatter)\n console_handler.setLevel(logging.INFO)\n logger.addHandler(console_handler)\n\n if not os.path.exists(log_path + logger_name):\n os.makedirs(log_path + logger_name)\n # 打开下面的输出到文件\n if critical_log:\n log_handler = logging.FileHandler(\n log_path + logger_name + '/critical.log'\n )\n log_handler.setLevel(logging.CRITICAL)\n log_handler.setFormatter(formatter)\n logger.addHandler(log_handler)\n if error_log:\n log_handler = logging.FileHandler(\n log_path + logger_name + '/error.log'\n )\n log_handler.setLevel(logging.ERROR)\n log_handler.setFormatter(formatter)\n logger.addHandler(log_handler)\n if warning_log:\n log_handler = logging.FileHandler(\n log_path + logger_name + '/warning.log'\n )\n log_handler.setLevel(logging.WARNING)\n log_handler.setFormatter(formatter)\n logger.addHandler(log_handler)\n if info_log:\n log_handler = logging.FileHandler(\n log_path + logger_name + '/info.log'\n )\n log_handler.setLevel(logging.INFO)\n log_handler.setFormatter(formatter)\n logger.addHandler(log_handler)\n if debug_log:\n log_handler = logging.FileHandler(\n log_path + logger_name + '/debug.log'\n )\n log_handler.setLevel(logging.DEBUG)\n log_handler.setFormatter(formatter)\n logger.addHandler(log_handler)\n return logger\n\n\ndef generate_token():\n token = hashlib.sha1()\n token.update(str(time.time()).encode())\n token = token.hexdigest()\n return token\n\ndef read_config(file_path):\n # 读取配置\n try:\n f_config = open(file_path)\n cfg = json.load(f_config)\n except Exception as e:\n print(\"{}\".format(e))\n cfg = dict()\n print(\n \"未能正确加载{}，请检查路径，json文档格式，或者忽略此警告\"\n .format(file_path)\n )\n return cfg\n" }, { "path": "HySAS/log/DB/error.log", "content": "" }, { "path": "HySAS/log/DB/info.log", "content": "2019-01-15 10:38:23,552 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:38:25,562 - DB - 68 - WARNING - Failed to connect to redis\n2019-01-15 10:49:10,014 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:55:46,379 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:55:49,677 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:56:19,580 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:56:45,196 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:57:28,660 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:57:31,169 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:12:30,375 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:12:38,731 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:17:00,225 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:17:03,378 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:19:08,291 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:19:14,068 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:20:30,782 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:20:50,996 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:27:14,317 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:27:17,802 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:28:06,379 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:33:25,918 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:33:28,955 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:34:53,982 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:34:56,690 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:35:04,770 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:35:24,233 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:36:02,853 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:36:11,119 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 11:36:17,741 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:55:08,689 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:55:30,131 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:55:58,190 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:56:45,528 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:57:11,346 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:57:57,146 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 14:58:13,734 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:00:27,200 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:01:49,462 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:01:49,480 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:02:13,434 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:10:44,988 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:10:45,020 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:36:17,193 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:36:17,237 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:36:17,704 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 15:36:17,717 - DB - 59 - INFO - Trying to connect to redis\n2019-01-16 11:23:18,509 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:18,517 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:18,668 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:19,312 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:19,318 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:19,326 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:44,698 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:44,706 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:44,735 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:45,256 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:45,263 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:45,270 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:23:45,272 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:25:34,629 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:40,779 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,228 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,235 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,271 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,794 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,801 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,808 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:25:59,809 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:26:41,487 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:41,494 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:41,548 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,183 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,190 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,197 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,199 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:26:42,256 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,263 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,292 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:26:42,294 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:59:43,416 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:43,423 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:43,453 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:44,014 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:44,022 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 11:59:44,035 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 12:00:42,046 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,053 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,082 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,633 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,637 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,640 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,644 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,648 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,650 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 12:00:42,651 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 12:00:42,652 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 12:00:42,653 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 12:00:42,655 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:05:27,277 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:05:40,119 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:07:35,467 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:08:00,707 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:08:57,073 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:09:44,100 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:12:09,237 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:14:16,598 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:16:39,681 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:17:03,198 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:19:56,705 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:20:22,377 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:21:44,117 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:22:49,185 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:24:28,167 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:37:33,920 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:53,691 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:53,701 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:53,748 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,371 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,378 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,385 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,386 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:39:54,392 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:39:54,409 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,417 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,426 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:39:54,428 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:39:54,431 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:44:17,721 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:17,729 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:17,765 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,326 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,333 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,334 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:44:18,335 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:45:06,377 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,384 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,419 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,964 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,965 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,980 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,981 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:45:06,982 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:45:06,983 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:45:06,984 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:45:06,986 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:53:47,231 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,239 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,271 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,836 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:53:47,852 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:53:47,854 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:54:42,455 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:54:56,807 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:20,724 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:20,731 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:20,770 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,360 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,367 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,368 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,374 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,375 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:56:21,376 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:56:21,379 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:56:21,402 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:47,471 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:47,478 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:47,507 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,083 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,090 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,097 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,099 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:58:48,101 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 14:58:48,104 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,109 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,112 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,120 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 14:58:48,122 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 14:58:48,124 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 15:14:27,070 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,077 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,112 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,672 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,679 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 15:14:27,691 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 15:14:27,698 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,140 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,148 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,187 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,771 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:22:55,788 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 15:22:55,798 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:52:49,239 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:53:33,871 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:53:48,123 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:53:55,740 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:54:53,430 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:55:31,092 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:58:03,617 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:58:33,713 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 15:58:50,782 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:01:24,014 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:01:43,708 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:01:56,358 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:02:52,853 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:03:15,355 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:41,913 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:41,920 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:41,950 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,580 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,587 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,590 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,594 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,596 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:06:42,597 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,602 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:06:42,605 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:06:42,607 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:06:42,609 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:06:42,618 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:09,805 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:09,811 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:09,841 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,416 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:07:10,426 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:58,732 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:58,739 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:58,768 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:59,340 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:59,347 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:15:59,359 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:15:59,379 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,013 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,020 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,051 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,610 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,617 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,618 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,626 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:30:05,629 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:30:05,632 - DB - 61 - INFO - Trying to connect to redis\n2019-01-16 16:30:05,635 - DB - 37 - INFO - ӵMySQL-db\n2019-01-16 16:30:05,637 - DB - 42 - INFO - ѾɹӵMySQL\n2019-01-16 16:30:05,643 - DB - 61 - INFO - Trying to connect to redis\n" }, { "path": "HySAS/log/Monitor/error.log", "content": "" }, { "path": "HySAS/log/Monitor/info.log", "content": "2019-01-16 11:23:45,365 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 11:25:59,817 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 11:26:42,206 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 11:59:44,056 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 12:00:42,652 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 14:39:54,393 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 14:44:18,339 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 14:45:06,985 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 14:53:47,855 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 14:56:21,380 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 14:58:48,102 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 15:14:27,692 - Monitor - 320 - INFO - ʼWorker\n2019-01-16 15:22:55,791 - Monitor - 327 - INFO - ʼWorker\n2019-01-16 16:06:42,603 - Monitor - 327 - INFO - ʼWorker\n2019-01-16 16:07:10,421 - Monitor - 327 - INFO - ʼWorker\n2019-01-16 16:15:59,359 - Monitor - 327 - INFO - ʼWorker\n2019-01-16 16:30:05,630 - Monitor - 327 - INFO - ʼWorker\n" }, { "path": "HySAS/log/Process/error.log", "content": "" }, { "path": "HySAS/log/Process/info.log", "content": "2019-01-16 11:26:42,301 - Process - 326 - INFO - ʼWorker\n2019-01-16 11:59:44,056 - Process - 326 - INFO - ʼWorker\n2019-01-16 12:00:42,656 - Process - 326 - INFO - ʼWorker\n2019-01-16 14:39:54,432 - Process - 320 - INFO - ʼWorker\n2019-01-16 14:44:18,338 - Process - 320 - INFO - ʼWorker\n2019-01-16 14:45:06,986 - Process - 320 - INFO - ʼWorker\n2019-01-16 14:53:47,855 - Process - 320 - INFO - ʼWorker\n2019-01-16 14:56:21,380 - Process - 320 - INFO - ʼWorker\n2019-01-16 14:58:48,125 - Process - 320 - INFO - ʼWorker\n2019-01-16 15:14:27,692 - Process - 320 - INFO - ʼWorker\n2019-01-16 15:22:55,791 - Process - 327 - INFO - ʼWorker\n2019-01-16 16:06:42,610 - Process - 327 - INFO - ʼWorker\n2019-01-16 16:07:10,421 - Process - 327 - INFO - ʼWorker\n2019-01-16 16:15:59,359 - Process - 327 - INFO - ʼWorker\n2019-01-16 16:30:05,638 - Process - 327 - INFO - ʼWorker\n" }, { "path": "HySAS/log/WorkerManager/error.log", "content": "" }, { "path": "HySAS/log/WorkerManager/info.log", "content": "" }, { "path": "HySAS/log/console/error.log", "content": "2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n" }, { "path": "HySAS/log/console/info.log", "content": "2019-01-15 15:02:10,563 - console - 86 - INFO - Monitor has started\n2019-01-15 15:10:45,019 - console - 86 - INFO - Monitor has started\n2019-01-15 15:36:17,236 - console - 86 - INFO - Monitor has started\n2019-01-16 11:23:18,665 - console - 139 - INFO - Monitor has started\n2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:23:44,734 - console - 139 - INFO - Monitor has started\n2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:25:59,270 - console - 139 - INFO - Monitor has started\n2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:26:41,524 - console - 139 - INFO - Monitor has started\n2019-01-16 11:59:43,452 - console - 139 - INFO - Monitor has started\n2019-01-16 12:00:42,081 - console - 139 - INFO - Monitor has started\n2019-01-16 14:39:53,748 - console - 139 - INFO - Monitor has started\n2019-01-16 14:44:17,764 - console - 139 - INFO - Monitor has started\n2019-01-16 14:45:06,418 - console - 139 - INFO - Monitor has started\n2019-01-16 14:53:47,270 - console - 139 - INFO - Monitor has started\n2019-01-16 14:56:20,769 - console - 139 - INFO - Monitor has started\n2019-01-16 14:58:47,506 - console - 139 - INFO - Monitor has started\n2019-01-16 15:14:27,111 - console - 145 - INFO - Monitor has started\n2019-01-16 15:22:55,187 - console - 145 - INFO - Monitor has started\n2019-01-16 16:06:41,950 - console - 145 - INFO - Monitor has started\n2019-01-16 16:07:09,840 - console - 145 - INFO - Monitor has started\n2019-01-16 16:15:58,767 - console - 145 - INFO - Monitor has started\n2019-01-16 16:30:05,050 - console - 145 - INFO - Monitor has started\n" }, { "path": "HySAS/main.py", "content": "from console import logger\nfrom core.Functions import *\nimport time\nimport signal\nimport click\nimport pickle\n\n__redis__ = get_vendor(\"DB\").get_redis()\nworker_dict = dict()\n\n\ndef __on_termination__(sig, frame):\n logger.info(\"The HySAS Server is about to terminate, pid:{}\"\n .format(os.getpid())\n )\n sys.exit(0)\n\n\ndef bind_quit_signals():\n import signal\n shutdown_signals = [\n \"SIGQUIT\", # quit 信号\n \"SIGINT\", # 键盘信号\n \"SIGHUP\", # nohup 命令\n \"SIGTERM\", # kill 命令\n ]\n for s in shutdown_signals:\n if hasattr(signal, s):\n signal.signal(\n getattr(signal, s, None),\n __on_termination__\n )\n\n\ndef start_worker(worker_name, **kwargs):\n worker = get_worker_class(worker_name=worker_name, **kwargs)\n worker_dict[worker.nickname] = worker\n worker.start()\n\n\ndef terminate_worker(nickname=None, pid=None):\n logger.info(\"{}\".format(worker_dict))\n if pid is None:\n pid = get_pid_by_nickname(redis_cli=__redis__, nickname=nickname)\n os.kill(pid, signal.SIGTERM)\n i = 0\n while worker_dict[nickname]._popen is None and i < 30:\n time.sleep(0.1)\n i += 1\n worker_dict[nickname]._popen.wait(1)\n worker_dict.pop(nickname)\n\n\ndef get_workers_info(redis_cli=None, by=\"nickname\", nickname=None, worker_name=None):\n if redis_cli is None:\n redis_cli = __redis__\n result = list()\n keys = list()\n if by == \"nickname\" and nickname is not None:\n keys = redis_cli.keys(\"HySAS.Worker.*.\" + nickname + \".Info\")\n elif by == \"worker_name\" and worker_name is not None:\n keys = redis_cli.keys(\"HySAS.Worker.\" + worker_name + \".*.Info\")\n for k in keys:\n result.append(redis_cli.hgetall(k))\n return result\n\n\ndef get_pid_by_nickname(redis_cli=None, nickname=None):\n if redis_cli is None:\n redis_cli = __redis__\n workers_info = get_workers_info(redis_cli=__redis__, nickname=nickname)\n workers_info = list(map(convert, workers_info))\n if len(workers_info) == 1:\n return int(workers_info[0][\"pid\"])\n else:\n logger.warning(\"Worker is not Unique.\")\n return 0\n\n\n\n\ndef __command_handler__(msg_command):\n # msg_command is a dict with the following structure:\n \"\"\"\n msg_command = {\n \"type\"\t:\t\t\"sys/user/data\",\n \"operation\"\t:\t\"operation_name\",\n \"kwargs\"\t:\t\"suppose that the operation is a function, we need to\n pass some arguments\",\n \"token\"\t\t:\t\"the token is used to verify the authentication of the\n operation\"\n }\n \"\"\"\n import sys\n try:\n msg_command = pickle.loads(msg_command)\n if not isinstance(msg_command, dict):\n return\n except Exception as e:\n traceback.print_exc()\n print(\"msg_command is not dict \\n\",e)\n if msg_command[\"type\"] == \"sys\":\n if \"func\" in msg_command and msg_command[\"func\"] == \"shutdown\":\n signal_handler()\n elif hasattr(\n sys.modules[\"HySAS.main\"],\n msg_command[\"operation_name\"]\n ):\n func = getattr(\n sys.modules[\"HySAS.main\"],\n msg_command[\"operation_name\"]\n )\n try:\n print(msg_command[\"kwargs\"])\n result = func(**msg_command[\"kwargs\"])\n except Exception as e:\n traceback.print_exc()\n logger.error(e)\n\n elif msg_command[\"type\"] == \"user\":\n \"\"\"\n 用于接受前端用户数据\n \"\"\"\n pass\n\n elif msg_command[\"type\"] == \"data\":\n \"\"\"\n 用于接受关联分析时可能向外传输的交互信号\n \"\"\"\n pass\n\n\n\ndef signal_handler():\n for work_name in list(worker_dict.keys()):\n terminate_worker(work_name)\n print('Shutdown HySAS!')\n sys.exit(0)\n\n\n# @click.command()\n# @click.argument('what', nargs=-1)\ndef main(what=None):\n try:\n if what:\n if what[0] != \"HySAS\":\n print(\"Please input HySAS for starting\")\n exit(0)\n else:\n print(\n \"Welcome to HySAS! Following is the awesome!!!\"\n )\n logo = \"◆　　　　◆　　　　　　　　　　　　　　　　　　　　　　 ◆\\n\"\\\n\"◆　　　　◆　　　　　　　　　　　　　◆◆◆◆◆　　　　　◆◆　　　　　　◆◆◆◆◆\\n\"\\\n\"◆　　　　◆　　　　　　　　　　　　◆◆◆◆◆ 　　　　 ◆◆◆　　　　　◆◆◆\\n\"\\\n\"◆　　　　◆　　　◆◆　　◆◆　　　◆◆◆◆　　　　　　 ◆◆◆◆　　　　　　◆◆◆　　　　\\n\"\\\n\"◆◆◆◆◆◆　　　　◆◆　◆◆　　　　◆◆◆◆◆　　　　◆◆　◆◆　　　　　　◆◆◆◆◆　　\\n\"\\\n\"◆　　　　◆　　　　◆◆◆◆　　　　　　　　◆◆◆　　　◆◆◆◆◆◆　　　　　　　　◆◆◆　\\n\"\\\n\"◆　　　　◆　　　　　◆◆◆　　　◆◆◆　◆◆◆　　　 ◆◆　　◆◆◆　　　◆◆◆ ◆◆◆　　\\n\"\\\n\"◆　　　　◆　　　　　◆◆　　　　◆◆◆◆◆◆　　　　◆◆　　　◆◆◆　　　◆◆◆◆◆◆　　\\n\"\\\n\"　　　　　　　　　　　◆◆　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　\\n\"\\\n\"　　　　　　　　　　　◆◆　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　\\n\"\n\n print(logo)\n\n\n\n # open a thread for the Worker of Monitor\n start_worker(worker_name=\"Monitor\",nickname=\"Monitor\")\n logger.info(\"Monitor has started\")\n\n # 开启Web\n if len(what) == 1:\n # 没指定http端口，不开启Tornado\n pass\n else:\n port = int(what[1])\n start_worker(worker_name=\"Web\", nickname=\"Tornado\")\n\n # 绑定退出信号\n bind_quit_signals()\n\n redis_conn = get_vendor(\"DB\").get_redis()\n command_listener = redis_conn.pubsub()\n channel_name = \"HySAS.Command\"\n command_listener.subscribe([channel_name])\n\n # 数据关联 态势分析\n start_worker(worker_name=\"Process\",nickname=\"Process\")\n\n for item in command_listener.listen(): # 阻塞式接收\n if isinstance(item['data'], bytes):\n __command_handler__(item[\"data\"])\n\n else:\n print(\"HySAS What?\")\n except Exception as e:\n traceback.print_exc()\n logger.error(\"{}\".format(e))\n print(\"Hail error?\")\n\n\nif __name__ == '__main__':\n main(what = [\"HySAS\"])" }, { "path": "HySAS/test.py", "content": "from multiprocessing import Process\nimport threading\nimport time\nimport os\nimport redis\nimport pickle\nimport logging\n\nfrom HySAS.core.Functions import *\n\n\ndef __command_handler__(msg_command):\n print(msg_command)\n\n\nredis_conn = get_vendor(\"DB\").get_redis()\ncommand_listener = redis_conn.pubsub()\nchannel_name = \"dHydra.Command\"\ncommand_listener.subscribe([channel_name])\n# for item in command_listener.listen():\n# if item[\"type\"] == \"message\":\n# print(pickle.loads(item['data']))\n\n\nfrom datetime import datetime\n\n# status = dict()\n# status[\"heart_beat\"] = time.time()\n# status[\"nickname\"] = \"zyyz\"\n# redis_key = \"HySAS\"\n#\n# redis_conn.hmset(redis_key + \"Info\", status)\n# info = redis_conn.hgetall(redis_key + \"Info\")\n\nworker_names = []\n# path = os.path.join(os.path.split(os.path.realpath(__file__))[0]+\"/Worker\")\n# worker_names.extend(os.listdir(path))\ntry:\n worker_names.extend(os.listdir(os.getcwd()+\"/Worker\"))\nexcept FileNotFoundError as e:\n print(\"HySAS运行目录下没有Worker文件夹\")\n\nprint(worker_names)\n\n\n" }, { "path": "HySAS/test1.py", "content": "import time\nimport os\nimport redis\nimport pickle\nfrom HySAS.core.Functions import *\n\nmsg = {\n \"type\": \"sys\",\n \"operation_name\": \"start_worker\",\n \"kwargs\":\"haha\"\n }\n\nprint(pickle.dumps(msg))\n__redis__ = get_vendor(\"DB\").get_redis()\n__redis__.publish(\"dHydra.Command\", pickle.dumps(msg))" }, { "path": "README.md", "content": "# HySAS简介\nHySAS(Hydrogen Situation Awareness System )是一个安全态势感知系统的开发框架，用于实现多进程多数据源的实时计算。不管是在开发过程还是运行过程中，都拥有极高的并行性和扩展性。 \n**特点**\n+ 采用Redis作为消息队列，实现进程间通信\n+ 每个Worker都可以被独立工作，必要时可以相互配合\n+ 支持动态联编，灵活加载功能模块\n\n\n# 运行环境\n - python 3.5以上 (开发环境windows 10, python 3.6)，**不对python2.7提供支持，多版本虚拟环境安装请参考安装HySAS文档**\n - Mysql 8.0.13\n - Redis X64 3.2\n\n # Quick Start\n1. Step 1:安装Redis（略）\n2. Step 2:安装Mysql（略）\n\n\n# 目录介绍\n\n\n# 核心对象\n\n## main.py\n主程序入口 \nmain程序订阅了`HySAS.Command`频道，通过向这个频道发送命令可以开启或关闭响应的服务 \n`worker_dict` : 对象字典`worker_dict[worker_name.nickname]：worker对象`用于记录系统中正在运行的worker对象\n\n## console.py\n允许通过命令行的形式，以第三方的身份动态加载Worker对象。旨在解决如果主程序中有部分Worker died之后，无需重启主程序，便可重启部分Worker，保证数据流的畅通性。\n\n**用命令行启动/关闭**\n```\nstart Demo DemoName\n```\n这样做就会开启一个叫`DemoName`的`Demo`进程 \n这里的`DemoName`是`nickname`，`nickname`是全局唯一的，可能有多个不同的进程都是Demo类，但是每个进程都有唯一的昵称。 \n以Worker为例：\n > - 自己订阅了自己的频道\n > - 每秒向Redis中的`HySAS.Worker.Demo.DemoName.Pub`频道发送一个数字\n > - 将自己订阅收到的内容打印到屏幕\n\n> 原理：之所以可以这样做是因为运行了`pip install --editable .`，它在你当前环境下认识了start命令，它会调用HySAS.console下的start方法，这个方法向Redis中的\"HySAS.Command\"频道发送了一条指令，HySAS Server监听到此指令以后就会实例化Demo目录下的DemoName类对象\n\n```\nstop DemoName\n```\n> 运行stop DemoName后，我们会向\"HySAS.Command\"频道发送一条指令，由HySAS Server去执行这个关闭Demo的任务，会捕捉到终止信号，并且执行用户自定义的`__before_termination__`\n\n## 1.Worker\nWorker类是HySAS的任务处理单元的基类，其继承于`multiprocessing.Process`。用于并行处理各个子任务，其派生类包括`Monitor`、`Process`。 \n###工作流程\n`__init__（）`：初始化成员变量。比较重要的成员：`self.__data_feeder__ = set()`用于存储本Worker订阅的内容。 \n`run（）`： \n`__on_start__()`：用于连接redis（所有Worker共同订阅了`HySAS`频道）、Mysql，并实例化logger\n`on_start()`：自定义启动项，用于拓展接口 \n`__is_unique__()`：首先检查是否已经有相同的进程被开启，通过worker启动的时间，每个worker会生产一个唯一的`token` \n`__thread_listen_command__`：监听命令线程（守护进程），订阅了`self.redis_key.Command`频道，其中`self.redis_key`由创建worker时传入的参数决定。 \n`__thread_pub__`：生产者进程（按需开启） \n`__thread_sub__`：消费者进程（按需开启），获取了订阅`HySAS`频道的redis句柄，并实时接受数据进行处理 \n`__command_handler__`：上述的数据处理函数 \n`__heart_beat__()`：为该worker注册心跳包，用于检验该worker是否被多重开启、存活性等。通过在`self.redis_key.Info`频道发布自身状态信息 ，心跳包内容如下：\n```python\nstatus = dict()\n#以下必带\nstatus[\"heart_beat\"] = time.time()\nstatus[\"nickname\"] = self.__nickname__\nstatus[\"pid\"] = self.pid\nstatus[\"token\"] = self.__token__\nstatus[\"heart_beat_interval\"] = self.__heart_beat_interval__\n#以下选带\nif self.__error_msg__:\n status[\"error_msg\"] = self.__error_msg__\nif self.__stop_info__:\n status[\"stop_info\"] = self.__stop_info__\nif self.__stop_info__:\n status[\"stop_time\"] = self.__stop_time__\nif self.__status__:\n status[\"status\"] = self.__status__\nif self.__threads__:\n status[\"threads\"] = copy.deepcopy(self.__threads__)\nif self.__data_feeder__:\n status[\"data_feeder\"] = self.__data_feeder__\nif self.__follower__:\n status[\"follower\"] = self.__follower__\nself.__redis__.hmset(self.redis_key + \"Info\", status)\n```\n## 2.Monitor\n继承于Worker，主要用于派生WorkerManger对象监视各个Worker的运行状态\n\n## 3.WorkerManger\nWorkerManger继承于Vendor，由Woker类的派生类对象Monitor重写了`run（）`中由`__thread_pub__`调用的`__producer__`生产。 \n### 工作流程\n`__init__（）`：初始化成员变量，获取redis句柄。 \n`update_workers（）`：更新Worker信息。从redis获取所有对象的心跳包，以`dict[workerName][nickName] ：心跳包`的字典格式存入`work_info`。如果发现当前时间间隔大于worker的`heart_beat_interval`，即心跳停止，则放出warning，并从redis中删除该Worker信息。\n" } ]