Repository: Happy-zyy/MyProject
Branch: master
Commit: d3187665a338
Files: 33
Total size: 67.4 KB
Directory structure:
gitextract_am3dz27m/
├── HySAS/
│ ├── Vendor/
│ │ ├── DB/
│ │ │ └── DB.py
│ │ └── WorkerManager/
│ │ ├── WorkerManager.py
│ │ ├── __init__.py
│ │ └── config.py
│ ├── Worker/
│ │ ├── Monitor/
│ │ │ ├── Controller.py
│ │ │ └── Monitor.py
│ │ └── Process/
│ │ └── Process.py
│ ├── config/
│ │ ├── event_mysql.json
│ │ ├── raw_mysql.json
│ │ └── redis.json
│ ├── console.py
│ ├── core/
│ │ ├── Functions.py
│ │ ├── Vendor.py
│ │ ├── Worker.py
│ │ ├── log/
│ │ │ ├── DB/
│ │ │ │ ├── error.log
│ │ │ │ └── info.log
│ │ │ └── console/
│ │ │ ├── error.log
│ │ │ └── info.log
│ │ └── util.py
│ ├── log/
│ │ ├── DB/
│ │ │ ├── error.log
│ │ │ └── info.log
│ │ ├── Monitor/
│ │ │ ├── error.log
│ │ │ └── info.log
│ │ ├── Process/
│ │ │ ├── error.log
│ │ │ └── info.log
│ │ ├── WorkerManager/
│ │ │ ├── error.log
│ │ │ └── info.log
│ │ └── console/
│ │ ├── error.log
│ │ └── info.log
│ ├── main.py
│ ├── test.py
│ └── test1.py
└── README.md
================================================
FILE CONTENTS
================================================
================================================
FILE: HySAS/Vendor/DB/DB.py
================================================
# -*- coding: utf-8 -*-
"""
# Created on
# @author:
# @contact:
"""
# 以下是自动生成的 #
# --- 导入系统配置
import core.util as util
from core.Vendor import Vendor
# --- 导入自定义配置
# 以上是自动生成的 #
import pymysql
import redis
class DB(Vendor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
def get_MySQLdb(self, config="raw_mysql.json", timeout=1500):
import os
if "/" in config:
cfg = util.read_config(config)
else:
cfg = util.read_config(
os.path.join(os.getcwd(), "config", config)
)
host = cfg["host"]
port = cfg["port"]
user = cfg["user"]
password = cfg["password"]
db = cfg["db"]
try:
self.logger.info("Trying to connect to MySQL-db")
db = pymysql.connect(host, user, password, db)
cursor = db.cursor()
cursor.execute("SELECT VERSION()")
cursor.fetchone()
self.logger.info("Successfully connected to MySQL")
return db
except:
self.logger.warning(
">>>>>>>>>>>>>>>>>>Failed to connect to MySQL<<<<<<<<<<<<<<<<<<<")
return False
def get_redis(self, config="redis.json"):
import os
if "/" in config:
cfg = util.read_config(config)
else:
cfg = util.read_config(
os.path.join(os.getcwd(), "config", config)
)
host = cfg["host"]
port = cfg["port"]
try:
self.logger.info("Trying to connect to redis")
self.redis = redis.StrictRedis(
host=host,
port=port
)
self.redis.client_list()
self.logger.info("Successfully connected to redis")
return self.redis
except:
self.logger.warning("Failed to connect to redis")
return False
================================================
FILE: HySAS/Vendor/WorkerManager/WorkerManager.py
================================================
# -*- coding: utf-8 -*-
"""
# Created on
# @author:
# @contact:
"""
# 以下是自动生成的 #
# --- 导入系统配置
import core.util as util
from core.Vendor import Vendor
# --- 导入自定义配置
# 以上是自动生成的 #
import time
import pickle
from core.Functions import get_vendor
import copy
class WorkerManager(Vendor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.redis = get_vendor("DB").get_redis()
self.worker_names = list()
self.worker_info = dict()
self.updating = False
if "auto_remove_terminated" in kwargs:
self.auto_remove_terminated = kwargs["auto_remove_terminated"]
else:
self.auto_remove_terminated = -1
# -1: never;
# otherwise: auto remove terminated workers
# after (self.auto_remove_terminated) seconds
def get_worker_names(self):
worker_names = util.get_worker_names(logger=self.logger)
return worker_names
def get_workers_from_redis(self):
keys = list(map(str,self.redis.keys("HySAS.Worker.*.*.Info")))
workers = dict()
for i in range(len(keys)):
channel = keys[i]
parsed_channel = channel.split('.')
if (len(parsed_channel) == 5) and (parsed_channel[4] == "Info") \
and (parsed_channel[0] == 'HySAS') \
and (parsed_channel[1] == 'Worker'):
if parsed_channel[2] not in workers:
workers[parsed_channel[2]] = dict()
worker = self.redis.hgetall(keys[i])
workers[parsed_channel[2]][parsed_channel[3]] = worker
return workers
def update_workers(self):
"""
更新Worker信息
:return:
"""
self.updating = True
# 根据文件目录检索
self.worker_names = self.get_worker_names()
# 从redis中获取worker信息并更新
worker_info = self.get_workers_from_redis()
# 去除掉过期信息
for worker_name, workers in worker_info.items():
if worker_name not in self.worker_info:
self.worker_info[worker_name] = dict()
for nickname, worker_info in workers.items():
heart_beat_interval = 1
difference = float("inf")
if "heart_beat_interval" in worker_info:
heart_beat_interval = int(worker_info["heart_beat_interval"])
try:
difference = time.time() - float(worker_info["heart_beat"])
except ValueError:
self.logger.error("时间错误")
if difference > heart_beat_interval+1:
# 已经停止
if worker_info["status"] == "started":
worker_info["status"] = "disappeared"
self.logger.warning(
"发现一个没有正常关闭的Worker:\n"
"\tworker_name: {}\n"
"\tnickname: {}"
.format(worker_name, nickname)
)
self.redis.hmset("HySAS.Worker."+worker_name+"."+nickname+".Info", worker_info)
self.worker_info[worker_name][nickname] = copy.deepcopy(worker_info)
if self.auto_remove_terminated > -1 and\
difference> self.auto_remove_terminated+1:
if "token" in worker_info:
token = worker_info["token"]
else:
token = None
removed = self.remove_worker(
worker_name=worker_name,
nickname=nickname,
token=token
)
if removed == 1:
self.logger.info(
"从redis中自动移除过期Worker:\n"
"\tworker_name:{}\n"
"\tnickname:{}\n"
"\ttoken:{}\n"
.format(worker_name,nickname,token)
)
else:
self.logger.warning(
"从redis中自动移除过期Worker:{}\n"
"\tworker_name:{}\n"
"\tnickname:{}\n"
"\ttoken:{}\n"
.format(
removed,
worker_name,
nickname,
token
)
)
self.updating = False
def remove_worker(
self,
worker_name,
nickname,
token
):
key = "HySAS.Worker."+worker_name+"."+nickname+"."+"Info"
worker_info = self.redis.hgetall(key)
if "token" in worker_info:
if worker_info["token"] == token:
return self.redis.delete(key)
else:
if token is None:
return self.redis.delete(key)
return 0
================================================
FILE: HySAS/Vendor/WorkerManager/__init__.py
================================================
================================================
FILE: HySAS/Vendor/WorkerManager/config.py
================================================
================================================
FILE: HySAS/Worker/Monitor/Controller.py
================================================
================================================
FILE: HySAS/Worker/Monitor/Monitor.py
================================================
from core.Worker import Worker
from core.Functions import get_vendor, get_worker_class
import os
import signal
class Monitor(Worker):
def __init__(self, **kwargs):
super().__init__(**kwargs)
def start_worker(self, worker_name, **kwargs):
worker = get_worker_class(worker_name=worker_name, **kwargs)
worker.start()
def terminate_worker(self, nickname=None, pid=None):
if pid is None:
pid = self.get_pid_by_nickname(redis_cli=self.__redis__, )
os.kill(pid, signal.SIGTERM)
def __producer__(self):
"""
定时更新redis中各个worker的状态
:return:
"""
import time
# wm: short for "worker_manager"
self.wm = get_vendor(
"WorkerManager",
auto_remove_terminated=3600*24
)
while True:
time.sleep(1)
self.wm.update_workers()
def get_workers_info(
self,
redis_cli=None,
by="nickname",
nickname=None,
worker_name=None
):
if redis_cli is None:
redis_cli = self.__redis__
result = list()
keys = list()
if by == "nickname" and nickname is not None:
keys = redis_cli.keys("dHydra.Worker.*." + nickname + ".Info")
elif by == "worker_name" and worker_name is not None:
keys = redis_cli.keys("dHydra.Worker." + worker_name + ".*.Info")
for k in keys:
result.append(redis_cli.hgetall(k))
return result
def get_pid_by_nickname(
self,
redis_cli=None,
nickname=None
):
if redis_cli is None:
redis_cli = self.__redis__
workers_info = self.get_workers_info(
redis_cli=self.__redis__, nickname=nickname)
if len(workers_info) == 1:
return int(workers_info[0]["pid"].decode("utf-8"))
else:
self.logger.warning("Worker is not Unique.")
return 0
================================================
FILE: HySAS/Worker/Process/Process.py
================================================
"""
数据处理函数
"""
from core.Worker import Worker
from core.Functions import get_vendor
from core import util
import sys
class Process(Worker):
def __init__(self, **kwargs):
super().__init__(**kwargs)
def init_MySQLdb(self):
self.raw_MySQL = get_vendor("DB").get_MySQLdb("raw_mysql.json")
if self.raw_MySQL is False:
self.logger.error("Cannot connect to raw_MySQLdb")
return False
self.event_MySQL = get_vendor("DB").get_MySQLdb("event_mysql.json")
if self.event_MySQL is False:
self.logger.error("Cannot connect to event_MySQLdb")
return False
def on_start(self):
self.init_MySQLdb()
def __producer__(self): #拥有数据库句柄 通过 self.db调用
"""
取数据
:return: msg
"""
msg = "data"
self.__data_handler__(msg)
pass
def __data_handler__(self, msg):
print(msg)
print("======")
pass
================================================
FILE: HySAS/config/event_mysql.json
================================================
{
"host": "127.0.0.1",
"port": 6379,
"user": "",
"password": "",
"db": ""
}
================================================
FILE: HySAS/config/raw_mysql.json
================================================
{
"host": "127.0.0.1",
"port": 3306,
"user": "root",
"password": "123",
"db": "test"
}
================================================
FILE: HySAS/config/redis.json
================================================
{
"host": "127.0.0.1",
"port": 6379
}
================================================
FILE: HySAS/console.py
================================================
# -*- coding: utf-8 -*-
import logging
from core.Functions import *
import core.util as util
import click
import traceback
import pickle
def init_logger():
return util.get_logger(logger_name="console")
@click.command()
@click.argument('worker_name', nargs=1)
@click.argument('nickname', nargs=1)
def start(worker_name=None, nickname=None):
"""
主要依靠nickname来指定配置文件，通过配置文件传入参数
:param worker_name:
:param nickname:
如果nickname以.json结尾，那么程序会去"./config/"目录下寻找对应配置文件
如果不是以.json结尾，则认为传入的就是nickname，同时会去"./config/"目录下寻找
nickname.json
:return:
"""
kwargs = dict()
kwargs["worker_name"] = worker_name
if nickname[-5:] == ".json":
kwargs["config"] = nickname
kwargs["nickname"] = nickname[0:-5]
else:
kwargs["config"] = nickname + ".json"
kwargs["nickname"] = nickname
config_file = os.path.join(os.getcwd(), "config", kwargs["config"])
if os.path.exists(config_file):
config = util.read_config(config_file)
else:
# 命令指定了配置文件，如果不存在，就结束
print("未能找到配置文件：", config_file)
print("默认无配置启动")
config = {}
for k in config.keys():
kwargs[k] = config[k]
msg = {
"type": "sys",
"operation_name": "start_worker",
"kwargs": kwargs
}
__redis__.publish("HySAS.Command", pickle.dumps(msg))
# @click.command()
# @click.argument('nickname', nargs=1)
def stop(nickname=None):
msg = {"type": "sys",
"operation_name": "terminate_worker",
"kwargs": {"nickname": nickname}
}
if nickname is not None:
msg["kwargs"]["nickname"] = nickname
__redis__.publish("HySAS.Command", pickle.dumps(msg))
def shutdown(nickname=None):
msg = {"type": "sys",
"func": "shutdown"
}
__redis__.publish("HySAS.Command", pickle.dumps(msg))
def send_command(
channel_name="HySAS.Command",
command_type="sys",
operation_name=None,
token=None,
kwargs={}
):
if operation_name is not None:
command = {
"type": command_type,
"operation_name": operation_name,
"token": token,
"kwargs": kwargs
}
__redis__.publish(channel_name, pickle.dumps(command))
logger = init_logger()
__redis__ = get_vendor("DB").get_redis()
================================================
FILE: HySAS/core/Functions.py
================================================
"""
框架的全局方法，在主程序运行时会被引用
---
Created on 01/14/2019
@author: Happy-zyy
@contact: 617532750@qq.com
"""
import importlib
import sys
import os
import logging
import traceback
def get_workers():
candidates = set(os.listdir("./Worker")) | set(os.listdir(
os.path.split(os.path.realpath(__file__))[0][:-4] + "Worker"))
candidates = list(candidates)
workers = list()
for item in candidates:
if (item[0] >= "A" and item[0] <= "Z"):
workers.append(item)
return workers
"""
动态加载web controller
"""
def get_controller_method(class_name, method):
logger = logging.getLogger("Functions")
# get instance of controller
if os.path.exists(
os.getcwd() + "/Worker/" +
class_name + "/" + "Controller.py"
):
func = getattr(
importlib.import_module("Worker." + class_name + ".Controller"),
method
)
return func
else:
try:
func = getattr(importlib.import_module(
"dHydra.Worker." + class_name + ".Controller"),
method
)
return func
except Exception as e:
return False
def V(name, vendor_name=None, **kwargs):
return get_vendor(name=name, vendor_name=None, **kwargs)
def get_vendor(name, vendor_name=None, **kwargs):
"""
get_vendor方法，动态加载vendor类
"""
logger = logging.getLogger('Functions')
if vendor_name is None:
vendor_name = "V-" + name
class_name = name
module_name = 'Vendor.' + name + '.' + class_name
if os.path.exists(
os.path.join(os.getcwd(), "Vendor", name, class_name + ".py")
):
try:
instance = getattr(
__import__(module_name, globals(), locals(), [class_name], 0),
class_name
)(**kwargs)
except ImportError:
traceback.print_exc()
else:
try:
instance = getattr(
__import__(
"HySAS." + module_name,
globals(),
locals(),
[class_name],
0
),
class_name
)(**kwargs)
except ImportError:
traceback.print_exc()
return instance
def get_worker_class(worker_name, **kwargs):
logger = logging.getLogger('Functions')
module_name = 'Worker.' + worker_name + '.' + worker_name
if os.path.exists(
os.getcwd() + "/Worker/" +
worker_name + "/" + worker_name + ".py"
):
try:
if module_name in sys.modules:
del sys.modules[module_name]
return getattr(
__import__(module_name, globals(), locals(), [worker_name], 0),
worker_name
)(**kwargs)
except ImportError:
traceback.print_exc()
else:
try:
if "HySAS."+module_name in sys.modules:
del sys.modules["HySAS."+module_name]
return getattr(
__import__(
"HySAS." +
module_name,
globals(),
locals(),
[worker_name],
0
),
worker_name
)(**kwargs)
except ImportError:
traceback.print_exc()
def convert(data):
if isinstance(data, bytes): return data.decode('ascii')
if isinstance(data, dict): return dict(map(convert, data.items()))
if isinstance(data, tuple): return map(convert, data)
return data
================================================
FILE: HySAS/core/Vendor.py
================================================
# -*- coding: utf-8 -*-
import logging
import core.util as util
class Vendor:
def __init__(
self,
log_path="log", #
console_log=True, # 屏幕打印日志开关，默认True
console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO
critical_log=False, # critica单独l写文件日志，默认关闭
error_log=True, # error级别单独写文件日志，默认开启
warning_log=False, # warning级别单独写日志，默认关闭
info_log=True, # info级别单独写日志，默认开启
debug_log=False, # debug级别日志，默认关闭
**kwargs
):
self.logger = util.get_logger(
log_path=log_path, #
console_log=console_log, # 屏幕打印日志开关，默认True
console_log_level=console_log_level, # 屏幕打印日志的级别，默认为INFO
critical_log=critical_log, # critica单独l写文件日志，默认关闭
error_log=error_log, # error级别单独写文件日志，默认开启
warning_log=warning_log, # warning级别单独写日志，默认关闭
info_log=info_log, # info级别单独写日志，默认开启
debug_log=debug_log, # debug级别日志，默认关闭
logger_name=self.__class__.__name__,
)
================================================
FILE: HySAS/core/Worker.py
================================================
# -*- coding: utf-8 -*-
"""
Worker抽象类
Created on 01/14/2019
@author: Happy-zyy
@contact: 617532750@qq.com
"""
import multiprocessing
import threading
import time
import logging
import traceback
import redis
import json
import copy
import core.util as util
from console import *
from datetime import datetime
from datetime import timedelta
from abc import ABCMeta
import signal
import sys
import os
import ast
import pickle
class Worker(multiprocessing.Process):
__metaclass__ = ABCMeta
def __init__(
self,
singleton=True, # 单例模式
nickname=None, # Worker的自定义名字
description="No Description", # 备注说明
heart_beat_interval=1, # 默认1秒心跳
log_path="log", #
console_log=True, # 屏幕打印日志开关，默认True
console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO
critical_log=False, # critical单独写文件日志，默认关闭
error_log=True, # error级别单独写文件日志，默认开启
warning_log=False, # warning级别单独写日志，默认关闭
info_log=True, # info级别单独写日志，默认开启
debug_log=False, # debug级别日志，默认关闭
**kwargs
):
super().__init__()
# 记录日志配置
self.__log_path__ = log_path
self.__console_log__ = console_log
self.__console_log_level__ = console_log_level
self.__critical_log__ = critical_log
self.__error_log__ = error_log
self.__warning_log__ = warning_log
self.__info_log__ = info_log
self.__debug_log__ = debug_log
self.logger = util.get_logger(
logger_name=self.__class__.__name__,
log_path=self.__log_path__,
console_log=self.__console_log__, # 屏幕打印日志开关，默认True
console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO
critical_log=self.__critical_log__, # critica单独l写文件日志，默认关闭
error_log=self.__error_log__, # error级别单独写文件日志，默认开启
warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭
info_log=self.__info_log__, # info级别单独写日志，默认开启
debug_log=self.__debug_log__, # debug级别日志，默认关闭
)
self.__token__ = util.generate_token()
if nickname is None:
self.__nickname__ = self.__class__.__name__
else:
self.__nickname__ = nickname
self.nickname = self.__nickname__
self.name = self.__nickname__
self.__singleton__ = singleton
self.__description__ = description
self.__heart_beat_interval__ = heart_beat_interval
self.__threads__ = dict() # 被监控的线程
self.__data_feeder__ = set() # 本Worker订阅的内容
self.__follower__ = set() # Follower
self.__error_msg__ = None #
self.__stop_info__ = None #
self.__stop_time__ = None #
self.__status__ = "init"
self.redis_key = "HySAS.Worker." + \
self.__class__.__name__ + "." + self.__nickname__ + "."
self.channel_pub = self.redis_key + "Pub"
"""
self.__threads__ = {
"nickname": {
"description" : "该线程功能备注说明",
"name" : "该线程的名字",
"target" : "该线程的target"
"restart_mode" : "重启模式，可以为 manual/auto/remove;
manual则代表允许管理员发送命令手工重启线程,
auto则一旦线程关闭立即自动开启，
remove则代表一旦线程结束就从监控列表移除",
"restart_func" : "自动/手动重启时调用的方法",
},
}
"""
self.shutdown_signals = [
"SIGQUIT", # quit 信号
"SIGINT", # 键盘信号
"SIGHUP", # nohup 命令
"SIGTERM", # kill 命令
]
for s in self.shutdown_signals:
# 捕获退出信号后的要调用的,唯一的 shutdown 接口
try:
if hasattr(signal, s):
signal.signal(
getattr(signal, s, None),
self.__on_termination__
)
except Exception as e:
self.logger.info("绑定退出信号：{}失败，可能与windows系统有关。".format(s))
# 清空它，在run以后重新实例化
# 否则windows下会无法pickle
del(self.logger)
def __is_unique__(self):
info = self.__redis__.hgetall(self.redis_key + "Info")
if "token" in info:
if info["token"] != self.__token__:
if "heart_beat" in info:
if (time.time() - float(info["heart_beat"])) < self.__heart_beat_interval__:
return False
return True
def __auto_restart_thread__(self):
# Worker内置的默认自动重启线程方法
pass
def __command_handler__(self, msg_command):
# cli is a dict with the following structure:
"""
msg_command = {
"type" : "sys/customized",
"operation" : "operation_name",
"kwargs" : "suppose that the operation is a function,
we need to pass some arguments",
"token" : "the token is used to verify the authentication
of the operation"
}
"""
try:
msg_command = pickle.loads(msg_command)
if not isinstance(msg_command, dict):
return
except Exception as e:
traceback.print_exc()
print(e)
if msg_command["type"] == "sys":
if hasattr(self, msg_command["operation_name"]):
func = getattr(self, msg_command["operation_name"])
try:
print(msg_command["kwargs"])
result = func(**msg_command["kwargs"])
except Exception as e:
traceback.print_exc()
self.logger.error(e)
def monitor_add_thread(self,
thread,
description="No Description",
restart_mode="manual",
restart_func=None):
# 将该线程加入管理员监控范围
pass
def monitor_remove_thread(self, thread):
# 取消管理员对线程thread的监控
pass
def init_redis(self):
# 检测redis, sqldb连接
try:
self.__redis__ = get_vendor("DB").get_redis()
#self.__redis__.client_list()
self.__listener__ = self.__redis__.pubsub()
self.__listener__.subscribe(["HySAS"])
except redis.ConnectionError:
self.logger.error("Cannot connect to redis")
return False
def check_prerequisites(self):
"""
检查是否满足开启进程的条件
"""
# 如果是单例，检测是否重复开启
return True
def __listen_command__(self):
#
self.command_listener = self.__redis__.pubsub()
channel_name = self.redis_key + "Command"
self.command_listener.subscribe([channel_name])
while True:
msg_command = self.command_listener.get_message()
if msg_command:
if (msg_command["type"] == "message") or \
(msg_command["type"] == "pmessage"):
self.__command_handler__(msg_command["data"])
else:
time.sleep(0.5)
def __heart_beat__(self):
# flush status infomation to redis
status = dict()
status["heart_beat"] = time.time()
status["nickname"] = self.__nickname__
status["pid"] = self.pid
status["token"] = self.__token__
status["heart_beat_interval"] = self.__heart_beat_interval__
if self.__error_msg__:
status["error_msg"] = self.__error_msg__
if self.__stop_info__:
status["stop_info"] = self.__stop_info__
if self.__stop_info__:
status["stop_time"] = self.__stop_time__
if self.__status__:
status["status"] = self.__status__
if self.__threads__:
status["threads"] = copy.deepcopy(self.__threads__)
if self.__data_feeder__:
status["data_feeder"] = self.__data_feeder__
if self.__follower__:
status["follower"] = self.__follower__
self.__redis__.hmset(self.redis_key + "Info", status)
def __producer__(self):
"""
在子类中被重写的用以作为生产者的线程
若不重写，线程启动后就结束了
"""
pass
def __consumer__(self):
"""
默认的消费者线程
随着Worker进程的start而启动
"""
pass
# while True:
# data = self.__listener__.get_message(timeout=10)
# if data is not None:
# self.__data_handler__(data)
# time.sleep(1)
# 需要在子类中重写的数据处理方法
def __data_handler__(self, msg):
"""
需要在子类中被重写的用以处理数据的方法，
接受到的msg数据是原始的从Redis中监听到的数据
"""
print("*****")
pass
def __before_termination__(self, sig):
self.logger.info("收到了退出信号。进程类：{}，进程名：{},进程pid:{}"
.format(
self.__class__.__name__,
self.__nickname__,
self.pid
)
)
def __on_termination__(self, sig, frame):
self.__before_termination__(sig)
self.__status__ = "terminated"
self.__heart_beat__() # The last heart_beat, sad...
sys.exit(0)
def publish(self,data,channel_name=None):
if channel_name is None:
channel_name = self.channel_pub
# publish data to redis
try:
self.__redis__.publish(channel_name, data)
except Exception as e:
self.logger.warning(e)
def __on_start__(self):
"""
进程开始运行时调用
:return:
"""
if self.check_prerequisites() is not True:
sys.exit(0)
self.init_redis()
# 实例化self.logger
self.logger = util.get_logger(
logger_name=self.__class__.__name__,
log_path=self.__log_path__, #
console_log=self.__console_log__, # 屏幕打印日志开关，默认True
console_log_level=self.__console_log_level__, # 屏幕打印日志的级别，默认为INFO
critical_log=self.__critical_log__, # critical写文件日志，默认关闭
error_log=self.__error_log__, # error级别单独写文件日志，默认开启
warning_log=self.__warning_log__, # warning级别单独写日志，默认关闭
info_log=self.__info_log__, # info级别单独写日志，默认开启
debug_log=self.__debug_log__, # debug级别日志，默认关闭
)
def on_start(self):
pass
def run(self):
"""
初始化Worker
"""
self.__on_start__()
self.logger.info("初始化{}".format(self.__class__.__name__))
# 用户自定义的on_start
self.on_start()
# 首先检查是否已经有相同的进程被开启
if self.__is_unique__():
self.__status__ = "started"
else:
self.error_msg = "Duplicated Process"
self.logger.warning(self.error_msg)
sys.exit(0)
# 开启监听命令线程
self.__thread_listen_command__ = threading.Thread(
target=self.__listen_command__
)
self.__thread_listen_command__.setDaemon(True)
self.monitor_add_thread(
thread=self.__thread_listen_command__,
description="Listening Command Channel",
restart_mode="auto",
restart_func=self.__auto_restart_thread__
)
self.__thread_listen_command__.start()
# 检查初始化设置，按需开启
# PUB线程
self.__thread_pub__ = threading.Thread(target=self.__producer__)
self.__thread_pub__.setDaemon(True)
self.monitor_add_thread(
thread=self.__thread_pub__,
description="DATA PUBLISHER",
restart_mode="auto",
restart_func=self.__auto_restart_thread__
)
self.__thread_pub__.start()
# LISTENER
self.__thread_sub__ = threading.Thread(target=self.__consumer__)
self.__thread_sub__.setDaemon(True)
self.monitor_add_thread(
thread=self.__thread_sub__,
description="DATA CONSUMER",
restart_mode="auto",
restart_func=self.__auto_restart_thread__
)
self.__thread_sub__.start()
while True:
# heart beat
self.__heart_beat__()
time.sleep(self.__heart_beat_interval__)
def subscribe(self,channel_name=None, worker_name=None, nickname=None):
"""
订阅Worker
可以直接填入channel_name
也可以通过填入worker_name/nickname/worker_name+nickname来自动订阅对应的内容
"""
if channel_name is not None:
self.__listener__.subscribe(channel_name)
return None
if (worker_name is not None) and (nickname is None):
# 订阅所有此类Worker
self.__listener__.psubscribe(
"HySAS.Worker." + worker_name + ".*.Pub"
)
self.logger.info(
"About to subscribe the Worker of worker_name: {}, pattern:{}"
.format(
worker_name,
"HySAS.Worker." + worker_name + ".*.Pub"
)
)
elif (worker_name is not None) and (nickname is not None):
channel_name = "HySAS.Worker." \
+ worker_name + "." + nickname + ".Pub"
self.__listener__.subscribe(channel_name)
self.logger.info(
"Subscribed: {}".format(channel_name)
)
elif (nickname is not None):
# 订阅nickname
self.__listener__.psubscribe(
"HySAS.Worker.*." +
nickname + ".Pub"
)
self.logger.info(
"About to subscribe the Worker of nickname: {}, pattern:{}"
.format(
nickname,
"HySAS.Worker.*." +
nickname +
".Pub"
)
)
else:
self.logger.warning("nickname/worker_name的输入方式不合理")
def unsubscribe(self, worker_name=None, nickname=None):
"""
退订Worker
"""
if (worker_name is not None) and (nickname is None):
# 订阅所有此类Worker
self.__listener__.punsubscribe(
"HySAS.Worker." + worker_name + ".*.Pub")
self.logger.info(
"About to unsubscribe the Worker of worker_name: {}, \
pattern:{}"
.format(
nickname,
"HySAS.Worker.*." + worker_name + ".Pub"
)
)
pass
elif (nickname is not None):
# 订阅nickname
self.__listener__.punsubscribe(
"HySAS.Worker.*." + nickname + ".Pub"
)
self.logger.info(
"About to subscribe the Worker of nickname: {}, pattern:{}"
.format(
nickname,
"HySAS.Worker.*." + nickname + ".Pub"
)
)
else:
self.logger.warning("nickname/worker_name的输入方式不合理")
================================================
FILE: HySAS/core/log/DB/error.log
================================================
================================================
FILE: HySAS/core/log/DB/info.log
================================================
================================================
FILE: HySAS/core/log/console/error.log
================================================
================================================
FILE: HySAS/core/log/console/info.log
================================================
================================================
FILE: HySAS/core/util.py
================================================
import logging
import os
import hashlib
import time
import json
def get_worker_names(logger=None):
"""
根据文件夹名字返回所有可能的worker_name
:return:
"""
worker_names = []
# path = os.path.split(os.path.realpath(__file__))[0][:-6]+"/Worker"
# worker_names.extend(os.listdir(path))
try:
worker_names.extend(os.listdir(os.getcwd()+"/Worker"))
except FileNotFoundError as e:
if logger is None:
print("HySAS运行目录下没有Worker文件夹")
else:
logger.warning("HySAS运行目录下没有Worker文件夹")
return worker_names
def get_logger(
logger_name="main",
log_path="log", #
console_log=True, # 屏幕打印日志开关，默认True
console_log_level=logging.INFO, # 屏幕打印日志的级别，默认为INFO
critical_log=False, # critical单独写文件日志，默认关闭
error_log=True, # error级别单独写文件日志，默认开启
warning_log=False, # warning级别单独写日志，默认关闭
info_log=True, # info级别单独写日志，默认开启
debug_log=False, # debug级别日志，默认关闭
):
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s'
)
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
if log_path:
# 补全文件夹
if log_path[-1] != '/':
log_path += '/'
if not logger.handlers:
# 屏幕日志打印设置
if console_log:
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
console_handler.setLevel(logging.INFO)
logger.addHandler(console_handler)
if not os.path.exists(log_path + logger_name):
os.makedirs(log_path + logger_name)
# 打开下面的输出到文件
if critical_log:
log_handler = logging.FileHandler(
log_path + logger_name + '/critical.log'
)
log_handler.setLevel(logging.CRITICAL)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
if error_log:
log_handler = logging.FileHandler(
log_path + logger_name + '/error.log'
)
log_handler.setLevel(logging.ERROR)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
if warning_log:
log_handler = logging.FileHandler(
log_path + logger_name + '/warning.log'
)
log_handler.setLevel(logging.WARNING)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
if info_log:
log_handler = logging.FileHandler(
log_path + logger_name + '/info.log'
)
log_handler.setLevel(logging.INFO)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
if debug_log:
log_handler = logging.FileHandler(
log_path + logger_name + '/debug.log'
)
log_handler.setLevel(logging.DEBUG)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
def generate_token():
token = hashlib.sha1()
token.update(str(time.time()).encode())
token = token.hexdigest()
return token
def read_config(file_path):
# 读取配置
try:
f_config = open(file_path)
cfg = json.load(f_config)
except Exception as e:
print("{}".format(e))
cfg = dict()
print(
"未能正确加载{}，请检查路径，json文档格式，或者忽略此警告"
.format(file_path)
)
return cfg
================================================
FILE: HySAS/log/DB/error.log
================================================
================================================
FILE: HySAS/log/DB/info.log
================================================
2019-01-15 10:38:23,552 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:38:25,562 - DB - 68 - WARNING - Failed to connect to redis
2019-01-15 10:49:10,014 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:55:46,379 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:55:49,677 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:56:19,580 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:56:45,196 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:57:28,660 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 10:57:31,169 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:12:30,375 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:12:38,731 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:17:00,225 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:17:03,378 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:19:08,291 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:19:14,068 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:20:30,782 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:20:50,996 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:27:14,317 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:27:17,802 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:28:06,379 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:33:25,918 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:33:28,955 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:34:53,982 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:34:56,690 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:35:04,770 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:35:24,233 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:36:02,853 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:36:11,119 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 11:36:17,741 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:55:08,689 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:55:30,131 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:55:58,190 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:56:45,528 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:57:11,346 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:57:57,146 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 14:58:13,734 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:00:27,200 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:01:49,462 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:01:49,480 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:02:13,434 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:10:44,988 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:10:45,020 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:36:17,193 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:36:17,237 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:36:17,704 - DB - 59 - INFO - Trying to connect to redis
2019-01-15 15:36:17,717 - DB - 59 - INFO - Trying to connect to redis
2019-01-16 11:23:18,509 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:18,517 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:18,668 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:19,312 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:19,318 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:19,326 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:44,698 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:44,706 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:44,735 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:45,256 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:45,263 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:45,270 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:23:45,272 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:25:34,629 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:40,779 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,228 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,235 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,271 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,794 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,801 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,808 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:25:59,809 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:26:41,487 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:41,494 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:41,548 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,183 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,190 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,197 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,199 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:26:42,256 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,263 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,292 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:26:42,294 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:59:43,416 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:43,423 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:43,453 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:44,014 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:44,022 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:44,030 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:59:44,032 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 11:59:44,035 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 12:00:42,046 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,053 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,082 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,633 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,637 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,640 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,644 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,648 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,650 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 12:00:42,651 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 12:00:42,652 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 12:00:42,653 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 12:00:42,655 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:05:27,277 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:05:40,119 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:07:35,467 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:08:00,707 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:08:57,073 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:09:44,100 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:12:09,237 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:14:16,598 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:16:39,681 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:17:03,198 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:19:56,705 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:20:22,377 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:21:44,117 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:22:49,185 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:24:28,167 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:37:33,920 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:53,691 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:53,701 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:53,748 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,371 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,378 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,385 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,386 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:39:54,392 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:39:54,409 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,417 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,426 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:39:54,428 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:39:54,431 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:44:17,721 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:17,729 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:17,765 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,319 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,326 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,333 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,334 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:44:18,335 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:44:18,338 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:45:06,377 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,384 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,419 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,964 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,965 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,973 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,980 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,981 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:45:06,982 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:45:06,983 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:45:06,984 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:45:06,986 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:53:47,231 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,239 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,271 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,836 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,843 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,850 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:53:47,852 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:53:47,854 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:54:42,455 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:54:56,807 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:20,724 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:20,731 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:20,770 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,360 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,367 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,368 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,374 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,375 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:56:21,376 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:56:21,379 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:56:21,402 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:47,471 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:47,478 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:47,507 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,083 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,090 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,097 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,099 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:58:48,101 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 14:58:48,104 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,109 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,112 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,120 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 14:58:48,122 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 14:58:48,124 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 15:14:27,070 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,077 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,112 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,672 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,679 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,687 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 15:14:27,689 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 15:14:27,691 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 15:14:27,698 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,140 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,148 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,187 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,771 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,778 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,786 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:22:55,788 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 15:22:55,790 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 15:22:55,798 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:52:49,239 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:53:33,871 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:53:48,123 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:53:55,740 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:54:53,430 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:55:31,092 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:58:03,617 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:58:33,713 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 15:58:50,782 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:01:24,014 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:01:43,708 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:01:56,358 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:02:52,853 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:03:15,355 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:41,913 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:41,920 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:41,950 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,580 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,587 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,590 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,594 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,596 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:06:42,597 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,602 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:06:42,605 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:06:42,607 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:06:42,609 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:06:42,618 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:09,805 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:09,811 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:09,841 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,401 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,408 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,416 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:07:10,418 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:07:10,420 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:07:10,426 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:58,732 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:58,739 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:58,768 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:59,340 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:59,347 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:59,355 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:15:59,357 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:15:59,359 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:15:59,379 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,013 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,020 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,051 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,610 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,617 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,618 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,624 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,626 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:30:05,629 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:30:05,632 - DB - 61 - INFO - Trying to connect to redis
2019-01-16 16:30:05,635 - DB - 37 - INFO - ӵMySQL-db
2019-01-16 16:30:05,637 - DB - 42 - INFO - ѾɹӵMySQL
2019-01-16 16:30:05,643 - DB - 61 - INFO - Trying to connect to redis
================================================
FILE: HySAS/log/Monitor/error.log
================================================
================================================
FILE: HySAS/log/Monitor/info.log
================================================
2019-01-16 11:23:45,365 - Monitor - 326 - INFO - ʼWorker
2019-01-16 11:25:59,817 - Monitor - 326 - INFO - ʼWorker
2019-01-16 11:26:42,206 - Monitor - 326 - INFO - ʼWorker
2019-01-16 11:59:44,056 - Monitor - 326 - INFO - ʼWorker
2019-01-16 12:00:42,652 - Monitor - 326 - INFO - ʼWorker
2019-01-16 14:39:54,393 - Monitor - 320 - INFO - ʼWorker
2019-01-16 14:44:18,339 - Monitor - 320 - INFO - ʼWorker
2019-01-16 14:45:06,985 - Monitor - 320 - INFO - ʼWorker
2019-01-16 14:53:47,855 - Monitor - 320 - INFO - ʼWorker
2019-01-16 14:56:21,380 - Monitor - 320 - INFO - ʼWorker
2019-01-16 14:58:48,102 - Monitor - 320 - INFO - ʼWorker
2019-01-16 15:14:27,692 - Monitor - 320 - INFO - ʼWorker
2019-01-16 15:22:55,791 - Monitor - 327 - INFO - ʼWorker
2019-01-16 16:06:42,603 - Monitor - 327 - INFO - ʼWorker
2019-01-16 16:07:10,421 - Monitor - 327 - INFO - ʼWorker
2019-01-16 16:15:59,359 - Monitor - 327 - INFO - ʼWorker
2019-01-16 16:30:05,630 - Monitor - 327 - INFO - ʼWorker
================================================
FILE: HySAS/log/Process/error.log
================================================
================================================
FILE: HySAS/log/Process/info.log
================================================
2019-01-16 11:26:42,301 - Process - 326 - INFO - ʼWorker
2019-01-16 11:59:44,056 - Process - 326 - INFO - ʼWorker
2019-01-16 12:00:42,656 - Process - 326 - INFO - ʼWorker
2019-01-16 14:39:54,432 - Process - 320 - INFO - ʼWorker
2019-01-16 14:44:18,338 - Process - 320 - INFO - ʼWorker
2019-01-16 14:45:06,986 - Process - 320 - INFO - ʼWorker
2019-01-16 14:53:47,855 - Process - 320 - INFO - ʼWorker
2019-01-16 14:56:21,380 - Process - 320 - INFO - ʼWorker
2019-01-16 14:58:48,125 - Process - 320 - INFO - ʼWorker
2019-01-16 15:14:27,692 - Process - 320 - INFO - ʼWorker
2019-01-16 15:22:55,791 - Process - 327 - INFO - ʼWorker
2019-01-16 16:06:42,610 - Process - 327 - INFO - ʼWorker
2019-01-16 16:07:10,421 - Process - 327 - INFO - ʼWorker
2019-01-16 16:15:59,359 - Process - 327 - INFO - ʼWorker
2019-01-16 16:30:05,638 - Process - 327 - INFO - ʼWorker
================================================
FILE: HySAS/log/WorkerManager/error.log
================================================
================================================
FILE: HySAS/log/WorkerManager/info.log
================================================
================================================
FILE: HySAS/log/console/error.log
================================================
2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
================================================
FILE: HySAS/log/console/info.log
================================================
2019-01-15 15:02:10,563 - console - 86 - INFO - Monitor has started
2019-01-15 15:10:45,019 - console - 86 - INFO - Monitor has started
2019-01-15 15:36:17,236 - console - 86 - INFO - Monitor has started
2019-01-16 11:23:18,665 - console - 139 - INFO - Monitor has started
2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
2019-01-16 11:23:44,734 - console - 139 - INFO - Monitor has started
2019-01-16 11:23:44,737 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
2019-01-16 11:25:59,270 - console - 139 - INFO - Monitor has started
2019-01-16 11:25:59,273 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'
2019-01-16 11:26:41,524 - console - 139 - INFO - Monitor has started
2019-01-16 11:59:43,452 - console - 139 - INFO - Monitor has started
2019-01-16 12:00:42,081 - console - 139 - INFO - Monitor has started
2019-01-16 14:39:53,748 - console - 139 - INFO - Monitor has started
2019-01-16 14:44:17,764 - console - 139 - INFO - Monitor has started
2019-01-16 14:45:06,418 - console - 139 - INFO - Monitor has started
2019-01-16 14:53:47,270 - console - 139 - INFO - Monitor has started
2019-01-16 14:56:20,769 - console - 139 - INFO - Monitor has started
2019-01-16 14:58:47,506 - console - 139 - INFO - Monitor has started
2019-01-16 15:14:27,111 - console - 145 - INFO - Monitor has started
2019-01-16 15:22:55,187 - console - 145 - INFO - Monitor has started
2019-01-16 16:06:41,950 - console - 145 - INFO - Monitor has started
2019-01-16 16:07:09,840 - console - 145 - INFO - Monitor has started
2019-01-16 16:15:58,767 - console - 145 - INFO - Monitor has started
2019-01-16 16:30:05,050 - console - 145 - INFO - Monitor has started
================================================
FILE: HySAS/main.py
================================================
from console import logger
from core.Functions import *
import time
import signal
import click
import pickle
__redis__ = get_vendor("DB").get_redis()
worker_dict = dict()
def __on_termination__(sig, frame):
logger.info("The HySAS Server is about to terminate, pid:{}"
.format(os.getpid())
)
sys.exit(0)
def bind_quit_signals():
import signal
shutdown_signals = [
"SIGQUIT", # quit 信号
"SIGINT", # 键盘信号
"SIGHUP", # nohup 命令
"SIGTERM", # kill 命令
]
for s in shutdown_signals:
if hasattr(signal, s):
signal.signal(
getattr(signal, s, None),
__on_termination__
)
def start_worker(worker_name, **kwargs):
worker = get_worker_class(worker_name=worker_name, **kwargs)
worker_dict[worker.nickname] = worker
worker.start()
def terminate_worker(nickname=None, pid=None):
logger.info("{}".format(worker_dict))
if pid is None:
pid = get_pid_by_nickname(redis_cli=__redis__, nickname=nickname)
os.kill(pid, signal.SIGTERM)
i = 0
while worker_dict[nickname]._popen is None and i < 30:
time.sleep(0.1)
i += 1
worker_dict[nickname]._popen.wait(1)
worker_dict.pop(nickname)
def get_workers_info(redis_cli=None, by="nickname", nickname=None, worker_name=None):
if redis_cli is None:
redis_cli = __redis__
result = list()
keys = list()
if by == "nickname" and nickname is not None:
keys = redis_cli.keys("HySAS.Worker.*." + nickname + ".Info")
elif by == "worker_name" and worker_name is not None:
keys = redis_cli.keys("HySAS.Worker." + worker_name + ".*.Info")
for k in keys:
result.append(redis_cli.hgetall(k))
return result
def get_pid_by_nickname(redis_cli=None, nickname=None):
if redis_cli is None:
redis_cli = __redis__
workers_info = get_workers_info(redis_cli=__redis__, nickname=nickname)
workers_info = list(map(convert, workers_info))
if len(workers_info) == 1:
return int(workers_info[0]["pid"])
else:
logger.warning("Worker is not Unique.")
return 0
def __command_handler__(msg_command):
# msg_command is a dict with the following structure:
"""
msg_command = {
"type" : "sys/user/data",
"operation" : "operation_name",
"kwargs" : "suppose that the operation is a function, we need to
pass some arguments",
"token" : "the token is used to verify the authentication of the
operation"
}
"""
import sys
try:
msg_command = pickle.loads(msg_command)
if not isinstance(msg_command, dict):
return
except Exception as e:
traceback.print_exc()
print("msg_command is not dict \n",e)
if msg_command["type"] == "sys":
if "func" in msg_command and msg_command["func"] == "shutdown":
signal_handler()
elif hasattr(
sys.modules["HySAS.main"],
msg_command["operation_name"]
):
func = getattr(
sys.modules["HySAS.main"],
msg_command["operation_name"]
)
try:
print(msg_command["kwargs"])
result = func(**msg_command["kwargs"])
except Exception as e:
traceback.print_exc()
logger.error(e)
elif msg_command["type"] == "user":
"""
用于接受前端用户数据
"""
pass
elif msg_command["type"] == "data":
"""
用于接受关联分析时可能向外传输的交互信号
"""
pass
def signal_handler():
for work_name in list(worker_dict.keys()):
terminate_worker(work_name)
print('Shutdown HySAS!')
sys.exit(0)
# @click.command()
# @click.argument('what', nargs=-1)
def main(what=None):
try:
if what:
if what[0] != "HySAS":
print("Please input HySAS for starting")
exit(0)
else:
print(
"Welcome to HySAS! Following is the awesome!!!"
)
logo = "◆ ◆ ◆\n"\
"◆ ◆ ◆◆◆◆◆ ◆◆ ◆◆◆◆◆\n"\
"◆ ◆ ◆◆◆◆◆ ◆◆◆ ◆◆◆\n"\
"◆ ◆ ◆◆ ◆◆ ◆◆◆◆ ◆◆◆◆ ◆◆◆ \n"\
"◆◆◆◆◆◆ ◆◆ ◆◆ ◆◆◆◆◆ ◆◆ ◆◆ ◆◆◆◆◆ \n"\
"◆ ◆ ◆◆◆◆ ◆◆◆ ◆◆◆◆◆◆ ◆◆◆ \n"\
"◆ ◆ ◆◆◆ ◆◆◆ ◆◆◆ ◆◆ ◆◆◆ ◆◆◆ ◆◆◆ \n"\
"◆ ◆ ◆◆ ◆◆◆◆◆◆ ◆◆ ◆◆◆ ◆◆◆◆◆◆ \n"\
" ◆◆ \n"\
" ◆◆ \n"
print(logo)
# open a thread for the Worker of Monitor
start_worker(worker_name="Monitor",nickname="Monitor")
logger.info("Monitor has started")
# 开启Web
if len(what) == 1:
# 没指定http端口，不开启Tornado
pass
else:
port = int(what[1])
start_worker(worker_name="Web", nickname="Tornado")
# 绑定退出信号
bind_quit_signals()
redis_conn = get_vendor("DB").get_redis()
command_listener = redis_conn.pubsub()
channel_name = "HySAS.Command"
command_listener.subscribe([channel_name])
# 数据关联 态势分析
start_worker(worker_name="Process",nickname="Process")
for item in command_listener.listen(): # 阻塞式接收
if isinstance(item['data'], bytes):
__command_handler__(item["data"])
else:
print("HySAS What?")
except Exception as e:
traceback.print_exc()
logger.error("{}".format(e))
print("Hail error?")
if __name__ == '__main__':
main(what = ["HySAS"])
================================================
FILE: HySAS/test.py
================================================
from multiprocessing import Process
import threading
import time
import os
import redis
import pickle
import logging
from HySAS.core.Functions import *
def __command_handler__(msg_command):
print(msg_command)
redis_conn = get_vendor("DB").get_redis()
command_listener = redis_conn.pubsub()
channel_name = "dHydra.Command"
command_listener.subscribe([channel_name])
# for item in command_listener.listen():
# if item["type"] == "message":
# print(pickle.loads(item['data']))
from datetime import datetime
# status = dict()
# status["heart_beat"] = time.time()
# status["nickname"] = "zyyz"
# redis_key = "HySAS"
#
# redis_conn.hmset(redis_key + "Info", status)
# info = redis_conn.hgetall(redis_key + "Info")
worker_names = []
# path = os.path.join(os.path.split(os.path.realpath(__file__))[0]+"/Worker")
# worker_names.extend(os.listdir(path))
try:
worker_names.extend(os.listdir(os.getcwd()+"/Worker"))
except FileNotFoundError as e:
print("HySAS运行目录下没有Worker文件夹")
print(worker_names)
================================================
FILE: HySAS/test1.py
================================================
import time
import os
import redis
import pickle
from HySAS.core.Functions import *
msg = {
"type": "sys",
"operation_name": "start_worker",
"kwargs":"haha"
}
print(pickle.dumps(msg))
__redis__ = get_vendor("DB").get_redis()
__redis__.publish("dHydra.Command", pickle.dumps(msg))
================================================
FILE: README.md
================================================
# HySAS简介
HySAS(Hydrogen Situation Awareness System )是一个安全态势感知系统的开发框架，用于实现多进程多数据源的实时计算。不管是在开发过程还是运行过程中，都拥有极高的并行性和扩展性。
**特点**
+ 采用Redis作为消息队列，实现进程间通信
+ 每个Worker都可以被独立工作，必要时可以相互配合
+ 支持动态联编，灵活加载功能模块
# 运行环境
- python 3.5以上 (开发环境windows 10, python 3.6)，**不对python2.7提供支持，多版本虚拟环境安装请参考安装HySAS文档**
- Mysql 8.0.13
- Redis X64 3.2
# Quick Start
1. Step 1:安装Redis（略）
2. Step 2:安装Mysql（略）
# 目录介绍
# 核心对象
## main.py
主程序入口
main程序订阅了`HySAS.Command`频道，通过向这个频道发送命令可以开启或关闭响应的服务
`worker_dict` : 对象字典`worker_dict[worker_name.nickname]：worker对象`用于记录系统中正在运行的worker对象
## console.py
允许通过命令行的形式，以第三方的身份动态加载Worker对象。旨在解决如果主程序中有部分Worker died之后，无需重启主程序，便可重启部分Worker，保证数据流的畅通性。
**用命令行启动/关闭**
```
start Demo DemoName
```
这样做就会开启一个叫`DemoName`的`Demo`进程
这里的`DemoName`是`nickname`，`nickname`是全局唯一的，可能有多个不同的进程都是Demo类，但是每个进程都有唯一的昵称。
以Worker为例：
> - 自己订阅了自己的频道
> - 每秒向Redis中的`HySAS.Worker.Demo.DemoName.Pub`频道发送一个数字
> - 将自己订阅收到的内容打印到屏幕
> 原理：之所以可以这样做是因为运行了`pip install --editable .`，它在你当前环境下认识了start命令，它会调用HySAS.console下的start方法，这个方法向Redis中的"HySAS.Command"频道发送了一条指令，HySAS Server监听到此指令以后就会实例化Demo目录下的DemoName类对象
```
stop DemoName
```
> 运行stop DemoName后，我们会向"HySAS.Command"频道发送一条指令，由HySAS Server去执行这个关闭Demo的任务，会捕捉到终止信号，并且执行用户自定义的`__before_termination__`
## 1.Worker
Worker类是HySAS的任务处理单元的基类，其继承于`multiprocessing.Process`。用于并行处理各个子任务，其派生类包括`Monitor`、`Process`。
###工作流程
`__init__（）`：初始化成员变量。比较重要的成员：`self.__data_feeder__ = set()`用于存储本Worker订阅的内容。
`run（）`：
`__on_start__()`：用于连接redis（所有Worker共同订阅了`HySAS`频道）、Mysql，并实例化logger
`on_start()`：自定义启动项，用于拓展接口
`__is_unique__()`：首先检查是否已经有相同的进程被开启，通过worker启动的时间，每个worker会生产一个唯一的`token`
`__thread_listen_command__`：监听命令线程（守护进程），订阅了`self.redis_key.Command`频道，其中`self.redis_key`由创建worker时传入的参数决定。
`__thread_pub__`：生产者进程（按需开启）
`__thread_sub__`：消费者进程（按需开启），获取了订阅`HySAS`频道的redis句柄，并实时接受数据进行处理
`__command_handler__`：上述的数据处理函数
`__heart_beat__()`：为该worker注册心跳包，用于检验该worker是否被多重开启、存活性等。通过在`self.redis_key.Info`频道发布自身状态信息 ，心跳包内容如下：
```python
status = dict()
#以下必带
status["heart_beat"] = time.time()
status["nickname"] = self.__nickname__
status["pid"] = self.pid
status["token"] = self.__token__
status["heart_beat_interval"] = self.__heart_beat_interval__
#以下选带
if self.__error_msg__:
status["error_msg"] = self.__error_msg__
if self.__stop_info__:
status["stop_info"] = self.__stop_info__
if self.__stop_info__:
status["stop_time"] = self.__stop_time__
if self.__status__:
status["status"] = self.__status__
if self.__threads__:
status["threads"] = copy.deepcopy(self.__threads__)
if self.__data_feeder__:
status["data_feeder"] = self.__data_feeder__
if self.__follower__:
status["follower"] = self.__follower__
self.__redis__.hmset(self.redis_key + "Info", status)
```
## 2.Monitor
继承于Worker，主要用于派生WorkerManger对象监视各个Worker的运行状态
## 3.WorkerManger
WorkerManger继承于Vendor，由Woker类的派生类对象Monitor重写了`run（）`中由`__thread_pub__`调用的`__producer__`生产。
### 工作流程
`__init__（）`：初始化成员变量，获取redis句柄。
`update_workers（）`：更新Worker信息。从redis获取所有对象的心跳包，以`dict[workerName][nickName] ：心跳包`的字典格式存入`work_info`。如果发现当前时间间隔大于worker的`heart_beat_interval`，即心跳停止，则放出warning，并从redis中删除该Worker信息。
gitextract_am3dz27m/ ├── HySAS/ │ ├── Vendor/ │ │ ├── DB/ │ │ │ └── DB.py │ │ └── WorkerManager/ │ │ ├── WorkerManager.py │ │ ├── __init__.py │ │ └── config.py │ ├── Worker/ │ │ ├── Monitor/ │ │ │ ├── Controller.py │ │ │ └── Monitor.py │ │ └── Process/ │ │ └── Process.py │ ├── config/ │ │ ├── event_mysql.json │ │ ├── raw_mysql.json │ │ └── redis.json │ ├── console.py │ ├── core/ │ │ ├── Functions.py │ │ ├── Vendor.py │ │ ├── Worker.py │ │ ├── log/ │ │ │ ├── DB/ │ │ │ │ ├── error.log │ │ │ │ └── info.log │ │ │ └── console/ │ │ │ ├── error.log │ │ │ └── info.log │ │ └── util.py │ ├── log/ │ │ ├── DB/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── Monitor/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── Process/ │ │ │ ├── error.log │ │ │ └── info.log │ │ ├── WorkerManager/ │ │ │ ├── error.log │ │ │ └── info.log │ │ └── console/ │ │ ├── error.log │ │ └── info.log │ ├── main.py │ ├── test.py │ └── test1.py └── README.md
SYMBOL INDEX (72 symbols across 11 files)
FILE: HySAS/Vendor/DB/DB.py
class DB (line 18) | class DB(Vendor):
method __init__ (line 20) | def __init__(self, **kwargs):
method get_MySQLdb (line 23) | def get_MySQLdb(self, config="raw_mysql.json", timeout=1500):
method get_redis (line 49) | def get_redis(self, config="redis.json"):
FILE: HySAS/Vendor/WorkerManager/WorkerManager.py
class WorkerManager (line 19) | class WorkerManager(Vendor):
method __init__ (line 21) | def __init__(self, **kwargs):
method get_worker_names (line 35) | def get_worker_names(self):
method get_workers_from_redis (line 39) | def get_workers_from_redis(self):
method update_workers (line 54) | def update_workers(self):
method remove_worker (line 127) | def remove_worker(
FILE: HySAS/Worker/Monitor/Monitor.py
class Monitor (line 7) | class Monitor(Worker):
method __init__ (line 9) | def __init__(self, **kwargs):
method start_worker (line 12) | def start_worker(self, worker_name, **kwargs):
method terminate_worker (line 16) | def terminate_worker(self, nickname=None, pid=None):
method __producer__ (line 21) | def __producer__(self):
method get_workers_info (line 36) | def get_workers_info(
method get_pid_by_nickname (line 55) | def get_pid_by_nickname(
FILE: HySAS/Worker/Process/Process.py
class Process (line 9) | class Process(Worker):
method __init__ (line 10) | def __init__(self, **kwargs):
method init_MySQLdb (line 14) | def init_MySQLdb(self):
method on_start (line 24) | def on_start(self):
method __producer__ (line 27) | def __producer__(self): #拥有数据库句柄 通过 self.db调用
method __data_handler__ (line 36) | def __data_handler__(self, msg):
FILE: HySAS/console.py
function init_logger (line 10) | def init_logger():
function start (line 17) | def start(worker_name=None, nickname=None):
function stop (line 58) | def stop(nickname=None):
function shutdown (line 68) | def shutdown(nickname=None):
function send_command (line 75) | def send_command(
FILE: HySAS/core/Functions.py
function get_workers (line 14) | def get_workers():
function get_controller_method (line 29) | def get_controller_method(class_name, method):
function V (line 52) | def V(name, vendor_name=None, **kwargs):
function get_vendor (line 56) | def get_vendor(name, vendor_name=None, **kwargs):
function get_worker_class (line 92) | def get_worker_class(worker_name, **kwargs):
function convert (line 126) | def convert(data):
FILE: HySAS/core/Vendor.py
class Vendor (line 6) | class Vendor:
method __init__ (line 8) | def __init__(
FILE: HySAS/core/Worker.py
class Worker (line 27) | class Worker(multiprocessing.Process):
method __init__ (line 30) | def __init__(
method __is_unique__ (line 126) | def __is_unique__(self):
method __auto_restart_thread__ (line 135) | def __auto_restart_thread__(self):
method __command_handler__ (line 139) | def __command_handler__(self, msg_command):
method monitor_add_thread (line 168) | def monitor_add_thread(self,
method monitor_remove_thread (line 176) | def monitor_remove_thread(self, thread):
method init_redis (line 180) | def init_redis(self):
method check_prerequisites (line 192) | def check_prerequisites(self):
method __listen_command__ (line 199) | def __listen_command__(self):
method __heart_beat__ (line 213) | def __heart_beat__(self):
method __producer__ (line 237) | def __producer__(self):
method __consumer__ (line 244) | def __consumer__(self):
method __data_handler__ (line 257) | def __data_handler__(self, msg):
method __before_termination__ (line 265) | def __before_termination__(self, sig):
method __on_termination__ (line 274) | def __on_termination__(self, sig, frame):
method publish (line 280) | def publish(self,data,channel_name=None):
method __on_start__ (line 289) | def __on_start__(self):
method on_start (line 312) | def on_start(self):
method run (line 315) | def run(self):
method subscribe (line 374) | def subscribe(self,channel_name=None, worker_name=None, nickname=None):
method unsubscribe (line 420) | def unsubscribe(self, worker_name=None, nickname=None):
FILE: HySAS/core/util.py
function get_worker_names (line 7) | def get_worker_names(logger=None):
function get_logger (line 25) | def get_logger(
function generate_token (line 96) | def generate_token():
function read_config (line 102) | def read_config(file_path):
FILE: HySAS/main.py
function __on_termination__ (line 12) | def __on_termination__(sig, frame):
function bind_quit_signals (line 19) | def bind_quit_signals():
function start_worker (line 35) | def start_worker(worker_name, **kwargs):
function terminate_worker (line 41) | def terminate_worker(nickname=None, pid=None):
function get_workers_info (line 54) | def get_workers_info(redis_cli=None, by="nickname", nickname=None, worke...
function get_pid_by_nickname (line 68) | def get_pid_by_nickname(redis_cli=None, nickname=None):
function __command_handler__ (line 82) | def __command_handler__(msg_command):
function signal_handler (line 134) | def signal_handler():
function main (line 143) | def main(what=None):
FILE: HySAS/test.py
function __command_handler__ (line 12) | def __command_handler__(msg_command):
Condensed preview — 33 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (79K chars).
[
{
"path": "HySAS/Vendor/DB/DB.py",
"chars": 1930,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n# Created on\n# @author:\n# @contact:\n\"\"\"\n# 以下是自动生成的 #\n# --- 导入系统配置\nimport core.util as util\nf"
},
{
"path": "HySAS/Vendor/WorkerManager/WorkerManager.py",
"chars": 5069,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n# Created on\n# @author:\n# @contact:\n\"\"\"\n# 以下是自动生成的 #\n# --- 导入系统配置\nimport core.util as util\nf"
},
{
"path": "HySAS/Vendor/WorkerManager/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/Vendor/WorkerManager/config.py",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/Worker/Monitor/Controller.py",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/Worker/Monitor/Monitor.py",
"chars": 1979,
"preview": "from core.Worker import Worker\nfrom core.Functions import get_vendor, get_worker_class\nimport os\nimport signal\n\n\nclass M"
},
{
"path": "HySAS/Worker/Process/Process.py",
"chars": 967,
"preview": "\"\"\"\n数据处理函数\n\"\"\"\nfrom core.Worker import Worker\nfrom core.Functions import get_vendor\nfrom core import util\nimport sys\n\ncl"
},
{
"path": "HySAS/config/event_mysql.json",
"chars": 86,
"preview": "{\n \"host\": \"127.0.0.1\",\n \"port\": 6379,\n \"user\": \"\",\n \"password\": \"\",\n \"db\": \"\"\n}\n"
},
{
"path": "HySAS/config/raw_mysql.json",
"chars": 97,
"preview": "{\n \"host\": \"127.0.0.1\",\n \"port\": 3306,\n \"user\": \"root\",\n \"password\": \"123\",\n \"db\": \"test\"\n}\n"
},
{
"path": "HySAS/config/redis.json",
"chars": 42,
"preview": "{\n \"host\": \"127.0.0.1\",\n \"port\": 6379\n}\n"
},
{
"path": "HySAS/console.py",
"chars": 2357,
"preview": "# -*- coding: utf-8 -*-\nimport logging\nfrom core.Functions import *\nimport core.util as util\nimport click\nimport traceba"
},
{
"path": "HySAS/core/Functions.py",
"chars": 3612,
"preview": "\"\"\"\n框架的全局方法，在主程序运行时会被引用\n---\nCreated on 01/14/2019\n@author: Happy-zyy\n@contact: 617532750@qq.com\n\"\"\"\nimport importlib\nimp"
},
{
"path": "HySAS/core/Vendor.py",
"chars": 1250,
"preview": "# -*- coding: utf-8 -*-\nimport logging\nimport core.util as util\n\n\nclass Vendor:\n\n def __init__(\n self,\n "
},
{
"path": "HySAS/core/Worker.py",
"chars": 14968,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\nWorker抽象类\nCreated on 01/14/2019\n@author: Happy-zyy\n@contact: 617532750@qq.com\n\"\"\"\nimport mul"
},
{
"path": "HySAS/core/log/DB/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/core/log/DB/info.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/core/log/console/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/core/log/console/info.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/core/util.py",
"chars": 3629,
"preview": "import logging\nimport os\nimport hashlib\nimport time\nimport json\n\ndef get_worker_names(logger=None):\n \"\"\"\n 根据文件夹名字返"
},
{
"path": "HySAS/log/DB/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/log/DB/info.log",
"chars": 18783,
"preview": "2019-01-15 10:38:23,552 - DB - 59 - INFO - Trying to connect to redis\n2019-01-15 10:38:25,562 - DB - 68 - WARNING - Fail"
},
{
"path": "HySAS/log/Monitor/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/log/Monitor/info.log",
"chars": 969,
"preview": "2019-01-16 11:23:45,365 - Monitor - 326 - INFO - ʼWorker\n2019-01-16 11:25:59,817 - Monitor - 326 - INFO - ʼWorker\n2019-0"
},
{
"path": "HySAS/log/Process/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/log/Process/info.log",
"chars": 855,
"preview": "2019-01-16 11:26:42,301 - Process - 326 - INFO - ʼWorker\n2019-01-16 11:59:44,056 - Process - 326 - INFO - ʼWorker\n2019-0"
},
{
"path": "HySAS/log/WorkerManager/error.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/log/WorkerManager/info.log",
"chars": 0,
"preview": ""
},
{
"path": "HySAS/log/console/error.log",
"chars": 288,
"preview": "2019-01-16 11:23:18,674 - console - 168 - ERROR - 'NoneType' object has no attribute 'nickname'\n2019-01-16 11:23:44,737 "
},
{
"path": "HySAS/log/console/info.log",
"chars": 1734,
"preview": "2019-01-15 15:02:10,563 - console - 86 - INFO - Monitor has started\n2019-01-15 15:10:45,019 - console - 86 - INFO - Moni"
},
{
"path": "HySAS/main.py",
"chars": 5958,
"preview": "from console import logger\nfrom core.Functions import *\nimport time\nimport signal\nimport click\nimport pickle\n\n__redis__ "
},
{
"path": "HySAS/test.py",
"chars": 1022,
"preview": "from multiprocessing import Process\nimport threading\nimport time\nimport os\nimport redis\nimport pickle\nimport logging\n\nfr"
},
{
"path": "HySAS/test1.py",
"chars": 309,
"preview": "import time\nimport os\nimport redis\nimport pickle\nfrom HySAS.core.Functions import *\n\nmsg = {\n \"type\": \"sys\",\n "
},
{
"path": "README.md",
"chars": 3133,
"preview": "# HySAS简介\nHySAS(Hydrogen Situation Awareness System )是一个安全态势感知系统的开发框架，用于实现多进程多数据源的实时计算。不管是在开发过程还是运行过程中，都拥有极高的并行性和扩展性。 \n"
}
]
About this extraction
This page contains the full source code of the Happy-zyy/MyProject GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 33 files (67.4 KB), approximately 22.2k tokens, and a symbol index with 72 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.