SYMBOL INDEX (346 symbols across 70 files) FILE: apps/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: apps/client_db.py function get_item (line 32) | def get_item(model_class, pk_id): function get_all (line 41) | def get_all(model_class, *args, **kwargs): function get_distinct (line 50) | def get_distinct(model_class, field, *args, **kwargs): function get_group (line 59) | def get_group(model_class, field, min_count=0, *args, **kwargs): function add_item (line 70) | def add_item(model_class, data): FILE: apps/client_rk.py function get_img_code (line 36) | def get_img_code(im, im_type_id): function img_report_error (line 52) | def img_report_error(im_id): function check_counter_limit (line 56) | def check_counter_limit(): function check_cookies_count (line 65) | def check_cookies_count(spider_name): function counter_clear (line 74) | def counter_clear(): FILE: db/schema/mysql.sql type `channel` (line 8) | CREATE TABLE `channel` ( type `fetch_task` (line 20) | CREATE TABLE `fetch_task` ( type `fetch_result` (line 37) | CREATE TABLE `fetch_result` ( type `log_task_scheduling` (line 65) | CREATE TABLE `log_task_scheduling` ( FILE: libs/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: libs/counter.py class CounterClient (line 14) | class CounterClient(object): method __init__ (line 19) | def __init__(self, redis_client, entity_name, prefix='counter'): method increase (line 28) | def increase(self, amount=1): method decrease (line 36) | def decrease(self, amount=1): method get (line 44) | def get(self): method clear (line 51) | def clear(self): FILE: libs/ft.py class FTClient (line 20) | class FTClient(object): method __init__ (line 21) | def __init__(self, pd_id, pd_key, app_id='', app_key=''): method calc_sign (line 31) | def calc_sign(pd_id, pd_key, timestamp): method calc_card_sign (line 42) | def calc_card_sign(card_id, card_key, timestamp, pd_key): method query_balance (line 47) | def query_balance(self): method query_tts (line 60) | def query_tts(self, predict_type): method predict (line 78) | def predict(self, predict_type, img_data): method predict_from_file (line 98) | def predict_from_file(self, predict_type, file_name): method justice (line 104) | def justice(self, request_id): method charge (line 120) | def charge(self, card_id, card_key): function test_ft (line 137) | def test_ft(): FILE: libs/optical_modem.py class OpticalModemChinaNet (line 21) | class OpticalModemChinaNet(object): method __init__ (line 27) | def __init__(self, host='192.168.1.1', username='useradmin', password=... method _get_tc (line 43) | def _get_tc(): method login (line 47) | def login(self): method get_wan_wifi_status (line 59) | def get_wan_wifi_status(self): method reboot (line 84) | def reboot(self): method get_net_ip (line 104) | def get_net_ip(self): method check_reboot_status (line 118) | def check_reboot_status(self): class OpticalModemChinaMobile (line 124) | class OpticalModemChinaMobile(object): method __init__ (line 133) | def __init__(self, host='192.168.1.1', username='user', password='gkw4... method _get_pwd_random (line 151) | def _get_pwd_random(): method _get_encryption_pwd (line 156) | def _get_encryption_pwd(pwd, r): method _get_token (line 160) | def _get_token(self): method _get_pid (line 168) | def _get_pid(self): method _get_session_token (line 177) | def _get_session_token(self): method login (line 186) | def login(self): method reboot (line 202) | def reboot(self): method get_net_ip (line 216) | def get_net_ip(self): method check_reboot_status (line 230) | def check_reboot_status(self): function test_china_net (line 236) | def test_china_net(): function test_china_mobile (line 259) | def test_china_mobile(): FILE: libs/redis_pub_sub.py class RedisPubSub (line 14) | class RedisPubSub(object): method __init__ (line 22) | def __init__(self, name, namespace='pub/sub', redis_client=None, **red... method pub (line 27) | def pub(self, k, v): method sub (line 37) | def sub(self, k): method p_sub (line 52) | def p_sub(self, k): method sub_not_loop (line 71) | def sub_not_loop(self, k): method p_sub_not_loop (line 84) | def p_sub_not_loop(self, k): FILE: libs/redis_queue.py class RedisQueue (line 14) | class RedisQueue(object): method __init__ (line 17) | def __init__(self, name, namespace='queue', redis_client=None, **redis... method qsize (line 22) | def qsize(self): method empty (line 26) | def empty(self): method put (line 30) | def put(self, item): method get (line 34) | def get(self, block=True, timeout=None): method get_nowait (line 50) | def get_nowait(self): FILE: libs/rk.py class RKClient (line 16) | class RKClient(object): method __init__ (line 17) | def __init__(self, username, password, soft_id, soft_key): method rk_create (line 34) | def rk_create(self, im, im_type, timeout=60): method rk_report_error (line 54) | def rk_report_error(self, im_id): FILE: libs/weed_fs.py class WeedFSClient (line 25) | class WeedFSClient(object): method __init__ (line 32) | def __init__(self, weed_fs_url): method _get_assign (line 35) | def _get_assign(self): method _get_locations (line 49) | def _get_locations(self, fid): method save_file (line 58) | def save_file(self, local_file_path=None, remote_file_path=None, file_... method get_file_url (line 82) | def get_file_url(self, fid, separator=None): method read_csv (line 90) | def read_csv(self, fid, encoding=None): FILE: maps/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: models/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: models/news.py function to_dict (line 10) | def to_dict(self): class Channel (line 16) | class Channel(Base): class FetchResult (line 27) | class FetchResult(Base): class FetchTask (line 53) | class FetchTask(Base): class LogTaskScheduling (line 72) | class LogTaskScheduling(Base): FILE: news/items.py class FetchTaskItem (line 11) | class FetchTaskItem(scrapy.Item): class FetchResultItem (line 26) | class FetchResultItem(scrapy.Item): class ChannelItem (line 47) | class ChannelItem(scrapy.Item): FILE: news/middlewares.py class NewsSpiderMiddleware (line 11) | class NewsSpiderMiddleware(object): method from_crawler (line 17) | def from_crawler(cls, crawler): method process_spider_input (line 23) | def process_spider_input(self, response, spider): method process_spider_output (line 30) | def process_spider_output(self, response, result, spider): method process_spider_exception (line 38) | def process_spider_exception(self, response, exception, spider): method process_start_requests (line 46) | def process_start_requests(self, start_requests, spider): method spider_opened (line 55) | def spider_opened(self, spider): class NewsDownloaderMiddleware (line 59) | class NewsDownloaderMiddleware(object): method from_crawler (line 65) | def from_crawler(cls, crawler): method process_request (line 71) | def process_request(self, request, spider): method process_response (line 83) | def process_response(self, request, response, spider): method process_exception (line 92) | def process_exception(self, request, exception, spider): method spider_opened (line 102) | def spider_opened(self, spider): FILE: news/middlewares/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: news/middlewares/anti_spider.py class AntiSpiderMiddleware (line 18) | class AntiSpiderMiddleware(object): method __init__ (line 26) | def __init__(self, settings): method from_crawler (line 34) | def from_crawler(cls, crawler): method process_request (line 37) | def process_request(self, request, spider): method process_response (line 51) | def process_response(self, request, response, spider): FILE: news/middlewares/content_type.py class ContentTypeGb2312Middleware (line 9) | class ContentTypeGb2312Middleware(object): method process_response (line 17) | def process_response(self, request, response, spider): FILE: news/middlewares/de_duplication_request.py class DeDuplicationRequestMiddleware (line 14) | class DeDuplicationRequestMiddleware(object): method process_request (line 19) | def process_request(self, request, spider): FILE: news/middlewares/httpproxy.py class HttpProxyMiddleware (line 13) | class HttpProxyMiddleware(object): method __init__ (line 17) | def __init__(self, settings): method from_crawler (line 25) | def from_crawler(cls, crawler): method process_request (line 28) | def process_request(self, request, spider): method process_exception (line 35) | def process_exception(self, request, exception, spider): FILE: news/middlewares/useragent.py class UserAgentMiddleware (line 12) | class UserAgentMiddleware(object): method __init__ (line 16) | def __init__(self, agents): method from_crawler (line 20) | def from_crawler(cls, crawler): method process_request (line 23) | def process_request(self, request, spider): FILE: news/pipelines.py class NewsPipeline (line 9) | class NewsPipeline(object): method process_item (line 10) | def process_item(self, item, spider): FILE: news/pipelines/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: news/pipelines/de_duplication_request.py class DeDuplicationRequestPipeline (line 14) | class DeDuplicationRequestPipeline(object): method process_item (line 21) | def process_item(self, item, spider): FILE: news/pipelines/de_duplication_store_mysql.py class DeDuplicationStoreMysqlPipeline (line 18) | class DeDuplicationStoreMysqlPipeline(object): method process_item (line 24) | def process_item(self, item, spider): FILE: news/pipelines/exporter_csv.py class CsvExportPipeline (line 13) | class CsvExportPipeline(object): method __init__ (line 14) | def __init__(self): method from_crawler (line 19) | def from_crawler(cls, crawler): method spider_opened (line 25) | def spider_opened(self, spider): method spider_closed (line 31) | def spider_closed(self, spider): method process_item (line 36) | def process_item(self, item, spider): FILE: news/pipelines/img_remote_to_local_fs.py function remote_to_local (line 25) | def remote_to_local(remote_file_path): function add_src (line 36) | def add_src(html_body, base=''): function replace_src (line 61) | def replace_src(html_body, base=''): class ImgRemoteToLocalFSPipeline (line 89) | class ImgRemoteToLocalFSPipeline(object): method process_item (line 96) | def process_item(self, item, spider): FILE: news/pipelines/store_mysql.py class StoreMysqlPipeline (line 14) | class StoreMysqlPipeline(object): method process_item (line 19) | def process_item(self, item, spider): FILE: news/spiders/ip.py class IpSpider (line 5) | class IpSpider(scrapy.Spider): method parse (line 38) | def parse(self, response): FILE: news/spiders/toutiao_m.py class ToutiaoMSpider (line 23) | class ToutiaoMSpider(scrapy.Spider): method start_requests (line 56) | def start_requests(self): method get_profile (line 91) | def get_profile(self, response): method jssdk_signature (line 107) | def jssdk_signature(self, response): method parse_article_list (line 132) | def parse_article_list(self, response): method parse_article_detail (line 166) | def parse_article_detail(self, response): FILE: news/spiders/weibo.py class WeiboSpider (line 27) | class WeiboSpider(scrapy.Spider): method parse (line 81) | def parse(self, response): method passport_weibo_login (line 84) | def passport_weibo_login(self): method login_sina_sso_prelogin (line 87) | def login_sina_sso_prelogin(self, response): method passport_weibo_sso_login (line 101) | def passport_weibo_sso_login(self, response): method after_login (line 110) | def after_login(self, response): method crossdomain_weibo_com (line 137) | def crossdomain_weibo_com(self, response): method crossdomain_sina_com_cn (line 147) | def crossdomain_sina_com_cn(self, response): method crossdomain_weibo_cn (line 157) | def crossdomain_weibo_cn(self, response): method weibo_cn_index (line 166) | def weibo_cn_index(self, response): method weibo_com_index (line 183) | def weibo_com_index(self, response): method parse_follow_list (line 199) | def parse_follow_list(self, response): method follow_home_list (line 217) | def follow_home_list(self, response): method get_article_task (line 225) | def get_article_task(self): method replace_all (line 254) | def replace_all(input_html, replace_dict): method parse_article_list (line 262) | def parse_article_list(self, response): method parse_article_detail_html (line 325) | def parse_article_detail_html(self, response): method trans_time (line 352) | def trans_time(time_str): method parse_article_detail_js (line 364) | def parse_article_detail_js(self, response): FILE: news/spiders/weixin.py class WeixinSpider (line 21) | class WeixinSpider(scrapy.Spider): method start_requests (line 53) | def start_requests(self): method parse_article_search_list (line 87) | def parse_article_search_list(self, response): method parse_account_search_list (line 97) | def parse_account_search_list(self, response): method parse_account_article_list (line 107) | def parse_account_article_list(self, response): method parse_detail (line 127) | def parse_detail(self, response): FILE: tasks/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: tasks/job_put_tasks.py function job_put_tasks (line 19) | def job_put_tasks(spider_name): function usage (line 50) | def usage(): function run (line 61) | def run(): FILE: tasks/job_reboot_net_china_net.py function job_reboot_net_china_net (line 19) | def job_reboot_net_china_net(): FILE: tasks/jobs_proxies.py function job_proxies (line 19) | def job_proxies(spider_name, mix_num=0): function usage (line 28) | def usage(): function run (line 39) | def run(): FILE: tasks/jobs_sogou.py function job_sogou_cookies (line 17) | def job_sogou_cookies(spider_name): FILE: tasks/jobs_weixin.py function set_anti_spider_task (line 23) | def set_anti_spider_task(spider_name, msg): function _get_anti_spider_task (line 42) | def _get_anti_spider_task(spider_name): function _set_anti_spider_result (line 50) | def _set_anti_spider_result(spider_name, msg): function _get_anti_spider_result (line 58) | def _get_anti_spider_result(spider_name): function sub_anti_spider (line 66) | def sub_anti_spider(spider_name): function _pub_anti_spider (line 77) | def _pub_anti_spider(spider_name, msg): function job_weixin_cookies (line 88) | def job_weixin_cookies(spider_name): function usage (line 127) | def usage(): function run (line 132) | def run(): FILE: tasks/run_job_counter_clear.py function run (line 23) | def run(): FILE: tasks/run_job_put_tasks_toutiao.py function run (line 24) | def run(): FILE: tasks/run_job_put_tasks_weibo.py function run (line 24) | def run(): FILE: tasks/run_job_put_tasks_weixin.py function run (line 24) | def run(): FILE: tasks/run_job_reboot_net_china_net.py function run (line 24) | def run(): FILE: tasks/run_job_sogou_cookies.py function run (line 23) | def run(): FILE: tasks/run_job_weixin_cookies.py function run (line 23) | def run(): FILE: tasks/run_jobs.py function run (line 37) | def run(): FILE: tasks/run_jobs_apscheduler.py function add_job_store_redis (line 30) | def add_job_store_redis(): function add_job (line 69) | def add_job(): function run_blocking (line 131) | def run_blocking(): FILE: tests/__init__.py function func (line 12) | def func(): class Main (line 16) | class Main(object): method __init__ (line 17) | def __init__(self): FILE: tests/test_date_time.py class DateTimeTest (line 21) | class DateTimeTest(unittest.TestCase): method setUp (line 25) | def setUp(self): method test_local_to_utc (line 37) | def test_local_to_utc(self): method test_utc_to_local (line 48) | def test_utc_to_local(self): method tearDown (line 59) | def tearDown(self): FILE: tests/test_finger.py class FingerTest (line 20) | class FingerTest(unittest.TestCase): method setUp (line 25) | def setUp(self): method test_request (line 29) | def test_request(self): method tearDown (line 42) | def tearDown(self): class MD5Test (line 46) | class MD5Test(unittest.TestCase): method setUp (line 51) | def setUp(self): method test_request (line 55) | def test_request(self): method tearDown (line 70) | def tearDown(self): FILE: tools/__init__.py function catch_keyboard_interrupt (line 14) | def catch_keyboard_interrupt(func): FILE: tools/anti_spider_sogou.py function _get_tc (line 47) | def _get_tc(): function _save_img (line 52) | def _save_img(res): function anti_spider (line 62) | def anti_spider(): function code_img_save (line 78) | def code_img_save(): function code_img_obj (line 99) | def code_img_obj(): function pv_refresh (line 115) | def pv_refresh(): function pv_index (line 139) | def pv_index(): function pv_img_cost (line 163) | def pv_img_cost(): function pv_mouse (line 189) | def pv_mouse(): function pv_img_success (line 215) | def pv_img_success(): function pv_real_index (line 241) | def pv_real_index(): function pv_seccode_focus (line 267) | def pv_seccode_focus(): function pv_seccode_input (line 293) | def pv_seccode_input(): function pv_seccode_blur (line 319) | def pv_seccode_blur(): function thank (line 345) | def thank(code_anti_spider): function _get_cookies (line 378) | def _get_cookies(): function check_n (line 383) | def check_n(): function check_y (line 389) | def check_y(): function manual_cookies (line 395) | def manual_cookies(): function auto_cookies (line 425) | def auto_cookies(): FILE: tools/anti_spider_weixin.py function _get_tc (line 47) | def _get_tc(): function _save_img (line 52) | def _save_img(res): function anti_spider (line 62) | def anti_spider(url): function code_img_save (line 78) | def code_img_save(): function code_img_obj (line 99) | def code_img_obj(): function verify_code (line 115) | def verify_code(input_code): function _get_cookies (line 140) | def _get_cookies(): function manual_cookies (line 145) | def manual_cookies(): function auto_cookies (line 157) | def auto_cookies(url): FILE: tools/char.py function un_escape (line 21) | def un_escape(char_str): function get_js_36_str (line 30) | def get_js_36_str(i): FILE: tools/cookies.py function _get_cookies_str (line 20) | def _get_cookies_str(cookies_dict): function _get_finger (line 55) | def _get_finger(cookies_str): function get_cookies (line 66) | def get_cookies(spider_name): function add_cookies (line 83) | def add_cookies(spider_name, cookies_obj): function del_cookies (line 104) | def del_cookies(spider_name, cookies_id): function len_cookies (line 118) | def len_cookies(spider_name): FILE: tools/date_time.py function get_tc (line 20) | def get_tc(): function get_current_day_time_ends (line 29) | def get_current_day_time_ends(): function get_current_month_time_ends (line 40) | def get_current_month_time_ends(): function get_current_year_time_ends (line 52) | def get_current_year_time_ends(): function get_hours (line 63) | def get_hours(zerofill=True): function get_days (line 74) | def get_days(year=1970, month=1, zerofill=True): function get_weeks (line 91) | def get_weeks(): function get_months (line 99) | def get_months(zerofill=True): function time_local_to_utc (line 110) | def time_local_to_utc(local_time): function time_utc_to_local (line 128) | def time_utc_to_local(utc_time): FILE: tools/duplicate.py function is_dup_detail (line 18) | def is_dup_detail(detail_url, spider_name, channel_id=0): function add_dup_detail (line 31) | def add_dup_detail(detail_url, spider_name, channel_id=0): FILE: tools/gen.py function gen_models (line 25) | def gen_models(): function gen_items (line 50) | def gen_items(): function run (line 84) | def run(): function usage (line 101) | def usage(): FILE: tools/img.py function filter_img_size (line 26) | def filter_img_size(min_width=0, min_height=0, *img_url): function filter_local_img_type (line 50) | def filter_local_img_type(ignore_type='gif', *img_path): function filter_remote_img_type (line 67) | def filter_remote_img_type(ignore_type='gif', *img_url): FILE: tools/import_task.py function read_csv (line 21) | def read_csv(filename): function import_csv (line 37) | def import_csv(filename): function usage (line 51) | def usage(): function run (line 59) | def run(): FILE: tools/net_status.py function get_reboot_net_status (line 15) | def get_reboot_net_status(net_name='optical_modem_china_net'): function set_reboot_net_status (line 21) | def set_reboot_net_status(net_name='optical_modem_china_net'): function del_reboot_net_status (line 27) | def del_reboot_net_status(net_name='optical_modem_china_net'): FILE: tools/proxies.py function add_proxy (line 23) | def add_proxy(spider_name, *proxy): function del_proxy (line 28) | def del_proxy(spider_name, proxy): function get_proxy (line 33) | def get_proxy(spider_name): function len_proxy (line 38) | def len_proxy(spider_name): function fetch_proxy (line 43) | def fetch_proxy(country='China', scheme='http'): FILE: tools/scrapy_tasks.py function pop_task (line 15) | def pop_task(spider_name): function put_task (line 20) | def put_task(spider_name, *task_ids): function get_tasks_count (line 25) | def get_tasks_count(spider_name): FILE: tools/sys_monitor.py function bytes2human (line 16) | def bytes2human(n): function _format_info (line 34) | def _format_info(k, v): function _print_info (line 43) | def _print_info(contents, topic=''): function _cpu (line 51) | def _cpu(): function _memory (line 59) | def _memory(): function _disks (line 72) | def _disks(): function _network (line 88) | def _network(speed=True): function _sensors (line 101) | def _sensors(): function stats (line 121) | def stats(): FILE: tools/toutiao_m.py function get_as_cp (line 21) | def get_as_cp(): function parse_toutiao_js_body (line 46) | def parse_toutiao_js_body(html_body, url=''): class ParseJsTt (line 61) | class ParseJsTt(object): method __init__ (line 66) | def __init__(self, js_body): method _add_js_item_id_fn (line 78) | def _add_js_item_id_fn(self): method _add_js_title_fn (line 86) | def _add_js_title_fn(self): method _add_js_abstract_fn (line 94) | def _add_js_abstract_fn(self): method _add_js_content_fn (line 102) | def _add_js_content_fn(self): method _add_js_pub_time (line 110) | def _add_js_pub_time(self): method _add_js_tags_fn (line 118) | def _add_js_tags_fn(self): method parse_js_item_id (line 126) | def parse_js_item_id(self): method parse_js_title (line 129) | def parse_js_title(self): method parse_js_abstract (line 132) | def parse_js_abstract(self): method parse_js_content (line 135) | def parse_js_content(self): method parse_js_pub_time (line 138) | def parse_js_pub_time(self): method parse_js_tags (line 141) | def parse_js_tags(self): FILE: tools/url.py function get_update_url (line 21) | def get_update_url(url, data): function get_url_query_param (line 34) | def get_url_query_param(url, param): function get_request_finger (line 45) | def get_request_finger(url): function allow_url (line 55) | def allow_url(url, allow_domains): FILE: tools/weibo.py function get_su (line 20) | def get_su(user_name): function get_login_data (line 24) | def get_login_data(): FILE: tools/weixin.py function get_finger (line 31) | def get_finger(content_str): function parse_weixin_js_body (line 42) | def parse_weixin_js_body(html_body, url=''): function parse_weixin_article_id (line 56) | def parse_weixin_article_id(html_body): function add_img_src (line 62) | def add_img_src(html_body): function get_img_src_list (line 72) | def get_img_src_list(html_body, host_name='/', limit=None): function check_article_title_duplicate (line 80) | def check_article_title_duplicate(article_title): class ParseJsWc (line 94) | class ParseJsWc(object): method __init__ (line 98) | def __init__(self, js_body): method _add_js_msg_list_fn (line 106) | def _add_js_msg_list_fn(self): method parse_js_msg_list (line 114) | def parse_js_msg_list(self):