SYMBOL INDEX (1327 symbols across 95 files) FILE: docs/conf.py class Mock (line 12) | class Mock(MagicMock): method __getattr__ (line 14) | def __getattr__(cls, name): FILE: pyspider/database/__init__.py function connect_database (line 12) | def connect_database(url): function _connect_database (line 52) | def _connect_database(url): # NOQA function _connect_mysql (line 102) | def _connect_mysql(parsed,dbtype): function _connect_sqlite (line 128) | def _connect_sqlite(parsed,dbtype): function _connect_mongodb (line 151) | def _connect_mongodb(parsed,dbtype,url): function _connect_sqlalchemy (line 170) | def _connect_sqlalchemy(parsed, dbtype,url, other_scheme): function _connect_elasticsearch (line 187) | def _connect_elasticsearch(parsed, dbtype): function _connect_couchdb (line 209) | def _connect_couchdb(parsed, dbtype, url): FILE: pyspider/database/base/projectdb.py class ProjectDB (line 28) | class ProjectDB(object): method insert (line 37) | def insert(self, name, obj={}): method update (line 40) | def update(self, name, obj={}, **kwargs): method get_all (line 43) | def get_all(self, fields=None): method get (line 46) | def get(self, name, fields): method drop (line 49) | def drop(self, name): method check_update (line 52) | def check_update(self, timestamp, fields=None): method split_group (line 55) | def split_group(self, group, lower=True): method verify_project_name (line 61) | def verify_project_name(self, name): method copy (line 68) | def copy(self): FILE: pyspider/database/base/resultdb.py class ResultDB (line 20) | class ResultDB(object): method save (line 26) | def save(self, project, taskid, url, result): method select (line 29) | def select(self, project, fields=None, offset=0, limit=None): method count (line 32) | def count(self, project): method get (line 35) | def get(self, project, taskid, fields=None): method drop (line 38) | def drop(self, project): method copy (line 41) | def copy(self): FILE: pyspider/database/base/taskdb.py class TaskDB (line 59) | class TaskDB(object): method load_tasks (line 67) | def load_tasks(self, status, project=None, fields=None): method get_task (line 70) | def get_task(self, project, taskid, fields=None): method status_count (line 73) | def status_count(self, project): method insert (line 79) | def insert(self, project, taskid, obj={}): method update (line 82) | def update(self, project, taskid, obj={}, **kwargs): method drop (line 85) | def drop(self, project): method status_to_string (line 89) | def status_to_string(status): method status_to_int (line 98) | def status_to_int(status): method copy (line 106) | def copy(self): FILE: pyspider/database/basedb.py class BaseDB (line 17) | class BaseDB: method escape (line 29) | def escape(string): method dbcur (line 33) | def dbcur(self): method _execute (line 36) | def _execute(self, sql_query, values=[]): method _select (line 41) | def _select(self, tablename=None, what="*", where="", where_values=[],... method _select2dic (line 58) | def _select2dic(self, tablename=None, what="*", where="", where_values... method _replace (line 84) | def _replace(self, tablename=None, **values): method _insert (line 100) | def _insert(self, tablename=None, **values): method _update (line 116) | def _update(self, tablename=None, where="1=0", where_values=[], **valu... method _delete (line 126) | def _delete(self, tablename=None, where="1=0", where_values=[]): class DB (line 138) | class DB(BaseDB): method __init__ (line 142) | def __init__(self): method dbcur (line 151) | def dbcur(self): FILE: pyspider/database/couchdb/couchdbbase.py class SplitTableMixin (line 4) | class SplitTableMixin(object): method __init__ (line 7) | def __init__(self): method _collection_name (line 13) | def _collection_name(self, project): method projects (line 21) | def projects(self): method projects (line 28) | def projects(self, value): method _list_project (line 32) | def _list_project(self): method create_database (line 49) | def create_database(self, name): method get_doc (line 57) | def get_doc(self, db_name, doc_id): method get_docs (line 65) | def get_docs(self, db_name, selector): method get_all_docs (line 74) | def get_all_docs(self, db_name): method insert_doc (line 78) | def insert_doc(self, db_name, doc_id, doc): method update_doc (line 83) | def update_doc(self, db_name, doc_id, new_doc): method delete (line 93) | def delete(self, url): FILE: pyspider/database/couchdb/projectdb.py class ProjectDB (line 6) | class ProjectDB(BaseProjectDB): method __init__ (line 9) | def __init__(self, url, database='projectdb', username=None, password=... method _default_fields (line 37) | def _default_fields(self, each): method insert (line 49) | def insert(self, name, obj={}): method update (line 57) | def update(self, name, obj={}, **kwargs): method get_all (line 69) | def get_all(self, fields=None): method get (line 82) | def get(self, name, fields=None): method check_update (line 97) | def check_update(self, timestamp, fields=None): method drop (line 105) | def drop(self, name): method drop_database (line 111) | def drop_database(self): FILE: pyspider/database/couchdb/resultdb.py class ResultDB (line 6) | class ResultDB(SplitTableMixin, BaseResultDB): method __init__ (line 9) | def __init__(self, url, database='resultdb', username=None, password=N... method _get_collection_name (line 20) | def _get_collection_name(self, project): method _create_project (line 23) | def _create_project(self, project): method save (line 38) | def save(self, project, taskid, url, result): method select (line 50) | def select(self, project, fields=None, offset=0, limit=0): method count (line 76) | def count(self, project): method get (line 84) | def get(self, project, taskid, fields=None): method drop_database (line 101) | def drop_database(self): method drop (line 104) | def drop(self, project): FILE: pyspider/database/couchdb/taskdb.py class TaskDB (line 6) | class TaskDB(SplitTableMixin, BaseTaskDB): method __init__ (line 9) | def __init__(self, url, database='taskdb', username=None, password=None): method _get_collection_name (line 23) | def _get_collection_name(self, project): method _create_project (line 26) | def _create_project(self, project): method load_tasks (line 40) | def load_tasks(self, status, project=None, fields=None): method get_task (line 54) | def get_task(self, project, taskid, fields=None): method status_count (line 67) | def status_count(self, project): method insert (line 88) | def insert(self, project, taskid, obj={}): method update (line 97) | def update(self, project, taskid, obj={}, **kwargs): method drop_database (line 104) | def drop_database(self): method drop (line 107) | def drop(self, project): FILE: pyspider/database/elasticsearch/projectdb.py class ProjectDB (line 15) | class ProjectDB(BaseProjectDB): method __init__ (line 18) | def __init__(self, hosts, index='pyspider'): method insert (line 31) | def insert(self, name, obj={}): method update (line 46) | def update(self, name, obj={}, **kwargs): method get_all (line 53) | def get_all(self, fields=None): method get (line 59) | def get(self, name, fields=None): method check_update (line 64) | def check_update(self, timestamp, fields=None): method drop (line 71) | def drop(self, name): FILE: pyspider/database/elasticsearch/resultdb.py class ResultDB (line 16) | class ResultDB(BaseResultDB): method __init__ (line 19) | def __init__(self, hosts, index='pyspider'): method projects (line 35) | def projects(self): method save (line 42) | def save(self, project, taskid, url, result): method select (line 53) | def select(self, project, fields=None, offset=0, limit=0): method count (line 70) | def count(self, project): method get (line 75) | def get(self, project, taskid, fields=None): method drop (line 80) | def drop(self, project): method refresh (line 87) | def refresh(self): FILE: pyspider/database/elasticsearch/taskdb.py class TaskDB (line 17) | class TaskDB(BaseTaskDB): method __init__ (line 20) | def __init__(self, hosts, index='pyspider'): method _parse (line 35) | def _parse(self, data): method _stringify (line 46) | def _stringify(self, data): method projects (line 53) | def projects(self): method load_tasks (line 60) | def load_tasks(self, status, project=None, fields=None): method get_task (line 75) | def get_task(self, project, taskid, fields=None): method status_count (line 82) | def status_count(self, project): method insert (line 94) | def insert(self, project, taskid, obj={}): method update (line 103) | def update(self, project, taskid, obj={}, **kwargs): method drop (line 111) | def drop(self, project): method refresh (line 119) | def refresh(self): FILE: pyspider/database/local/projectdb.py class ProjectDB (line 17) | class ProjectDB(BaseProjectDB): method __init__ (line 20) | def __init__(self, files): method load_scripts (line 25) | def load_scripts(self): method _build_project (line 45) | def _build_project(self, filename): method get_all (line 75) | def get_all(self, fields=None): method get (line 79) | def get(self, name, fields=None): method check_update (line 91) | def check_update(self, timestamp, fields=None): FILE: pyspider/database/mongodb/mongodbbase.py class SplitTableMixin (line 11) | class SplitTableMixin(object): method _collection_name (line 14) | def _collection_name(self, project): method projects (line 21) | def projects(self): method projects (line 27) | def projects(self, value): method _list_project (line 30) | def _list_project(self): method drop (line 43) | def drop(self, project): FILE: pyspider/database/mongodb/projectdb.py class ProjectDB (line 14) | class ProjectDB(BaseProjectDB): method __init__ (line 17) | def __init__(self, url, database='projectdb'): method _default_fields (line 25) | def _default_fields(self, each): method insert (line 37) | def insert(self, name, obj={}): method update (line 43) | def update(self, name, obj={}, **kwargs): method get_all (line 49) | def get_all(self, fields=None): method get (line 55) | def get(self, name, fields=None): method check_update (line 61) | def check_update(self, timestamp, fields=None): method drop (line 67) | def drop(self, name): FILE: pyspider/database/mongodb/resultdb.py class ResultDB (line 17) | class ResultDB(SplitTableMixin, BaseResultDB): method __init__ (line 20) | def __init__(self, url, database='resultdb'): method _create_project (line 34) | def _create_project(self, project): method _parse (line 39) | def _parse(self, data): method _stringify (line 45) | def _stringify(self, data): method save (line 50) | def save(self, project, taskid, url, result): method select (line 64) | def select(self, project, fields=None, offset=0, limit=0): method count (line 75) | def count(self, project): method get (line 83) | def get(self, project, taskid, fields=None): FILE: pyspider/database/mongodb/taskdb.py class TaskDB (line 17) | class TaskDB(SplitTableMixin, BaseTaskDB): method __init__ (line 20) | def __init__(self, url, database='taskdb'): method _create_project (line 34) | def _create_project(self, project): method _parse (line 40) | def _parse(self, data): method _stringify (line 53) | def _stringify(self, data): method load_tasks (line 59) | def load_tasks(self, status, project=None, fields=None): method get_task (line 73) | def get_task(self, project, taskid, fields=None): method status_count (line 84) | def status_count(self, project): method insert (line 123) | def insert(self, project, taskid, obj={}): method update (line 132) | def update(self, project, taskid, obj={}, **kwargs): FILE: pyspider/database/mysql/mysqlbase.py class MySQLMixin (line 12) | class MySQLMixin(object): method dbcur (line 16) | def dbcur(self): class SplitTableMixin (line 29) | class SplitTableMixin(object): method _tablename (line 32) | def _tablename(self, project): method projects (line 39) | def projects(self): method projects (line 46) | def projects(self, value): method _list_project (line 49) | def _list_project(self): method drop (line 61) | def drop(self, project): FILE: pyspider/database/mysql/projectdb.py class ProjectDB (line 16) | class ProjectDB(MySQLMixin, BaseProjectDB, BaseDB): method __init__ (line 19) | def __init__(self, host='localhost', port=3306, database='projectdb', method insert (line 39) | def insert(self, name, obj={}): method update (line 45) | def update(self, name, obj={}, **kwargs): method get_all (line 52) | def get_all(self, fields=None): method get (line 55) | def get(self, name, fields=None): method drop (line 61) | def drop(self, name): method check_update (line 65) | def check_update(self, timestamp, fields=None): FILE: pyspider/database/mysql/resultdb.py class ResultDB (line 20) | class ResultDB(MySQLMixin, SplitTableMixin, BaseResultDB, BaseDB): method __init__ (line 23) | def __init__(self, host='localhost', port=3306, database='resultdb', method _create_project (line 33) | def _create_project(self, project): method _parse (line 45) | def _parse(self, data): method _stringify (line 53) | def _stringify(self, data): method save (line 58) | def save(self, project, taskid, url, result): method select (line 71) | def select(self, project, fields=None, offset=0, limit=None): method count (line 82) | def count(self, project): method get (line 91) | def get(self, project, taskid, fields=None): FILE: pyspider/database/mysql/taskdb.py class TaskDB (line 21) | class TaskDB(MySQLMixin, SplitTableMixin, BaseTaskDB, BaseDB): method __init__ (line 24) | def __init__(self, host='localhost', port=3306, database='taskdb', method _create_project (line 34) | def _create_project(self, project): method _parse (line 53) | def _parse(self, data): method _stringify (line 65) | def _stringify(self, data): method load_tasks (line 71) | def load_tasks(self, status, project=None, fields=None): method get_task (line 88) | def get_task(self, project, taskid, fields=None): method status_count (line 99) | def status_count(self, project): method insert (line 111) | def insert(self, project, taskid, obj={}): method update (line 124) | def update(self, project, taskid, obj={}, **kwargs): FILE: pyspider/database/redis/taskdb.py class TaskDB (line 19) | class TaskDB(BaseTaskDB): method __init__ (line 23) | def __init__(self, host='localhost', port=6379, db=0): method _gen_key (line 32) | def _gen_key(self, project, taskid): method _gen_status_key (line 35) | def _gen_status_key(self, project, status): method _parse (line 38) | def _parse(self, data): method _stringify (line 61) | def _stringify(self, data): method projects (line 68) | def projects(self): method load_tasks (line 75) | def load_tasks(self, status, project=None, fields=None): method get_task (line 105) | def get_task(self, project, taskid, fields=None): method status_count (line 118) | def status_count(self, project): method insert (line 133) | def insert(self, project, taskid, obj={}): method update (line 149) | def update(self, project, taskid, obj={}, **kwargs): method drop (line 164) | def drop(self, project): FILE: pyspider/database/sqlalchemy/projectdb.py class ProjectDB (line 19) | class ProjectDB(BaseProjectDB): method __init__ (line 22) | def __init__(self, url): method _parse (line 52) | def _parse(data): method _stringify (line 56) | def _stringify(data): method insert (line 59) | def insert(self, name, obj={}): method update (line 66) | def update(self, name, obj={}, **kwargs): method get_all (line 74) | def get_all(self, fields=None): method get (line 80) | def get(self, name, fields=None): method drop (line 88) | def drop(self, name): method check_update (line 92) | def check_update(self, timestamp, fields=None): FILE: pyspider/database/sqlalchemy/resultdb.py class ResultDB (line 22) | class ResultDB(SplitTableMixin, BaseResultDB): method __init__ (line 25) | def __init__(self, url): method _create_project (line 52) | def _create_project(self, project): method _parse (line 60) | def _parse(data): method _stringify (line 72) | def _stringify(data): method save (line 80) | def save(self, project, taskid, url, result): method select (line 100) | def select(self, project, fields=None, offset=0, limit=None): method count (line 115) | def count(self, project): method get (line 125) | def get(self, project, taskid, fields=None): FILE: pyspider/database/sqlalchemy/sqlalchemybase.py function result2dict (line 11) | def result2dict(columns, task): class SplitTableMixin (line 15) | class SplitTableMixin(object): method _tablename (line 18) | def _tablename(self, project): method projects (line 25) | def projects(self): method projects (line 32) | def projects(self, value): method _list_project (line 35) | def _list_project(self): method drop (line 48) | def drop(self, project): FILE: pyspider/database/sqlalchemy/taskdb.py class TaskDB (line 22) | class TaskDB(SplitTableMixin, BaseTaskDB): method __init__ (line 25) | def __init__(self, url): method _create_project (line 57) | def _create_project(self, project): method _parse (line 67) | def _parse(data): method _stringify (line 80) | def _stringify(data): method load_tasks (line 89) | def load_tasks(self, status, project=None, fields=None): method get_task (line 106) | def get_task(self, project, taskid, fields=None): method status_count (line 120) | def status_count(self, project): method insert (line 135) | def insert(self, project, taskid, obj={}): method update (line 149) | def update(self, project, taskid, obj={}, **kwargs): FILE: pyspider/database/sqlite/projectdb.py class ProjectDB (line 15) | class ProjectDB(SQLiteMixin, BaseProjectDB, BaseDB): method __init__ (line 19) | def __init__(self, path): method insert (line 30) | def insert(self, name, obj={}): method update (line 36) | def update(self, name, obj={}, **kwargs): method get_all (line 43) | def get_all(self, fields=None): method get (line 46) | def get(self, name, fields=None): method check_update (line 52) | def check_update(self, timestamp, fields=None): method drop (line 56) | def drop(self, name): FILE: pyspider/database/sqlite/resultdb.py class ResultDB (line 17) | class ResultDB(SQLiteMixin, SplitTableMixin, BaseResultDB, BaseDB): method __init__ (line 21) | def __init__(self, path): method _create_project (line 27) | def _create_project(self, project): method _parse (line 37) | def _parse(self, data): method _stringify (line 42) | def _stringify(self, data): method save (line 47) | def save(self, project, taskid, url, result): method select (line 60) | def select(self, project, fields=None, offset=0, limit=None): method count (line 71) | def count(self, project): method get (line 80) | def get(self, project, taskid, fields=None): FILE: pyspider/database/sqlite/sqlitebase.py class SQLiteMixin (line 14) | class SQLiteMixin(object): method dbcur (line 17) | def dbcur(self): class SplitTableMixin (line 25) | class SplitTableMixin(object): method _tablename (line 28) | def _tablename(self, project): method projects (line 35) | def projects(self): method projects (line 42) | def projects(self, value): method _list_project (line 45) | def _list_project(self): method drop (line 58) | def drop(self, project): FILE: pyspider/database/sqlite/taskdb.py class TaskDB (line 17) | class TaskDB(SQLiteMixin, SplitTableMixin, BaseTaskDB, BaseDB): method __init__ (line 21) | def __init__(self, path): method _create_project (line 27) | def _create_project(self, project): method _parse (line 42) | def _parse(self, data): method _stringify (line 51) | def _stringify(self, data): method load_tasks (line 57) | def load_tasks(self, status, project=None, fields=None): method get_task (line 72) | def get_task(self, project, taskid, fields=None): method status_count (line 85) | def status_count(self, project): method insert (line 100) | def insert(self, project, taskid, obj={}): method update (line 111) | def update(self, project, taskid, obj={}, **kwargs): FILE: pyspider/fetcher/cookie_utils.py class MockResponse (line 11) | class MockResponse(object): method __init__ (line 13) | def __init__(self, headers): method info (line 16) | def info(self): method getheaders (line 19) | def getheaders(self, name): method get_all (line 23) | def get_all(self, name, default=None): function extract_cookies_to_jar (line 30) | def extract_cookies_to_jar(jar, request, response): FILE: pyspider/fetcher/phantomjs_fetcher.js function make_result (line 135) | function make_result(page) { function _make_result (line 179) | function _make_result(page) { FILE: pyspider/fetcher/puppeteer_fetcher.js function fetch (line 35) | async function fetch(options) { function _fetch (line 51) | async function _fetch(page, options) { function make_result (line 146) | async function make_result(page, options, error) { FILE: pyspider/fetcher/tornado_fetcher.py class MyCurlAsyncHTTPClient (line 39) | class MyCurlAsyncHTTPClient(CurlAsyncHTTPClient): method free_size (line 41) | def free_size(self): method size (line 44) | def size(self): class MySimpleAsyncHTTPClient (line 48) | class MySimpleAsyncHTTPClient(SimpleAsyncHTTPClient): method free_size (line 50) | def free_size(self): method size (line 53) | def size(self): class Fetcher (line 66) | class Fetcher(object): method __init__ (line 81) | def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async_... method send_result (line 108) | def send_result(self, type, task, result): method fetch (line 116) | def fetch(self, task, callback=None): method async_fetch (line 123) | def async_fetch(self, task, callback=None): method sync_fetch (line 155) | def sync_fetch(self, task): method data_fetch (line 178) | def data_fetch(self, url, task): method handle_error (line 202) | def handle_error(self, type, url, task, start_time, error): method pack_tornado_request_parameters (line 220) | def pack_tornado_request_parameters(self, url, task): method can_fetch (line 290) | def can_fetch(self, user_agent, url): method clear_robot_txt_cache (line 320) | def clear_robot_txt_cache(self): method http_fetch (line 327) | def http_fetch(self, url, task): method phantomjs_fetch (line 431) | def phantomjs_fetch(self, url, task): method splash_fetch (line 532) | def splash_fetch(self, url, task): method puppeteer_fetch (line 640) | def puppeteer_fetch(self, url, task): method run (line 743) | def run(self): method quit (line 780) | def quit(self): method size (line 789) | def size(self): method xmlrpc_run (line 792) | def xmlrpc_run(self, port=24444, bind='127.0.0.1', logRequests=False): method on_fetch (line 827) | def on_fetch(self, type, task): method on_result (line 831) | def on_result(self, type, task, result): FILE: pyspider/libs/ListIO.py class ListO (line 9) | class ListO(object): method __init__ (line 13) | def __init__(self, buffer=None): method isatty (line 18) | def isatty(self): method close (line 21) | def close(self): method flush (line 24) | def flush(self): method seek (line 27) | def seek(self, n, mode=0): method readline (line 30) | def readline(self): method reset (line 33) | def reset(self): method write (line 36) | def write(self, x): method writelines (line 39) | def writelines(self, x): FILE: pyspider/libs/base_handler.py function catch_status_code_error (line 26) | def catch_status_code_error(func): function not_send_status (line 35) | def not_send_status(func): function config (line 49) | def config(_config=None, **kwargs): class NOTSET (line 64) | class NOTSET(object): function every (line 68) | def every(minutes=NOTSET, seconds=NOTSET): class BaseHandlerMeta (line 100) | class BaseHandlerMeta(type): method __new__ (line 102) | def __new__(cls, name, bases, attrs): class BaseHandler (line 123) | class BaseHandler(object): method _reset (line 136) | def _reset(self): method _run_func (line 145) | def _run_func(self, function, *arguments): method _run_task (line 160) | def _run_task(self, task, response): method run_task (line 178) | def run_task(self, module, task, response): method task_join_crawl_config (line 228) | def task_join_crawl_config(task, crawl_config): method _crawl (line 255) | def _crawl(self, url, **kwargs): method get_taskid (line 342) | def get_taskid(self, task): method crawl (line 347) | def crawl(self, url, **kwargs): method is_debugger (line 400) | def is_debugger(self): method send_message (line 404) | def send_message(self, project, msg, url='data:,on_message'): method on_message (line 408) | def on_message(self, project, msg): method on_result (line 412) | def on_result(self, result): method on_finished (line 422) | def on_finished(self, response, task): method _on_message (line 430) | def _on_message(self, response): method _on_cronjob (line 435) | def _on_cronjob(self, response, task): method _on_get_info (line 451) | def _on_get_info(self, response, task): FILE: pyspider/libs/bench.py function bench_test_taskdb (line 22) | def bench_test_taskdb(taskdb): function bench_test_message_queue (line 129) | def bench_test_message_queue(queue): class BenchMixin (line 190) | class BenchMixin(object): method _bench_init (line 192) | def _bench_init(self): method _bench_report (line 198) | def _bench_report(self, name, prefix=0, rjust=0): class BenchScheduler (line 213) | class BenchScheduler(Scheduler, BenchMixin): method __init__ (line 214) | def __init__(self, *args, **kwargs): method on_task_status (line 218) | def on_task_status(self, task): class BenchFetcher (line 223) | class BenchFetcher(Fetcher, BenchMixin): method __init__ (line 224) | def __init__(self, *args, **kwargs): method on_result (line 228) | def on_result(self, type, task, result): class BenchProcessor (line 233) | class BenchProcessor(Processor, BenchMixin): method __init__ (line 234) | def __init__(self, *args, **kwargs): method on_task (line 238) | def on_task(self, task, response): class BenchResultWorker (line 243) | class BenchResultWorker(ResultWorker, BenchMixin): method __init__ (line 244) | def __init__(self, *args, **kwargs): method on_result (line 248) | def on_result(self, task, result): class Handler (line 256) | class Handler(BaseHandler): method on_start (line 257) | def on_start(self, response): method index_page (line 262) | def index_page(self, response): FILE: pyspider/libs/counter.py class BaseCounter (line 23) | class BaseCounter(object): method __init__ (line 25) | def __init__(self): method event (line 28) | def event(self, value=1): method value (line 32) | def value(self, value): method avg (line 37) | def avg(self): method sum (line 42) | def sum(self): method empty (line 46) | def empty(self): class TotalCounter (line 51) | class TotalCounter(BaseCounter): method __init__ (line 54) | def __init__(self): method event (line 58) | def event(self, value=1): method value (line 61) | def value(self, value): method avg (line 65) | def avg(self): method sum (line 69) | def sum(self): method empty (line 72) | def empty(self): class AverageWindowCounter (line 76) | class AverageWindowCounter(BaseCounter): method __init__ (line 81) | def __init__(self, window_size=300): method event (line 86) | def event(self, value=1): method avg (line 92) | def avg(self): method sum (line 96) | def sum(self): method empty (line 99) | def empty(self): class TimebaseAverageEventCounter (line 104) | class TimebaseAverageEventCounter(BaseCounter): method __init__ (line 111) | def __init__(self, window_size=30, window_interval=10): method event (line 125) | def event(self, value=1): method value (line 147) | def value(self, value): method _trim_window (line 150) | def _trim_window(self): method avg (line 170) | def avg(self): method sum (line 177) | def sum(self): method empty (line 181) | def empty(self): method on_append (line 186) | def on_append(self, value, time): class TimebaseAverageWindowCounter (line 190) | class TimebaseAverageWindowCounter(BaseCounter): method __init__ (line 197) | def __init__(self, window_size=30, window_interval=10): method event (line 209) | def event(self, value=1): method value (line 227) | def value(self, value): method _trim_window (line 230) | def _trim_window(self): method avg (line 248) | def avg(self): method sum (line 255) | def sum(self): method empty (line 259) | def empty(self): method on_append (line 264) | def on_append(self, value, time): class CounterValue (line 268) | class CounterValue(DictMixin): method __init__ (line 273) | def __init__(self, manager, keys): method __getitem__ (line 277) | def __getitem__(self, key): method __len__ (line 299) | def __len__(self): method __iter__ (line 302) | def __iter__(self): method __contains__ (line 305) | def __contains__(self, key): method keys (line 308) | def keys(self): method to_dict (line 316) | def to_dict(self, get_value=None): class CounterManager (line 329) | class CounterManager(DictMixin): method __init__ (line 340) | def __init__(self, cls=TimebaseAverageWindowCounter): method event (line 345) | def event(self, key, value=1): method value (line 355) | def value(self, key, value=1): method trim (line 366) | def trim(self): method __getitem__ (line 372) | def __getitem__(self, key): method __delitem__ (line 389) | def __delitem__(self, key): method __iter__ (line 398) | def __iter__(self): method __len__ (line 401) | def __len__(self): method keys (line 404) | def keys(self): method to_dict (line 410) | def to_dict(self, get_value=None): method dump (line 423) | def dump(self, filename): method load (line 433) | def load(self, filename): FILE: pyspider/libs/dataurl.py function encode (line 14) | def encode(data, mime_type='', charset='utf-8', base64=True): function decode (line 41) | def decode(data_url): FILE: pyspider/libs/log.py class LogFormatter (line 18) | class LogFormatter(_LogFormatter, object): method __init__ (line 20) | def __init__(self, fmt=None, datefmt=None, color=True, *args, **kwargs): class SaveLogHandler (line 26) | class SaveLogHandler(logging.Handler): method __init__ (line 29) | def __init__(self, saveto=None, *args, **kwargs): method emit (line 33) | def emit(self, record): function enable_pretty_logging (line 40) | def enable_pretty_logging(logger=logging.getLogger()): FILE: pyspider/libs/multiprocessing_queue.py class SharedCounter (line 10) | class SharedCounter(object): method __init__ (line 22) | def __init__(self, n=0): method increment (line 25) | def increment(self, n=1): method value (line 31) | def value(self): class MultiProcessingQueue (line 36) | class MultiProcessingQueue(BaseQueue): method __init__ (line 47) | def __init__(self, *args, **kwargs): method put (line 51) | def put(self, *args, **kwargs): method get (line 55) | def get(self, *args, **kwargs): method qsize (line 60) | def qsize(self): function Queue (line 67) | def Queue(maxsize=0): function Queue (line 70) | def Queue(maxsize=0): FILE: pyspider/libs/pprint.py function pprint (line 54) | def pprint(object, stream=None, indent=1, width=80, depth=None): function pformat (line 61) | def pformat(object, indent=1, width=80, depth=None): function saferepr (line 66) | def saferepr(object): function isreadable (line 71) | def isreadable(object): function isrecursive (line 76) | def isrecursive(object): function _sorted (line 81) | def _sorted(iterable): class PrettyPrinter (line 85) | class PrettyPrinter: method __init__ (line 87) | def __init__(self, indent=1, width=80, depth=None, stream=None): method pprint (line 118) | def pprint(self, object): method pformat (line 122) | def pformat(self, object): method isrecursive (line 127) | def isrecursive(self, object): method isreadable (line 130) | def isreadable(self, object): method _format (line 134) | def _format(self, object, stream, indent, allowance, context, level): method _repr (line 234) | def _repr(self, object, context, level): method format (line 243) | def format(self, object, context, maxlevels, level): function _safe_repr (line 253) | def _safe_repr(object, context, maxlevels, level): function _recursion (line 359) | def _recursion(object): function _perfcheck (line 364) | def _perfcheck(object=None): FILE: pyspider/libs/response.py class Response (line 22) | class Response(object): method __init__ (line 24) | def __init__(self, status_code=None, url=None, orig_url=None, headers=... method __repr__ (line 40) | def __repr__(self): method __bool__ (line 43) | def __bool__(self): method __nonzero__ (line 47) | def __nonzero__(self): method ok (line 52) | def ok(self): method encoding (line 61) | def encoding(self): method encoding (line 89) | def encoding(self, value): method text (line 98) | def text(self): method json (line 129) | def json(self): method doc (line 140) | def doc(self): method etree (line 150) | def etree(self): method raise_for_status (line 165) | def raise_for_status(self, allow_redirects=True): method isok (line 186) | def isok(self): function rebuild_response (line 194) | def rebuild_response(r): function get_encoding (line 211) | def get_encoding(headers, content): FILE: pyspider/libs/result_dump.py function result_formater (line 16) | def result_formater(results): function dump_as_json (line 46) | def dump_as_json(results, valid=False): function dump_as_txt (line 64) | def dump_as_txt(results): function dump_as_csv (line 72) | def dump_as_csv(results): FILE: pyspider/libs/sample_handler.py class Handler (line 9) | class Handler(BaseHandler): method on_start (line 14) | def on_start(self): method index_page (line 18) | def index_page(self, response): method detail_page (line 23) | def detail_page(self, response): FILE: pyspider/libs/url.py function get_content_type (line 16) | def get_content_type(filename): function _encode_multipart_formdata (line 24) | def _encode_multipart_formdata(fields, files): function _build_url (line 29) | def _build_url(url, _params): function quote_chinese (line 62) | def quote_chinese(url, encodeing="utf-8"): function curl_to_arguments (line 73) | def curl_to_arguments(curl): FILE: pyspider/libs/utils.py class ReadOnlyDict (line 23) | class ReadOnlyDict(dict): method __setitem__ (line 26) | def __setitem__(self, key, value): function getitem (line 30) | def getitem(obj, key=0, default=None): function hide_me (line 38) | def hide_me(tb, g=globals()): function run_in_thread (line 54) | def run_in_thread(func, *args, **kwargs): function run_in_subprocess (line 63) | def run_in_subprocess(func, *args, **kwargs): function format_date (line 72) | def format_date(date, gmt_offset=0, relative=True, shorter=False, full_f... function fix_full_format (line 133) | def fix_full_format(days, seconds, relative, shorter, local_date, local_... class TimeoutError (line 160) | class TimeoutError(Exception): class timeout (line 168) | class timeout: method __init__ (line 176) | def __init__(self, seconds=1, error_message='Timeout'): method handle_timeout (line 180) | def handle_timeout(self, signum, frame): method __enter__ (line 183) | def __enter__(self): method __exit__ (line 191) | def __exit__(self, type, value, traceback): method __init__ (line 203) | def __init__(self, seconds=1, error_message='Timeout'): method __enter__ (line 206) | def __enter__(self): method __exit__ (line 209) | def __exit__(self, type, value, traceback): class timeout (line 198) | class timeout: method __init__ (line 176) | def __init__(self, seconds=1, error_message='Timeout'): method handle_timeout (line 180) | def handle_timeout(self, signum, frame): method __enter__ (line 183) | def __enter__(self): method __exit__ (line 191) | def __exit__(self, type, value, traceback): method __init__ (line 203) | def __init__(self, seconds=1, error_message='Timeout'): method __enter__ (line 206) | def __enter__(self): method __exit__ (line 209) | def __exit__(self, type, value, traceback): function utf8 (line 213) | def utf8(string): function text (line 227) | def text(string, encoding='utf8'): function pretty_unicode (line 241) | def pretty_unicode(string): function unicode_string (line 253) | def unicode_string(string): function unicode_dict (line 267) | def unicode_dict(_dict): function unicode_list (line 277) | def unicode_list(_list): function unicode_obj (line 284) | def unicode_obj(obj): function decode_unicode_string (line 307) | def decode_unicode_string(string): function decode_unicode_obj (line 316) | def decode_unicode_obj(obj): class Get (line 333) | class Get(object): method __init__ (line 338) | def __init__(self, getter): method __get__ (line 341) | def __get__(self, instance, owner): class ObjectDict (line 345) | class ObjectDict(dict): method __getattr__ (line 352) | def __getattr__(self, name): function load_object (line 359) | def load_object(name): function get_python_console (line 373) | def get_python_console(namespace=None): function python_console (line 418) | def python_console(namespace=None): function check_port_open (line 434) | def check_port_open(port, addr='127.0.0.1'): FILE: pyspider/libs/wsgi_xmlrpc.py class WSGIXMLRPCApplication (line 24) | class WSGIXMLRPCApplication(object): method __init__ (line 27) | def __init__(self, instance=None, methods=None): method register_instance (line 42) | def register_instance(self, instance): method register_function (line 45) | def register_function(self, function, name=None): method handler (line 48) | def handler(self, environ, start_response): method handle_POST (line 57) | def handle_POST(self, environ, start_response): method __call__ (line 94) | def __call__(self, environ, start_response): FILE: pyspider/message_queue/__init__.py function connect_message_queue (line 16) | def connect_message_queue(name, url=None, maxsize=0, lazy_limit=True): FILE: pyspider/message_queue/kombu_queue.py class KombuQueue (line 20) | class KombuQueue(object): method __init__ (line 31) | def __init__(self, name, url="amqp://", maxsize=0, lazy_limit=True): method qsize (line 51) | def qsize(self): method empty (line 57) | def empty(self): method full (line 63) | def full(self): method put (line 69) | def put(self, obj, block=True, timeout=None): method put_nowait (line 87) | def put_nowait(self, obj): method get (line 96) | def get(self, block=True, timeout=None): method get_nowait (line 103) | def get_nowait(self): method delete (line 110) | def delete(self): method __del__ (line 113) | def __del__(self): FILE: pyspider/message_queue/rabbitmq.py function catch_error (line 24) | def catch_error(func): class PikaQueue (line 52) | class PikaQueue(object): method __init__ (line 61) | def __init__(self, name, amqp_url='amqp://guest:guest@localhost:5672/%... method reconnect (line 91) | def reconnect(self): method qsize (line 106) | def qsize(self): method empty (line 111) | def empty(self): method full (line 117) | def full(self): method put (line 124) | def put(self, obj, block=True, timeout=None): method put_nowait (line 143) | def put_nowait(self, obj): method get (line 155) | def get(self, block=True, timeout=None, ack=False): method get_nowait (line 174) | def get_nowait(self, ack=False): method delete (line 184) | def delete(self): class AmqpQueue (line 189) | class AmqpQueue(PikaQueue): method __init__ (line 194) | def __init__(self, name, amqp_url='amqp://guest:guest@localhost:5672/%... method reconnect (line 224) | def reconnect(self): method qsize (line 241) | def qsize(self): method put_nowait (line 248) | def put_nowait(self, obj): method get_nowait (line 261) | def get_nowait(self, ack=False): FILE: pyspider/message_queue/redis_queue.py class RedisQueue (line 14) | class RedisQueue(object): method __init__ (line 23) | def __init__(self, name, host='localhost', port=6379, db=0, method qsize (line 43) | def qsize(self): method empty (line 47) | def empty(self): method full (line 53) | def full(self): method put_nowait (line 59) | def put_nowait(self, obj): method put (line 67) | def put(self, obj, block=True, timeout=None): method get_nowait (line 85) | def get_nowait(self): method get (line 91) | def get(self, block=True, timeout=None): FILE: pyspider/processor/processor.py class ProcessorResult (line 23) | class ProcessorResult(object): method __init__ (line 26) | def __init__(self, result=None, follows=(), messages=(), method rethrow (line 38) | def rethrow(self): method logstr (line 44) | def logstr(self): class Processor (line 62) | class Processor(object): method __init__ (line 69) | def __init__(self, projectdb, inqueue, status_queue, newtask_queue, re... method enable_projects_import (line 91) | def enable_projects_import(self): method __del__ (line 99) | def __del__(self): method on_task (line 102) | def on_task(self, task, response): method quit (line 205) | def quit(self): method run (line 209) | def run(self): FILE: pyspider/processor/project_module.py class ProjectManager (line 23) | class ProjectManager(object): method build_module (line 32) | def build_module(project, env=None): method __init__ (line 89) | def __init__(self, projectdb, env): method _need_update (line 96) | def _need_update(self, project_name, updatetime=None, md5sum=None): method _check_projects (line 108) | def _check_projects(self): method _update_project (line 118) | def _update_project(self, project_name): method _load_project (line 125) | def _load_project(self, project): method get (line 148) | def get(self, project_name, updatetime=None, md5sum=None): class ProjectLoader (line 157) | class ProjectLoader(object): method __init__ (line 160) | def __init__(self, project, mod=None): method load_module (line 166) | def load_module(self, fullname): method is_package (line 182) | def is_package(self, fullname): method get_code (line 185) | def get_code(self, fullname): method get_source (line 188) | def get_source(self, fullname): method create_module (line 282) | def create_module(self, spec): method exec_module (line 285) | def exec_module(self, module): method module_repr (line 288) | def module_repr(self, module): class ProjectFinder (line 196) | class ProjectFinder(object): method __init__ (line 199) | def __init__(self, projectdb): method projectdb (line 203) | def projectdb(self): method find_module (line 206) | def find_module(self, fullname, path=None): method load_module (line 218) | def load_module(self, fullname): method is_package (line 226) | def is_package(self, fullname): method __init__ (line 234) | def __init__(self, projectdb): method projectdb (line 238) | def projectdb(self): method find_spec (line 241) | def find_spec(self, fullname, path, target=None): method find_module (line 246) | def find_module(self, fullname, path): class ProjectFinder (line 231) | class ProjectFinder(importlib.abc.MetaPathFinder): method __init__ (line 199) | def __init__(self, projectdb): method projectdb (line 203) | def projectdb(self): method find_module (line 206) | def find_module(self, fullname, path=None): method load_module (line 218) | def load_module(self, fullname): method is_package (line 226) | def is_package(self, fullname): method __init__ (line 234) | def __init__(self, projectdb): method projectdb (line 238) | def projectdb(self): method find_spec (line 241) | def find_spec(self, fullname, path, target=None): method find_module (line 246) | def find_module(self, fullname, path): class ProjectsLoader (line 258) | class ProjectsLoader(importlib.abc.InspectLoader): method load_module (line 259) | def load_module(self, fullname): method module_repr (line 269) | def module_repr(self, module): method is_package (line 272) | def is_package(self, fullname): method get_source (line 275) | def get_source(self, path): method get_code (line 278) | def get_code(self, fullname): class ProjectLoader (line 281) | class ProjectLoader(ProjectLoader, importlib.abc.Loader): method __init__ (line 160) | def __init__(self, project, mod=None): method load_module (line 166) | def load_module(self, fullname): method is_package (line 182) | def is_package(self, fullname): method get_code (line 185) | def get_code(self, fullname): method get_source (line 188) | def get_source(self, fullname): method create_module (line 282) | def create_module(self, spec): method exec_module (line 285) | def exec_module(self, module): method module_repr (line 288) | def module_repr(self, module): FILE: pyspider/result/result_worker.py class ResultWorker (line 15) | class ResultWorker(object): method __init__ (line 22) | def __init__(self, resultdb, inqueue): method on_result (line 27) | def on_result(self, task, result): method quit (line 44) | def quit(self): method run (line 47) | def run(self): class OneResultWorker (line 69) | class OneResultWorker(ResultWorker): method on_result (line 71) | def on_result(self, task, result): FILE: pyspider/run.py function read_config (line 25) | def read_config(ctx, param, value): function connect_db (line 40) | def connect_db(ctx, param, value): function load_cls (line 46) | def load_cls(ctx, param, value): function connect_rpc (line 52) | def connect_rpc(ctx, param, value): function cli (line 91) | def cli(ctx, **kwargs): function scheduler (line 198) | def scheduler(ctx, xmlrpc, no_xmlrpc, xmlrpc_host, xmlrpc_port, function fetcher (line 245) | def fetcher(ctx, xmlrpc, no_xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, ... function processor (line 286) | def processor(ctx, processor_cls, process_time_limit, enable_stdout_capt... function result_worker (line 310) | def result_worker(ctx, result_cls, get_object=False): function webui (line 346) | def webui(ctx, host, port, cdn, scheduler_rpc, fetcher_rpc, max_rate, ma... function phantomjs (line 414) | def phantomjs(ctx, phantomjs_path, port, auto_restart, args): function puppeteer (line 462) | def puppeteer(ctx, port, auto_restart, args): function all (line 510) | def all(ctx, fetcher_num, processor_num, result_worker_num, run_in): function bench (line 601) | def bench(ctx, fetcher_num, processor_num, result_worker_num, run_in, to... function one (line 735) | def one(ctx, interactive, enable_phantomjs, enable_puppeteer, scripts): function send_message (line 813) | def send_message(ctx, scheduler_rpc, project, message): function main (line 838) | def main(): FILE: pyspider/scheduler/scheduler.py class Project (line 26) | class Project(object): method __init__ (line 30) | def __init__(self, scheduler, project_info): method paused (line 52) | def paused(self): method update (line 104) | def update(self, project_info): method on_get_info (line 129) | def on_get_info(self, info): method active (line 136) | def active(self): class Scheduler (line 140) | class Scheduler(object): method __init__ (line 170) | def __init__(self, taskdb, projectdb, newtask_queue, status_queue, method _update_projects (line 206) | def _update_projects(self): method _update_project (line 222) | def _update_project(self, project): method _load_tasks (line 263) | def _load_tasks(self, project): method _update_project_cnt (line 282) | def _update_project_cnt(self, project_name): method task_verify (line 297) | def task_verify(self, task): method insert_task (line 317) | def insert_task(self, task): method update_task (line 321) | def update_task(self, task): method put_task (line 325) | def put_task(self, task): method send_task (line 334) | def send_task(self, task, force=True): method _check_task_done (line 348) | def _check_task_done(self): method _check_request (line 374) | def _check_request(self): method _check_cronjob (line 419) | def _check_cronjob(self): method _check_select (line 463) | def _check_select(self): method _load_put_task (line 568) | def _load_put_task(self, project, taskid): method _print_counter_log (line 578) | def _print_counter_log(self): method _dump_cnt (line 616) | def _dump_cnt(self): method _try_dump_cnt (line 622) | def _try_dump_cnt(self): method _check_delete (line 630) | def _check_delete(self): method __len__ (line 650) | def __len__(self): method quit (line 653) | def quit(self): method run_once (line 661) | def run_once(self): method run (line 673) | def run(self): method trigger_on_start (line 694) | def trigger_on_start(self, project): method xmlrpc_run (line 705) | def xmlrpc_run(self, port=23333, bind='127.0.0.1', logRequests=False): method on_request (line 813) | def on_request(self, task): method on_new_request (line 825) | def on_new_request(self, task): method on_old_request (line 839) | def on_old_request(self, task, old_task): method on_task_status (line 889) | def on_task_status(self, task): method on_task_done (line 914) | def on_task_done(self, task): method on_task_failed (line 937) | def on_task_failed(self, task): method on_select_task (line 990) | def on_select_task(self, task): class OneScheduler (line 1014) | class OneScheduler(Scheduler): method _check_select (line 1022) | def _check_select(self): method __getattr__ (line 1106) | def __getattr__(self, name): method on_task_status (line 1112) | def on_task_status(self, task): method init_one (line 1136) | def init_one(self, ioloop, fetcher, processor, method do_task (line 1146) | def do_task(self, task): method send_task (line 1162) | def send_task(self, task, force=True): method run (line 1170) | def run(self): method quit (line 1176) | def quit(self): class ThreadBaseScheduler (line 1186) | class ThreadBaseScheduler(Scheduler): method __init__ (line 1187) | def __init__(self, threads=4, *args, **kwargs): method taskdb (line 1207) | def taskdb(self): method taskdb (line 1213) | def taskdb(self, taskdb): method projectdb (line 1217) | def projectdb(self): method projectdb (line 1223) | def projectdb(self, projectdb): method resultdb (line 1227) | def resultdb(self): method resultdb (line 1233) | def resultdb(self, resultdb): method _start_threads (line 1236) | def _start_threads(self): method _thread_worker (line 1245) | def _thread_worker(self, queue): method _run_in_thread (line 1253) | def _run_in_thread(self, method, *args, **kwargs): method _wait_thread (line 1277) | def _wait_thread(self): method _update_project (line 1283) | def _update_project(self, project): method on_task_status (line 1286) | def on_task_status(self, task): method on_request (line 1290) | def on_request(self, task): method _load_put_task (line 1294) | def _load_put_task(self, project, taskid): method run_once (line 1298) | def run_once(self): FILE: pyspider/scheduler/task_queue.py class AtomInt (line 28) | class AtomInt(object): method get_value (line 33) | def get_value(cls): class InQueueTask (line 41) | class InQueueTask(DictMixin): method __init__ (line 49) | def __init__(self, taskid, priority=0, exetime=0): method __cmp__ (line 55) | def __cmp__(self, other): method __lt__ (line 65) | def __lt__(self, other): class PriorityTaskQueue (line 69) | class PriorityTaskQueue(Queue.Queue): method _init (line 76) | def _init(self, maxsize): method _qsize (line 80) | def _qsize(self, len=len): method _put (line 83) | def _put(self, item, heappush=heapq.heappush): method _get (line 97) | def _get(self, heappop=heapq.heappop): method top (line 107) | def top(self): method _resort (line 114) | def _resort(self): method __contains__ (line 117) | def __contains__(self, taskid): method __getitem__ (line 120) | def __getitem__(self, taskid): method __setitem__ (line 123) | def __setitem__(self, taskid, item): method __delitem__ (line 127) | def __delitem__(self, taskid): class TaskQueue (line 131) | class TaskQueue(object): method __init__ (line 137) | def __init__(self, rate=0, burst=0): method rate (line 145) | def rate(self): method rate (line 149) | def rate(self, value): method burst (line 153) | def burst(self): method burst (line 157) | def burst(self, value): method check_update (line 160) | def check_update(self): method _check_time_queue (line 169) | def _check_time_queue(self): method _check_processing (line 178) | def _check_processing(self): method put (line 190) | def put(self, taskid, priority=0, exetime=0): method get (line 227) | def get(self): method done (line 244) | def done(self, taskid): method delete (line 254) | def delete(self, taskid): method size (line 269) | def size(self): method is_processing (line 272) | def is_processing(self, taskid): method __len__ (line 278) | def __len__(self): method __contains__ (line 281) | def __contains__(self, taskid): FILE: pyspider/scheduler/token_bucket.py class Bucket (line 15) | class Bucket(object): method __init__ (line 23) | def __init__(self, rate=1, burst=None): method get (line 33) | def get(self): method set (line 49) | def set(self, value): method desc (line 53) | def desc(self, value=1): FILE: pyspider/webui/app.py class QuitableFlask (line 24) | class QuitableFlask(Flask): method logger (line 28) | def logger(self): method run (line 31) | def run(self, host=None, port=None, debug=None, **options): method quit (line 80) | def quit(self): function cdn_url_handler (line 104) | def cdn_url_handler(error, endpoint, kwargs): FILE: pyspider/webui/bench_test.py function bench_test (line 19) | def bench_test(): FILE: pyspider/webui/debug.py function debug (line 39) | def debug(project): function enable_projects_import (line 65) | def enable_projects_import(): function run (line 70) | def run(project): function save (line 170) | def save(project): function get_script (line 209) | def get_script(project): function blank_html (line 219) | def blank_html(): FILE: pyspider/webui/index.py function index (line 24) | def index(): function get_queues (line 32) | def get_queues(): function project_update (line 49) | def project_update(): function counter (line 95) | def counter(): function runtask (line 116) | def runtask(): function robots (line 153) | def robots(): FILE: pyspider/webui/login.py class AnonymousUser (line 20) | class AnonymousUser(login.AnonymousUserMixin): method is_anonymous (line 22) | def is_anonymous(self): method is_active (line 25) | def is_active(self): method is_authenticated (line 28) | def is_authenticated(self): method get_id (line 31) | def get_id(self): class User (line 35) | class User(login.UserMixin): method __init__ (line 37) | def __init__(self, id, password): method is_authenticated (line 41) | def is_authenticated(self): method is_active (line 49) | def is_active(self): function load_user_from_request (line 57) | def load_user_from_request(request): function before_request (line 74) | def before_request(): FILE: pyspider/webui/result.py function result (line 17) | def result(): function dump_result (line 34) | def dump_result(project, _format): FILE: pyspider/webui/static/src/css_selector_helper.js function arrayEquals (line 8) | function arrayEquals(a, b) { function getOffset (line 21) | function getOffset(elem) { function merge_name (line 31) | function merge_name(features) { function merge_pattern (line 40) | function merge_pattern(path, end) { function path_info (line 75) | function path_info(doc, element) { class CSSSelectorHelperServer (line 179) | class CSSSelectorHelperServer extends EventEmitter { method constructor (line 180) | constructor(window) { method overlay (line 198) | overlay(elements) { method heightlight (line 221) | heightlight(elements) { method getElementByXpath (line 245) | getElementByXpath(path) { FILE: pyspider/webui/static/src/debug.js function merge_name (line 14) | function merge_name(p) { function merge_pattern (line 27) | function merge_pattern(path, end) { function selector_changed (line 62) | function selector_changed(path) { function render_selector_helper (line 67) | function render_selector_helper(path) { function adjustHelper (line 137) | function adjustHelper() { function escape (line 221) | function escape(text) { FILE: pyspider/webui/static/src/index.js function init_editable (line 17) | function init_editable(projects_app) { function init_sortable (line 82) | function init_sortable() { function update_counters (line 110) | function update_counters() { function update_queues (line 145) | function update_queues() { FILE: pyspider/webui/static/src/splitter.js function moveSplitter (line 113) | function moveSplitter(pos) { function resetPrev (line 151) | function resetPrev() { FILE: pyspider/webui/task.py function task (line 16) | def task(taskid): function task_in_json (line 36) | def task_in_json(taskid): function tasks (line 51) | def tasks(): function active_tasks (line 81) | def active_tasks(): FILE: pyspider/webui/webdav.py function check_user (line 21) | def check_user(environ): class ContentIO (line 39) | class ContentIO(BytesIO): method close (line 40) | def close(self): class ScriptResource (line 45) | class ScriptResource(DAVNonCollection): method __init__ (line 46) | def __init__(self, path, environ, app, project=None): method project (line 58) | def project(self): method readonly (line 80) | def readonly(self): method getContentLength (line 90) | def getContentLength(self): method getContentType (line 93) | def getContentType(self): method getLastModified (line 96) | def getLastModified(self): method getContent (line 99) | def getContent(self): method beginWrite (line 102) | def beginWrite(self, contentType=None): method endWrite (line 109) | def endWrite(self, withErrors): class RootCollection (line 133) | class RootCollection(DAVCollection): method __init__ (line 134) | def __init__(self, path, environ, app): method getMemberList (line 139) | def getMemberList(self): method getMemberNames (line 155) | def getMemberNames(self): class ScriptProvider (line 165) | class ScriptProvider(DAVProvider): method __init__ (line 166) | def __init__(self, app): method __repr__ (line 170) | def __repr__(self): method getResourceInst (line 173) | def getResourceInst(self, path, environ): class NeedAuthController (line 182) | class NeedAuthController(object): method __init__ (line 183) | def __init__(self, app): method getDomainRealm (line 186) | def getDomainRealm(self, inputRelativeURL, environ): method requireAuthentication (line 189) | def requireAuthentication(self, realmname, environ): method isRealmUser (line 192) | def isRealmUser(self, realmname, username, environ): method getRealmUserPassword (line 195) | def getRealmUserPassword(self, realmname, username, environ): method authDomainUser (line 198) | def authDomainUser(self, realmname, username, password, environ): FILE: tests/data_fetcher_processor_handler.py class Handler (line 10) | class Handler(BaseHandler): method not_send_status (line 13) | def not_send_status(self, response): method url_deduplicated (line 17) | def url_deduplicated(self, response): method catch_http_error (line 25) | def catch_http_error(self, response): method json (line 29) | def json(self, response): method html (line 32) | def html(self, response): method links (line 35) | def links(self, response): method cookies (line 38) | def cookies(self, response): method get_save (line 41) | def get_save(self, response): method get_process_save (line 44) | def get_process_save(self, response): method set_process_save (line 47) | def set_process_save(self, response): class IgnoreHandler (line 50) | class IgnoreHandler(BaseHandler): FILE: tests/data_handler.py class IgnoreHandler (line 12) | class IgnoreHandler(object): class TestHandler (line 15) | class TestHandler(BaseHandler): method hello (line 21) | def hello(self): method echo (line 24) | def echo(self, response): method saved (line 27) | def saved(self, response): method echo_task (line 30) | def echo_task(self, response, task): method catch_status_code (line 34) | def catch_status_code(self, response): method raise_exception (line 37) | def raise_exception(self): method add_task (line 44) | def add_task(self, response): method on_cronjob1 (line 49) | def on_cronjob1(self, response): method on_cronjob2 (line 53) | def on_cronjob2(self, response): method generator (line 56) | def generator(self, response): method sleep (line 60) | def sleep(self, response): FILE: tests/data_sample_handler.py class Handler (line 9) | class Handler(BaseHandler): method on_start (line 14) | def on_start(self): method index_page (line 18) | def index_page(self, response): method detail_page (line 23) | def detail_page(self, response): FILE: tests/data_test_webpage.py function test_page (line 11) | def test_page(): function test_ajax (line 30) | def test_ajax(): function test_ajax_click (line 49) | def test_ajax_click(): FILE: tests/test_base_handler.py class TestBaseHandler (line 13) | class TestBaseHandler(unittest.TestCase): method test_task_join_crawl_config (line 36) | def test_task_join_crawl_config(self): FILE: tests/test_bench.py class TestBench (line 19) | class TestBench(unittest.TestCase): method setUpClass (line 22) | def setUpClass(self): method tearDownClass (line 27) | def tearDownClass(self): method test_10_bench (line 30) | def test_10_bench(self): FILE: tests/test_counter.py class TestCounter (line 14) | class TestCounter(unittest.TestCase): method test_010_TimebaseAverageEventCounter (line 15) | def test_010_TimebaseAverageEventCounter(self): method test_020_TotalCounter (line 24) | def test_020_TotalCounter(self): method test_030_AverageWindowCounter (line 31) | def test_030_AverageWindowCounter(self): method test_020_delete (line 42) | def test_020_delete(self): FILE: tests/test_database.py class TaskDBCase (line 19) | class TaskDBCase(object): method setUpClass (line 69) | def setUpClass(self): method test_20_insert (line 81) | def test_20_insert(self): method test_25_get_task (line 85) | def test_25_get_task(self): method test_30_status_count (line 104) | def test_30_status_count(self): method test_40_update_and_status_count (line 110) | def test_40_update_and_status_count(self): method test_50_load_tasks (line 120) | def test_50_load_tasks(self): method test_60_relist_projects (line 137) | def test_60_relist_projects(self): method test_z10_drop (line 142) | def test_z10_drop(self): method test_z20_update_projects (line 149) | def test_z20_update_projects(self): class ProjectDBCase (line 158) | class ProjectDBCase(object): method setUpClass (line 168) | def setUpClass(self): method test_10_insert (line 171) | def test_10_insert(self): method test_20_get_all (line 177) | def test_20_get_all(self): method test_30_update (line 201) | def test_30_update(self): method test_40_check_update (line 206) | def test_40_check_update(self): method test_45_check_update_when_bootup (line 221) | def test_45_check_update_when_bootup(self): method test_50_get (line 227) | def test_50_get(self): method test_z10_drop (line 240) | def test_z10_drop(self): class ResultDBCase (line 248) | class ResultDBCase(object): method setUpClass (line 251) | def setUpClass(self): method test_10_save (line 254) | def test_10_save(self): method test_20_get (line 265) | def test_20_get(self): method test_30_select (line 283) | def test_30_select(self): method test_35_select_limit (line 297) | def test_35_select_limit(self): method test_40_count (line 304) | def test_40_count(self): method test_50_select_not_finished (line 307) | def test_50_select_not_finished(self): method test_60_relist_projects (line 312) | def test_60_relist_projects(self): method test_z10_drop (line 317) | def test_z10_drop(self): method test_z20_update_projects (line 324) | def test_z20_update_projects(self): class TestSqliteTaskDB (line 333) | class TestSqliteTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 336) | def setUpClass(self): method tearDownClass (line 341) | def tearDownClass(self): class TestSqliteProjectDB (line 345) | class TestSqliteProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 348) | def setUpClass(self): method tearDownClass (line 353) | def tearDownClass(self): class TestSqliteResultDB (line 357) | class TestSqliteResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 360) | def setUpClass(self): method tearDownClass (line 365) | def tearDownClass(self): class TestMysqlTaskDB (line 370) | class TestMysqlTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 373) | def setUpClass(self): method tearDownClass (line 378) | def tearDownClass(self): class TestMysqlProjectDB (line 383) | class TestMysqlProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 386) | def setUpClass(self): method tearDownClass (line 393) | def tearDownClass(self): class TestMysqlResultDB (line 398) | class TestMysqlResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 401) | def setUpClass(self): method tearDownClass (line 408) | def tearDownClass(self): class TestMongoDBTaskDB (line 413) | class TestMongoDBTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 416) | def setUpClass(self): method tearDownClass (line 423) | def tearDownClass(self): method test_create_project (line 426) | def test_create_project(self): class TestMongoDBProjectDB (line 433) | class TestMongoDBProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 436) | def setUpClass(self): method tearDownClass (line 443) | def tearDownClass(self): class TestMongoDBResultDB (line 448) | class TestMongoDBResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 451) | def setUpClass(self): method tearDownClass (line 458) | def tearDownClass(self): method test_create_project (line 461) | def test_create_project(self): class TestSQLAlchemyMySQLTaskDB (line 468) | class TestSQLAlchemyMySQLTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 471) | def setUpClass(self): method tearDownClass (line 478) | def tearDownClass(self): class TestSQLAlchemyMySQLProjectDB (line 483) | class TestSQLAlchemyMySQLProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 486) | def setUpClass(self): method tearDownClass (line 493) | def tearDownClass(self): class TestSQLAlchemyMySQLResultDB (line 498) | class TestSQLAlchemyMySQLResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 501) | def setUpClass(self): method tearDownClass (line 508) | def tearDownClass(self): class TestSQLAlchemyTaskDB (line 512) | class TestSQLAlchemyTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 515) | def setUpClass(self): method tearDownClass (line 522) | def tearDownClass(self): class TestSQLAlchemyProjectDB (line 526) | class TestSQLAlchemyProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 529) | def setUpClass(self): method tearDownClass (line 536) | def tearDownClass(self): class TestSQLAlchemyResultDB (line 540) | class TestSQLAlchemyResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 543) | def setUpClass(self): method tearDownClass (line 550) | def tearDownClass(self): class TestPGTaskDB (line 555) | class TestPGTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 558) | def setUpClass(self): method tearDownClass (line 566) | def tearDownClass(self): class TestPGProjectDB (line 572) | class TestPGProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 575) | def setUpClass(self): method tearDownClass (line 583) | def tearDownClass(self): class TestPGResultDB (line 589) | class TestPGResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 592) | def setUpClass(self): method tearDownClass (line 600) | def tearDownClass(self): class TestRedisTaskDB (line 606) | class TestRedisTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 609) | def setUpClass(self): method tearDownClass (line 615) | def tearDownClass(self): class TestESProjectDB (line 621) | class TestESProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 624) | def setUpClass(self): method tearDownClass (line 632) | def tearDownClass(self): class TestESResultDB (line 637) | class TestESResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 640) | def setUpClass(self): method tearDownClass (line 648) | def tearDownClass(self): method test_15_save (line 651) | def test_15_save(self): method test_30_select (line 654) | def test_30_select(self): method test_35_select_limit (line 670) | def test_35_select_limit(self): method test_z20_update_projects (line 673) | def test_z20_update_projects(self): class TestESTaskDB (line 679) | class TestESTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 682) | def setUpClass(self): method tearDownClass (line 690) | def tearDownClass(self): class TestCouchDBProjectDB (line 695) | class TestCouchDBProjectDB(ProjectDBCase, unittest.TestCase): method setUpClass (line 698) | def setUpClass(self): method tearDownClass (line 706) | def tearDownClass(self): class TestCouchDBResultDB (line 712) | class TestCouchDBResultDB(ResultDBCase, unittest.TestCase): method setUpClass (line 715) | def setUpClass(self): method tearDownClass (line 723) | def tearDownClass(self): method test_create_project (line 727) | def test_create_project(self): class TestCouchDBTaskDB (line 734) | class TestCouchDBTaskDB(TaskDBCase, unittest.TestCase): method setUpClass (line 737) | def setUpClass(self): method tearDownClass (line 746) | def tearDownClass(self): method test_create_project (line 752) | def test_create_project(self): FILE: tests/test_fetcher.py class TestFetcher (line 31) | class TestFetcher(unittest.TestCase): method setUpClass (line 55) | def setUpClass(self): method tearDownClass (line 83) | def tearDownClass(self): method test_10_http_get (line 103) | def test_10_http_get(self): method test_15_http_post (line 117) | def test_15_http_post(self): method test_20_dataurl_get (line 136) | def test_20_dataurl_get(self): method test_30_with_queue (line 145) | def test_30_with_queue(self): method test_40_with_rpc (line 155) | def test_40_with_rpc(self): method test_50_base64_data (line 164) | def test_50_base64_data(self): method test_55_base64_data (line 178) | def test_55_base64_data(self): method test_60_timeout (line 191) | def test_60_timeout(self): method test_65_418 (line 206) | def test_65_418(self): method test_69_no_phantomjs (line 216) | def test_69_no_phantomjs(self): method test_70_phantomjs_url (line 232) | def test_70_phantomjs_url(self): method test_75_phantomjs_robots (line 249) | def test_75_phantomjs_robots(self): method test_80_phantomjs_timeout (line 261) | def test_80_phantomjs_timeout(self): method test_90_phantomjs_js_script (line 276) | def test_90_phantomjs_js_script(self): method test_a100_phantomjs_sharp_url (line 287) | def test_a100_phantomjs_sharp_url(self): method test_a110_dns_error (line 300) | def test_a110_dns_error(self): method test_a120_http_get_with_proxy_fail (line 314) | def test_a120_http_get_with_proxy_fail(self): method test_a130_http_get_with_proxy_ok (line 324) | def test_a130_http_get_with_proxy_ok(self): method test_a140_redirect (line 340) | def test_a140_redirect(self): method test_a150_too_much_redirect (line 350) | def test_a150_too_much_redirect(self): method test_a160_cookie (line 359) | def test_a160_cookie(self): method test_a170_validate_cert (line 368) | def test_a170_validate_cert(self): method test_a180_max_redirects (line 377) | def test_a180_max_redirects(self): method test_a200_robots_txt (line 386) | def test_a200_robots_txt(self): method test_zzzz_issue375 (line 401) | def test_zzzz_issue375(self): class TestSplashFetcher (line 418) | class TestSplashFetcher(unittest.TestCase): method sample_task_http (line 420) | def sample_task_http(self): method setUpClass (line 444) | def setUpClass(self): method tearDownClass (line 464) | def tearDownClass(self): method test_69_no_splash (line 482) | def test_69_no_splash(self): method test_70_splash_url (line 496) | def test_70_splash_url(self): method test_75_splash_robots (line 512) | def test_75_splash_robots(self): method test_80_splash_timeout (line 522) | def test_80_splash_timeout(self): method test_90_splash_js_script (line 535) | def test_90_splash_js_script(self): method test_95_splash_js_script_2 (line 544) | def test_95_splash_js_script_2(self): method test_a100_splash_sharp_url (line 557) | def test_a100_splash_sharp_url(self): method test_a120_http_get_with_proxy_fail_1 (line 568) | def test_a120_http_get_with_proxy_fail_1(self): method test_a120_http_get_with_proxy_fail (line 578) | def test_a120_http_get_with_proxy_fail(self): method test_a130_http_get_with_proxy_ok_1 (line 589) | def test_a130_http_get_with_proxy_ok_1(self): method test_a130_http_get_with_proxy_ok (line 605) | def test_a130_http_get_with_proxy_ok(self): FILE: tests/test_fetcher_processor.py class TestFetcherProcessor (line 22) | class TestFetcherProcessor(Handler, unittest.TestCase): method setUpClass (line 25) | def setUpClass(self): method tearDownClass (line 46) | def tearDownClass(self): method crawl (line 53) | def crawl(self, url=None, track=None, **kwargs): method assertStatusOk (line 80) | def assertStatusOk(self, status): method status_ok (line 85) | def status_ok(self, status, type): method test_10_not_status (line 90) | def test_10_not_status(self): method test_20_url_deduplicated (line 97) | def test_20_url_deduplicated(self): method test_30_catch_status_code_error (line 108) | def test_30_catch_status_code_error(self): method test_40_method (line 141) | def test_40_method(self): method test_50_params (line 154) | def test_50_params(self): method test_60_data (line 164) | def test_60_data(self): method test_70_redirect (line 174) | def test_70_redirect(self): method test_80_redirect_too_many (line 181) | def test_80_redirect_too_many(self): method test_90_files (line 190) | def test_90_files(self): method test_a100_files_with_data (line 199) | def test_a100_files_with_data(self): method test_a110_headers (line 212) | def test_a110_headers(self): method test_a115_user_agent (line 223) | def test_a115_user_agent(self): method test_a120_cookies (line 231) | def test_a120_cookies(self): method test_a130_cookies_with_headers (line 242) | def test_a130_cookies_with_headers(self): method test_a140_response_cookie (line 258) | def test_a140_response_cookie(self): method test_a145_redirect_cookie (line 265) | def test_a145_redirect_cookie(self): method test_a150_timeout (line 272) | def test_a150_timeout(self): method test_a160_etag (line 280) | def test_a160_etag(self): method test_a170_last_modified (line 287) | def test_a170_last_modified(self): method test_a180_save (line 294) | def test_a180_save(self): method test_a190_taskid (line 302) | def test_a190_taskid(self): method test_a200_no_proxy (line 311) | def test_a200_no_proxy(self): method test_a210_proxy_failed (line 323) | def test_a210_proxy_failed(self): method test_a220_proxy_ok (line 337) | def test_a220_proxy_ok(self): method test_a230_proxy_parameter_fail (line 351) | def test_a230_proxy_parameter_fail(self): method test_a240_proxy_parameter_ok (line 362) | def test_a240_proxy_parameter_ok(self): method test_a250_proxy_userpass (line 375) | def test_a250_proxy_userpass(self): method test_a260_process_save (line 386) | def test_a260_process_save(self): method test_zzz_links (line 400) | def test_zzz_links(self): method test_zzz_html (line 407) | def test_zzz_html(self): method test_zzz_etag_enabled (line 414) | def test_zzz_etag_enabled(self): method test_zzz_etag_not_working (line 425) | def test_zzz_etag_not_working(self): method test_zzz_unexpected_crawl_argument (line 436) | def test_zzz_unexpected_crawl_argument(self): method test_zzz_curl_get (line 440) | def test_zzz_curl_get(self): method test_zzz_curl_post (line 449) | def test_zzz_curl_post(self): method test_zzz_curl_put (line 458) | def test_zzz_curl_put(self): method test_zzz_curl_no_url (line 467) | def test_zzz_curl_no_url(self): method test_zzz_curl_bad_option (line 473) | def test_zzz_curl_bad_option(self): method test_zzz_robots_txt (line 484) | def test_zzz_robots_txt(self): method test_zzz_connect_timeout (line 489) | def test_zzz_connect_timeout(self): FILE: tests/test_message_queue.py class TestMessageQueue (line 17) | class TestMessageQueue(object): method setUpClass (line 20) | def setUpClass(self): method test_10_put (line 23) | def test_10_put(self): method test_20_get (line 32) | def test_20_get(self): method test_30_full (line 40) | def test_30_full(self): method test_40_multiple_threading_error (line 53) | def test_40_multiple_threading_error(self): class BuiltinQueue (line 67) | class BuiltinQueue(TestMessageQueue, unittest.TestCase): method setUpClass (line 69) | def setUpClass(self): class TestPikaRabbitMQ (line 78) | class TestPikaRabbitMQ(TestMessageQueue, unittest.TestCase): method setUpClass (line 81) | def setUpClass(self): method tearDownClass (line 93) | def tearDownClass(self): method test_30_full (line 100) | def test_30_full(self): class TestAmqpRabbitMQ (line 120) | class TestAmqpRabbitMQ(TestMessageQueue, unittest.TestCase): method setUpClass (line 123) | def setUpClass(self): method tearDownClass (line 138) | def tearDownClass(self): method test_30_full (line 145) | def test_30_full(self): class TestRedisQueue (line 164) | class TestRedisQueue(TestMessageQueue, unittest.TestCase): method setUpClass (line 167) | def setUpClass(self): method tearDownClass (line 183) | def tearDownClass(self): class TestKombuQueue (line 191) | class TestKombuQueue(TestMessageQueue, unittest.TestCase): method setUpClass (line 195) | def setUpClass(self): method tearDownClass (line 209) | def tearDownClass(self): class TestKombuAmpqQueue (line 222) | class TestKombuAmpqQueue(TestKombuQueue): class TestKombuRedisQueue (line 227) | class TestKombuRedisQueue(TestKombuQueue): class TestKombuMongoDBQueue (line 231) | class TestKombuMongoDBQueue(TestKombuQueue): FILE: tests/test_processor.py class TestProjectModule (line 20) | class TestProjectModule(unittest.TestCase): method base_task (line 23) | def base_task(self): method fetch_result (line 51) | def fetch_result(self): method setUp (line 66) | def setUp(self): method test_2_hello (line 83) | def test_2_hello(self): method test_3_echo (line 90) | def test_3_echo(self): method test_4_saved (line 97) | def test_4_saved(self): method test_5_echo_task (line 104) | def test_5_echo_task(self): method test_6_catch_status_code (line 111) | def test_6_catch_status_code(self): method test_7_raise_exception (line 120) | def test_7_raise_exception(self): method test_8_add_task (line 130) | def test_8_add_task(self): method test_10_cronjob (line 138) | def test_10_cronjob(self): method test_20_get_info (line 175) | def test_20_get_info(self): method test_30_generator (line 197) | def test_30_generator(self): method test_40_sleep (line 204) | def test_40_sleep(self): method test_50_timeout (line 214) | def test_50_timeout(self): method test_60_timeout_in_thread (line 231) | def test_60_timeout_in_thread(self): class TestProcessor (line 253) | class TestProcessor(unittest.TestCase): method setUpClass (line 257) | def setUpClass(self): method tearDownClass (line 278) | def tearDownClass(self): method test_10_update_project (line 285) | def test_10_update_project(self): method test_20_broken_project (line 315) | def test_20_broken_project(self): method test_30_new_task (line 331) | def test_30_new_task(self): method test_40_index_page (line 357) | def test_40_index_page(self): method test_50_fetch_error (line 401) | def test_50_fetch_error(self): method test_60_call_broken_project (line 446) | def test_60_call_broken_project(self): method test_70_update_project (line 481) | def test_70_update_project(self): method test_80_import_project (line 549) | def test_80_import_project(self): FILE: tests/test_response.py class TestResponse (line 23) | class TestResponse(unittest.TestCase): method setUpClass (line 31) | def setUpClass(self): method tearDownClass (line 38) | def tearDownClass(self): method get (line 41) | def get(self, url, **kwargs): method test_10_html (line 51) | def test_10_html(self): method test_20_xml (line 56) | def test_20_xml(self): method test_30_gzip (line 61) | def test_30_gzip(self): method test_40_deflate (line 66) | def test_40_deflate(self): method test_50_ok (line 71) | def test_50_ok(self): method test_60_not_ok (line 81) | def test_60_not_ok(self): method test_70_reraise_exception (line 92) | def test_70_reraise_exception(self): FILE: tests/test_result_dump.py class TestResultDump (line 45) | class TestResultDump(unittest.TestCase): method test_result_formater_1 (line 46) | def test_result_formater_1(self): method test_result_formater_2 (line 50) | def test_result_formater_2(self): method test_result_formater_error (line 54) | def test_result_formater_error(self): method test_dump_as_json (line 58) | def test_dump_as_json(self): method test_dump_as_json_valid (line 63) | def test_dump_as_json_valid(self): method test_dump_as_txt (line 68) | def test_dump_as_txt(self): method test_dump_as_csv (line 74) | def test_dump_as_csv(self): method test_dump_as_csv_case_1 (line 79) | def test_dump_as_csv_case_1(self): FILE: tests/test_result_worker.py class TestProcessor (line 21) | class TestProcessor(unittest.TestCase): method setUpClass (line 25) | def setUpClass(self): method tearDownClass (line 41) | def tearDownClass(self): method test_10_bad_result (line 48) | def test_10_bad_result(self): method test_10_bad_result_2 (line 54) | def test_10_bad_result_2(self): method test_20_insert_result (line 60) | def test_20_insert_result(self): method test_30_overwrite (line 77) | def test_30_overwrite(self): method test_40_insert_list (line 87) | def test_40_insert_list(self): FILE: tests/test_run.py class TestRun (line 25) | class TestRun(unittest.TestCase): method setUpClass (line 28) | def setUpClass(self): method tearDownClass (line 38) | def tearDownClass(self): method test_10_cli (line 50) | def test_10_cli(self): method test_20_cli_config (line 61) | def test_20_cli_config(self): method test_30_cli_command_line (line 81) | def test_30_cli_command_line(self): method test_30a_cli_command_line (line 94) | def test_30a_cli_command_line(self): method test_40_cli_env (line 107) | def test_40_cli_env(self): method test_50_docker_rabbitmq (line 120) | def test_50_docker_rabbitmq(self): method test_60_docker_mongodb (line 139) | def test_60_docker_mongodb(self): method test_60a_docker_couchdb (line 156) | def test_60a_docker_couchdb(self): method test_70_docker_mysql (line 176) | def test_70_docker_mysql(self): method test_80_docker_phantomjs (line 192) | def test_80_docker_phantomjs(self): method test_90_docker_scheduler (line 206) | def test_90_docker_scheduler(self): method test_a100_all (line 226) | def test_a100_all(self): method test_a110_one (line 273) | def test_a110_one(self): class TestSendMessage (line 336) | class TestSendMessage(unittest.TestCase): method setUpClass (line 339) | def setUpClass(self): method tearDownClass (line 358) | def tearDownClass(self): method test_10_send_message (line 372) | def test_10_send_message(self): FILE: tests/test_scheduler.py class TestTaskQueue (line 20) | class TestTaskQueue(unittest.TestCase): method setUpClass (line 23) | def setUpClass(self): method test_10_put (line 29) | def test_10_put(self): method test_20_update (line 36) | def test_20_update(self): method test_30_get_from_priority_queue (line 42) | def test_30_get_from_priority_queue(self): method test_40_time_queue_1 (line 46) | def test_40_time_queue_1(self): method test_50_time_queue_2 (line 51) | def test_50_time_queue_2(self): method test_60_processing_queue (line 58) | def test_60_processing_queue(self): method test_70_done (line 68) | def test_70_done(self): class TestBucket (line 80) | class TestBucket(unittest.TestCase): method test_bucket (line 82) | def test_bucket(self): class TestScheduler (line 105) | class TestScheduler(unittest.TestCase): method setUpClass (line 113) | def setUpClass(self): method tearDownClass (line 152) | def tearDownClass(self): method test_10_new_task_ignore (line 166) | def test_10_new_task_ignore(self): method test_20_new_project (line 178) | def test_20_new_project(self): method test_30_update_project (line 192) | def test_30_update_project(self): method test_32_get_info (line 207) | def test_32_get_info(self): method test_34_new_not_used_project (line 218) | def test_34_new_not_used_project(self): method test_35_new_task (line 234) | def test_35_new_task(self): method test_37_force_update_processing_task (line 267) | def test_37_force_update_processing_task(self): method test_40_taskdone_error_no_project (line 283) | def test_40_taskdone_error_no_project(self): method test_50_taskdone_error_no_track (line 295) | def test_50_taskdone_error_no_track(self): method test_60_taskdone_failed_retry (line 315) | def test_60_taskdone_failed_retry(self): method test_70_taskdone_ok (line 338) | def test_70_taskdone_ok(self): method test_75_on_finished_msg (line 358) | def test_75_on_finished_msg(self): method test_80_newtask_age_ignore (line 379) | def test_80_newtask_age_ignore(self): method test_82_newtask_via_rpc (line 400) | def test_82_newtask_via_rpc(self): method test_90_newtask_with_itag (line 421) | def test_90_newtask_with_itag(self): method test_a10_newtask_restart_by_age (line 450) | def test_a10_newtask_restart_by_age(self): method test_a20_failed_retry (line 470) | def test_a20_failed_retry(self): method test_a30_task_verify (line 511) | def test_a30_task_verify(self): method test_a40_success_recrawl (line 538) | def test_a40_success_recrawl(self): method test_a50_failed_recrawl (line 585) | def test_a50_failed_recrawl(self): method test_a60_disable_recrawl (line 620) | def test_a60_disable_recrawl(self): method test_38_cancel_task (line 648) | def test_38_cancel_task(self): method test_x10_inqueue_limit (line 691) | def test_x10_inqueue_limit(self): method test_x20_delete_project (line 716) | def test_x20_delete_project(self): method test_z10_startup (line 726) | def test_z10_startup(self): method test_z20_quit (line 729) | def test_z20_quit(self): class TestProject (line 741) | class TestProject(unittest.TestCase): method setUpClass (line 795) | def setUpClass(self): method test_pause_10_unpaused (line 809) | def test_pause_10_unpaused(self): method test_pause_20_no_enough_fail_tasks (line 812) | def test_pause_20_no_enough_fail_tasks(self): method test_pause_30_paused (line 833) | def test_pause_30_paused(self): method test_pause_40_unpause_checking (line 840) | def test_pause_40_unpause_checking(self): method test_pause_50_paused_again (line 844) | def test_pause_50_paused_again(self): method test_pause_60_unpause_checking (line 849) | def test_pause_60_unpause_checking(self): method test_pause_70_unpaused (line 853) | def test_pause_70_unpaused(self): method test_pause_x_disable_auto_pause (line 863) | def test_pause_x_disable_auto_pause(self): FILE: tests/test_task_queue.py class TestTaskQueue (line 13) | class TestTaskQueue(unittest.TestCase): method test_task_queue_in_time_order (line 18) | def test_task_queue_in_time_order(self): class TestTimeQueue (line 54) | class TestTimeQueue(unittest.TestCase): method test_time_queue (line 55) | def test_time_queue(self): FILE: tests/test_utils.py class TestFetcher (line 14) | class TestFetcher(unittest.TestCase): method test_readonlydict (line 15) | def test_readonlydict(self): method test_getitem (line 23) | def test_getitem(self): method test_format_data (line 37) | def test_format_data(self): FILE: tests/test_webdav.py class TestWebDav (line 22) | class TestWebDav(unittest.TestCase): method setUpClass (line 24) | def setUpClass(self): method tearDownClass (line 50) | def tearDownClass(self): method test_10_ls (line 64) | def test_10_ls(self): method test_20_create_error (line 67) | def test_20_create_error(self): method test_30_create_ok (line 76) | def test_30_create_ok(self): method test_40_get_404 (line 81) | def test_40_get_404(self): method test_50_get (line 88) | def test_50_get(self): method test_60_edit (line 99) | def test_60_edit(self): method test_70_get (line 102) | def test_70_get(self): method test_80_password (line 108) | def test_80_password(self): class TestWebDavNeedAuth (line 124) | class TestWebDavNeedAuth(unittest.TestCase): method setUpClass (line 126) | def setUpClass(self): method tearDownClass (line 153) | def tearDownClass(self): method test_10_ls (line 167) | def test_10_ls(self): method test_30_create_ok (line 173) | def test_30_create_ok(self): method test_50_get (line 177) | def test_50_get(self): FILE: tests/test_webui.py class TestWebUI (line 20) | class TestWebUI(unittest.TestCase): method setUpClass (line 23) | def setUpClass(self): method tearDownClass (line 73) | def tearDownClass(self): method test_10_index_page (line 92) | def test_10_index_page(self): method test_20_debug (line 97) | def test_20_debug(self): method test_25_debug_post (line 112) | def test_25_debug_post(self): method test_30_run (line 133) | def test_30_run(self): method test_32_run_bad_task (line 144) | def test_32_run_bad_task(self): method test_33_run_bad_script (line 154) | def test_33_run_bad_script(self): method test_35_run_http_task (line 164) | def test_35_run_http_task(self): method test_40_save (line 173) | def test_40_save(self): method test_42_get (line 180) | def test_42_get(self): method test_45_run_with_saved_script (line 187) | def test_45_run_with_saved_script(self): method test_50_index_page_list (line 199) | def test_50_index_page_list(self): method test_52_change_status (line 204) | def test_52_change_status(self): method test_55_reopen (line 213) | def test_55_reopen(self): method test_57_resave (line 218) | def test_57_resave(self): method test_58_index_page_list (line 225) | def test_58_index_page_list(self): method test_60_change_rate (line 230) | def test_60_change_rate(self): method test_70_change_status (line 239) | def test_70_change_status(self): method test_80_change_group (line 248) | def test_80_change_group(self): method test_90_run (line 261) | def test_90_run(self): method test_a10_counter (line 269) | def test_a10_counter(self): method test_a15_queues (line 285) | def test_a15_queues(self): method test_a20_tasks (line 296) | def test_a20_tasks(self): method test_a22_active_tasks (line 315) | def test_a22_active_tasks(self): method test_a24_task (line 334) | def test_a24_task(self): method test_a25_task_json (line 339) | def test_a25_task_json(self): method test_a26_debug_task (line 344) | def test_a26_debug_task(self): method test_a30_results (line 348) | def test_a30_results(self): method test_a30_export_json (line 354) | def test_a30_export_json(self): method test_a32_export_json_style_full (line 359) | def test_a32_export_json_style_full(self): method test_a34_export_json_style_full_limit_1 (line 365) | def test_a34_export_json_style_full_limit_1(self): method test_a40_export_url_json (line 371) | def test_a40_export_url_json(self): method test_a50_export_csv (line 376) | def test_a50_export_csv(self): method test_a60_fetch_via_cannot_connect_fetcher (line 381) | def test_a60_fetch_via_cannot_connect_fetcher(self): method test_a70_fetch_via_fetcher (line 396) | def test_a70_fetch_via_fetcher(self): method test_h000_auth (line 412) | def test_h000_auth(self): method test_h005_no_such_project (line 422) | def test_h005_no_such_project(self): method test_h005_unknown_field (line 430) | def test_h005_unknown_field(self): method test_h005_rate_wrong_format (line 438) | def test_h005_rate_wrong_format(self): method test_h010_change_group (line 446) | def test_h010_change_group(self): method test_h020_change_group_lock_failed (line 459) | def test_h020_change_group_lock_failed(self): method test_h020_change_group_lock_ok (line 467) | def test_h020_change_group_lock_ok(self): method test_h030_need_auth (line 477) | def test_h030_need_auth(self): method test_h040_auth_fail (line 488) | def test_h040_auth_fail(self): method test_h050_auth_fail2 (line 492) | def test_h050_auth_fail2(self): method test_h060_auth_fail3 (line 498) | def test_h060_auth_fail3(self): method test_h070_auth_ok (line 504) | def test_h070_auth_ok(self): method test_x0_disconnected_scheduler (line 510) | def test_x0_disconnected_scheduler(self): method test_x10_project_update (line 518) | def test_x10_project_update(self): method test_x20_counter (line 527) | def test_x20_counter(self): method test_x30_run_not_exists_project (line 532) | def test_x30_run_not_exists_project(self): method test_x30_run (line 538) | def test_x30_run(self): method test_x40_debug_save (line 545) | def test_x40_debug_save(self): method test_x50_tasks (line 552) | def test_x50_tasks(self): method test_x60_robots (line 556) | def test_x60_robots(self): method test_x70_bench (line 561) | def test_x70_bench(self): FILE: tests/test_xmlrpc.py class TestXMLRPCServer (line 23) | class TestXMLRPCServer(unittest.TestCase): method setUpClass (line 25) | def setUpClass(self): method tearDownClass (line 48) | def tearDownClass(self): method test_xmlrpc_server (line 52) | def test_xmlrpc_server(self, uri='http://127.0.0.1:3423'): FILE: tools/migrate.py function taskdb_migrating (line 20) | def taskdb_migrating(project, from_connection, to_connection): function resultdb_migrating (line 30) | def resultdb_migrating(project, from_connection, to_connection): function migrate (line 43) | def migrate(pool, from_connection, to_connection):