SYMBOL INDEX (244 symbols across 73 files)

FILE: example-spider/bilibili/main.go
  function main (line 9) | func main() {

FILE: example-spider/dilidili/main.go
  function main (line 9) | func main() {

FILE: example-spider/douban-movie-code/main.go
  type Movies (line 11) | type Movies struct
  type Movie (line 14) | type Movie struct
  type PageProcess (line 24) | type PageProcess struct
    method Process (line 26) | func (p *PageProcess) Process(context model.Context) (*model.Page, err...
  function main (line 38) | func main() {

FILE: example-spider/douban-movie/main.go
  function main (line 9) | func main() {

FILE: example-spider/empty/main.go
  function main (line 7) | func main() {

FILE: example-spider/haoqi/main.go
  function main (line 9) | func main() {

FILE: example-spider/jingdong/main.go
  function main (line 10) | func main() {

FILE: example-spider/qiongyou/main.go
  function main (line 9) | func main() {

FILE: example-spider/qiubai/main.go
  function main (line 9) | func main() {

FILE: example-spider/qiubai/sem_test.go
  function Example (line 12) | func Example() {

FILE: example-spider/ttkb-author/main.go
  function main (line 9) | func main() {

FILE: example-spider/ttkb/main.go
  function main (line 10) | func main() {

FILE: example-spider/tuiku/main.go
  function main (line 9) | func main() {

FILE: example-spider/wangyi-music/main.go
  function main (line 11) | func main() {

FILE: example-spider/woshipm/main.go
  function main (line 9) | func main() {

FILE: manage/config/config.go
  type Config (line 12) | type Config struct
  function InitConfig (line 21) | func InitConfig() error {

FILE: manage/discover/discover.go
  type Discover (line 9) | type Discover interface
  function InitDiscover (line 16) | func InitDiscover() error {
  function GetNodes (line 29) | func GetNodes() map[string]*model.Node {

FILE: manage/discover/etcd/etcd.go
  type Cluster (line 15) | type Cluster struct
    method Start (line 40) | func (c *Cluster) Start() error {
    method GetNodes (line 46) | func (c *Cluster) GetNodes() map[string]*model.Node {
    method addWorker (line 51) | func (c *Cluster) addWorker(info *model.WorkerInfo) {
    method updateWorker (line 63) | func (c *Cluster) updateWorker(info *model.WorkerInfo) {
    method WatchWorkers (line 77) | func (c *Cluster) WatchWorkers() {
  function NewCluster (line 20) | func NewCluster(endpoints []string) (*Cluster, error) {
  function unmarshal (line 67) | func unmarshal(node *client.Node) *model.WorkerInfo {

FILE: manage/discover/file/file.go
  function init (line 3) | func init() {

FILE: manage/discover/zookeeper/zookeeper.go
  function init (line 3) | func init() {

FILE: manage/http/controller.go
  function AddTask (line 17) | func AddTask(w http.ResponseWriter, req *http.Request) {
  function StopTask (line 42) | func StopTask(w http.ResponseWriter, req *http.Request) {
  function RunTask (line 64) | func RunTask(w http.ResponseWriter, req *http.Request) {
  function EndTask (line 86) | func EndTask(w http.ResponseWriter, req *http.Request) {
  function ListTask (line 108) | func ListTask(w http.ResponseWriter, req *http.Request) {
  function ListNode (line 130) | func ListNode(w http.ResponseWriter, req *http.Request) {

FILE: manage/http/request.go
  function init (line 16) | func init() {
  function makeCookiejar (line 20) | func makeCookiejar() http.CookieJar {
  function MakeClient (line 29) | func MakeClient(transport http.RoundTripper) *http.Client {
  function Get (line 33) | func Get(url string) ([]byte, error) {
  function Post (line 47) | func Post(url string, data interface{}) ([]byte, error) {
  function DoRequest (line 65) | func DoRequest(method string, url string, data []byte) (resp *http.Respo...

FILE: manage/http/server.go
  function InitHttpServer (line 9) | func InitHttpServer() {

FILE: manage/http/service.go
  function AddTaskS (line 11) | func AddTaskS(task *model.Task) ([]byte, error) {
  function RunTaskS (line 16) | func RunTaskS(name string) ([]byte, error) {
  function StopTaskS (line 21) | func StopTaskS(name string) ([]byte, error) {
  function EndTaskS (line 26) | func EndTaskS(name string) ([]byte, error) {
  function ListTaskS (line 31) | func ListTaskS(name string) ([]byte, error) {
  function ListNodesS (line 37) | func ListNodesS() ([]byte, error) {
  function getUrl (line 42) | func getUrl(ip string, path string) string {

FILE: manage/logger/logger.go
  function Info (line 5) | func Info(v ...interface{}) {
  function Debug (line 9) | func Debug(v ...interface{}) {
  function Warn (line 13) | func Warn(v ...interface{}) {
  function Error (line 17) | func Error(v ...interface{}) {

FILE: manage/main.go
  function main (line 10) | func main() {

FILE: manage/model/node_info.go
  type Node (line 3) | type Node struct
  type WorkerInfo (line 12) | type WorkerInfo struct
  type SpiderData (line 20) | type SpiderData struct

FILE: manage/model/task.go
  type Task (line 3) | type Task struct
  type RequestBody (line 24) | type RequestBody struct
  type Cookies (line 29) | type Cookies struct
  type Process (line 34) | type Process struct
  type TemplateRule (line 42) | type TemplateRule struct
  type JsonRule (line 46) | type JsonRule struct

FILE: manage/schedule/request.go
  function init (line 17) | func init() {
  function makeCookiejar (line 21) | func makeCookiejar() http.CookieJar {
  function MakeClient (line 30) | func MakeClient(transport http.RoundTripper) *http.Client {
  function Get (line 34) | func Get(url string) ([]byte, error) {
  function Post (line 48) | func Post(url string, data interface{}) ([]byte, error) {
  function DoRequest (line 66) | func DoRequest(method string, url string, data []byte) (resp *http.Respo...

FILE: manage/schedule/schedule.go
  function AddTask (line 10) | func AddTask(task *model.Task) ([]byte, error) {
  function RunTask (line 15) | func RunTask(name string) ([]byte, error) {
  function StopTask (line 20) | func StopTask(name string) ([]byte, error) {
  function EndTask (line 25) | func EndTask(name string) ([]byte, error) {
  function ListTask (line 30) | func ListTask(name string) ([]byte, error) {
  function getUrl (line 35) | func getUrl(ip string, path string) string {

FILE: manage/strategy/rand_strategy.go
  function GetNode (line 8) | func GetNode() *model.Node {

FILE: manage/task/task.go
  function init (line 3) | func init() {

FILE: spider/boot.go
  type Boot (line 11) | type Boot struct
    method AddSpider (line 29) | func (s *Boot) AddSpider(spider *spider.Spider) *core.Engine {
    method Run (line 33) | func (s *Boot) Run() {
  function init (line 15) | func init() {
  function New (line 23) | func New() *Boot {

FILE: spider/common/encode.go
  function ToUtf8 (line 45) | func ToUtf8(html []byte) ([]byte, error) {

FILE: spider/common/prase_req.go
  function PraseReq (line 11) | func PraseReq(reqs []*model.Request, ctx map[string]interface{}) []*mode...
  function FindRule (line 24) | func FindRule(text string) [][]string {
  function isRuleReq (line 29) | func isRuleReq(req *model.Request, ctx map[string]interface{}) ([]*model...
  function PraseOffset (line 61) | func PraseOffset(req *model.Request) []*model.Request {
  function PraseOr (line 114) | func PraseOr(req *model.Request) []*model.Request {
  function PraseParamCtx (line 143) | func PraseParamCtx(req *model.Request, rules [][]string, ctx map[string]...

FILE: spider/common/prase_req_test.go
  function TestPraseOffset (line 9) | func TestPraseOffset(t *testing.T) {
  function TestPraseOr (line 23) | func TestPraseOr(t *testing.T) {
  function TestPraseParamCtx (line 37) | func TestPraseParamCtx(t *testing.T) {
  function TestFindRule (line 54) | func TestFindRule(t *testing.T) {
  function TestOrAndOffset (line 63) | func TestOrAndOffset(t *testing.T){
  function TestOrAndOffset_2 (line 83) | func TestOrAndOffset_2(t *testing.T) {

FILE: spider/config/config.go
  type Config (line 12) | type Config struct
  function InitConfig (line 26) | func InitConfig() error {

FILE: spider/core/engine.go
  function New (line 13) | func New() *Engine {
  function GetEnine (line 20) | func GetEnine() *Engine {
  type Engine (line 24) | type Engine struct
    method AddSpider (line 28) | func (m *Engine) AddSpider(spider *spider.Spider) *Engine {
    method RunTask (line 35) | func (m *Engine) RunTask(name string) error {
    method StopTask (line 44) | func (m *Engine) StopTask(name string) error {
    method EndTask (line 53) | func (m *Engine) EndTask(name string) error {
    method ListTask (line 62) | func (m *Engine) ListTask() []*spider.Spider {
    method GetTaskMetas (line 70) | func (m *Engine) GetTaskMetas() map[string]*TaskMeta {
    method Run (line 78) | func (m *Engine) Run() {

FILE: spider/core/runtime.go
  constant Default_WorkNum (line 20) | Default_WorkNum = 1
  type SpiderRuntime (line 22) | type SpiderRuntime struct
    method SetSpider (line 70) | func (s *SpiderRuntime) SetSpider(spider *spider.Spider) {
    method GetSpider (line 74) | func (s *SpiderRuntime) GetSpider() *spider.Spider {
    method Run (line 78) | func (s *SpiderRuntime) Run() {
    method Stop (line 90) | func (s *SpiderRuntime) Stop() {
    method worker (line 94) | func (s *SpiderRuntime) worker() {
    method download (line 187) | func (s *SpiderRuntime) download(req *model.Request) (*http.Response, ...
    method Exit (line 199) | func (s *SpiderRuntime) Exit() {
  type TaskMeta (line 34) | type TaskMeta struct
  function NewSpiderRuntime (line 43) | func NewSpiderRuntime() *SpiderRuntime {
  function processWrapper (line 176) | func processWrapper(p process.Process, context model.Context) (*model.Pa...

FILE: spider/downloader/request.go
  function init (line 19) | func init() {
  function makeCookiejar (line 23) | func makeCookiejar() http.CookieJar {
  function makeClient (line 32) | func makeClient(transport http.RoundTripper, jar http.CookieJar) *http.C...
  function Get (line 36) | func Get(taskId string, url string) (*http.Response, error) {
  function PostJson (line 49) | func PostJson(taskId string, url string, data interface{}) (*http.Respon...
  function doRequest (line 65) | func doRequest(id string, method string, url string, data []byte) (resp ...
  function setClient (line 79) | func setClient(id string, client *http.Client) {
  function getClient (line 85) | func getClient(id string) *http.Client {

FILE: spider/downloader/request_test.go
  function TestGet (line 5) | func TestGet(t *testing.T) {
  function TestPostJson (line 11) | func TestPostJson(t *testing.T) {

FILE: spider/http/server.go
  function AddTask (line 21) | func AddTask(w http.ResponseWriter, req *http.Request) {
  function StopTask (line 46) | func StopTask(w http.ResponseWriter, req *http.Request) {
  function RunTask (line 66) | func RunTask(w http.ResponseWriter, req *http.Request) {
  function EndTask (line 86) | func EndTask(w http.ResponseWriter, req *http.Request) {
  function ListTask (line 106) | func ListTask(w http.ResponseWriter, req *http.Request) {
  function InitHttpServer (line 122) | func InitHttpServer() {

FILE: spider/logger/logger.go
  function Info (line 5) | func Info(v ...interface{}) {
  function Debug (line 9) | func Debug(v ...interface{}) {
  function Warn (line 13) | func Warn(v ...interface{}) {
  function Error (line 17) | func Error(v ...interface{}) {

FILE: spider/model/context.go
  type Context (line 5) | type Context struct
    method Clear (line 11) | func (c *Context) Clear() {

FILE: spider/model/page.go
  type Page (line 3) | type Page struct
    method AddUrl (line 9) | func (p *Page) AddUrl(req *Request) {
    method AddUrls (line 16) | func (p *Page) AddUrls(req []*Request) {
    method AddResult (line 23) | func (p *Page) AddResult(value map[string]interface{}) {

FILE: spider/model/task.go
  type Task (line 7) | type Task struct
  type Request (line 19) | type Request struct
    method Write (line 29) | func (r *Request) Write() ([]byte, error) {
    method Read (line 33) | func (r *Request) Read(b []byte) error {
  type Cookies (line 37) | type Cookies struct
  type Process (line 42) | type Process struct
  type TemplateRule (line 51) | type TemplateRule struct
  type JsonRule (line 55) | type JsonRule struct

FILE: spider/pipline/console/console.go
  type ConsolePipline (line 8) | type ConsolePipline struct
    method ProcessData (line 15) | func (c *ConsolePipline) ProcessData(v []map[string]interface{}, taskN...
  function NewConsolePipline (line 11) | func NewConsolePipline() *ConsolePipline {

FILE: spider/pipline/file/file.go
  type FilePipline (line 11) | type FilePipline struct
    method ProcessData (line 20) | func (c *FilePipline) ProcessData(v []map[string]interface{}, taskName...
    method Close (line 52) | func (c *FilePipline) Close() {
  function NewFilePipline (line 16) | func NewFilePipline(root string) *FilePipline {

FILE: spider/pipline/mysql/dbModel.go
  type Field (line 10) | type Field struct
    method Sql (line 16) | func (f *Field) Sql() string{
  type DBModel (line 44) | type DBModel struct
    method TableSql (line 49) | func (d *DBModel) TableSql() string{
    method InsertSql (line 60) | func (d *DBModel) InsertSql() string{
    method InsertArgs (line 69) | func (d *DBModel) InsertArgs() []interface{}{
  function NewDBModel (line 86) | func NewDBModel(name string,m map[string]interface{}) *DBModel{

FILE: spider/pipline/mysql/mysql.go
  function InitMysql (line 12) | func InitMysql(mysql string) {
  type C (line 19) | type C struct
  function CreateTable (line 23) | func CreateTable(m *DBModel) error{
  function Add (line 33) | func Add(m *DBModel) error{

FILE: spider/pipline/mysql/mysqlPipline.go
  type MysqlPipline (line 7) | type MysqlPipline struct
    method ProcessData (line 15) | func (c *MysqlPipline) ProcessData(v []map[string]interface{}, taskNam...
  function NewMysqlPipline (line 11) | func NewMysqlPipline() *MysqlPipline {

FILE: spider/pipline/pipline.go
  type Pipline (line 3) | type Pipline interface

FILE: spider/process/filter/repoat_filter.go
  function init (line 11) | func init() {
  function RepeatFilter (line 15) | func RepeatFilter(url string, process *model.Process) bool {
  function get (line 24) | func get(name string) bool {
  function put (line 31) | func put(name string) {

FILE: spider/process/filter/repoat_filter_test.go
  function TestRepeatFilter (line 9) | func TestRepeatFilter(t *testing.T) {

FILE: spider/process/filter/url_filter.go
  function Filter (line 8) | func Filter(url string, process *model.Process) bool {

FILE: spider/process/filter/url_filter_test.go
  function TestFilter (line 9) | func TestFilter(t *testing.T) {

FILE: spider/process/json-process/json_process.go
  type JsonProcess (line 7) | type JsonProcess struct
    method Process (line 15) | func (j *JsonProcess) Process(context model.Context) (*model.Page, err...
  function NewJsonProcess (line 11) | func NewJsonProcess(jsonProcess *model.Process) *JsonProcess {

FILE: spider/process/json-process/json_rule.go
  function JsonRuleProcess (line 11) | func JsonRuleProcess(process *model.Process, context model.Context) (*mo...
  function Process (line 15) | func Process(process *model.Process, context model.Context) (*model.Page...

FILE: spider/process/process.go
  type Process (line 7) | type Process interface

FILE: spider/process/template-process/template_process.go
  type TemplateProcess (line 7) | type TemplateProcess struct
    method Process (line 15) | func (t *TemplateProcess) Process(context model.Context) (*model.Page,...
  function NewTemplateProcess (line 11) | func NewTemplateProcess(tempProcess *model.Process) *TemplateProcess {

FILE: spider/process/template-process/template_rule.go
  function TemplateRuleProcess (line 16) | func TemplateRuleProcess(process *model.Process, context model.Context) ...
  function getMapFromDom (line 72) | func getMapFromDom(rule map[string]string, node *goquery.Selection) map[...
  function getComplateUrl (line 144) | func getComplateUrl(url *url2.URL, href string) string {

FILE: spider/process/template-process/template_rule_test.go
  function TestTemplateProcess (line 10) | func TestTemplateProcess(t *testing.T) {

FILE: spider/register/etcd/etcd.go
  type Worker (line 14) | type Worker struct
    method HeartBeat (line 56) | func (w *Worker) HeartBeat() {
  type WorkerInfo (line 20) | type WorkerInfo struct
  type SpiderData (line 27) | type SpiderData struct
  function NewWorker (line 36) | func NewWorker(name, IP string, endpoints []string) *Worker {
  function getSpiderData (line 80) | func getSpiderData() map[string]*SpiderData {

FILE: spider/schedule/schedule.go
  type Schedule (line 8) | type Schedule interface
  function RegisterSchedule (line 20) | func RegisterSchedule(name string, builder func(*config.Config) Schedule) {
  function GetSchedule (line 24) | func GetSchedule(c *config.Config) Schedule {

FILE: spider/schedule/schedule_chan.go
  type ChanSchedule (line 10) | type ChanSchedule struct
    method Push (line 20) | func (d *ChanSchedule) Push(req *model.Request) {
    method PushMuti (line 28) | func (d *ChanSchedule) PushMuti(reqs []*model.Request) {
    method Pop (line 36) | func (d *ChanSchedule) Pop() (*model.Request, bool) {
    method Count (line 42) | func (d *ChanSchedule) Count() int {
    method Close (line 46) | func (d *ChanSchedule) Close() {
  function NewChanSchedule (line 14) | func NewChanSchedule(config *config.Config) Schedule {
  function init (line 50) | func init() {

FILE: spider/schedule/schedule_chan_test.go
  function TestInitDownloader (line 7) | func TestInitDownloader(t *testing.T) {

FILE: spider/schedule/schedule_redis.go
  type RedisSchedule (line 12) | type RedisSchedule struct
    method connect (line 27) | func (r *RedisSchedule) connect() {
    method Push (line 37) | func (r *RedisSchedule) Push(req *model.Request) {
    method PushMuti (line 57) | func (r *RedisSchedule) PushMuti(reqs []*model.Request) {
    method Pop (line 77) | func (r *RedisSchedule) Pop() (*model.Request, bool) {
    method Count (line 97) | func (r *RedisSchedule) Count() int {
    method Close (line 109) | func (r *RedisSchedule) Close() {
    method CronCount (line 113) | func (r *RedisSchedule) CronCount(flushTime int) {
  function NewRedisSchedule (line 18) | func NewRedisSchedule(config *config.Config) Schedule {
  function init (line 122) | func init() {

FILE: spider/schedule/schedule_redis_test.go
  function TestRedisSchedule_Push (line 10) | func TestRedisSchedule_Push(t *testing.T) {
  function TestRedisSchedule_Pop (line 15) | func TestRedisSchedule_Pop(t *testing.T) {

FILE: spider/spider/spider.go
  type Spider (line 15) | type Spider struct
    method GetPipline (line 28) | func (s *Spider) GetPipline() pipline.Pipline {
    method GetProcess (line 32) | func (s *Spider) GetProcess(name string) []process.Process {
    method GetRequests (line 36) | func (s *Spider) GetRequests() []*model.Request {
    method AddProcess (line 40) | func (s *Spider) AddProcess(name string, p process.Process) {
  function InitWithTask (line 53) | func InitWithTask(task *model.Task) *Spider {

FILE: storage/config/config.go
  type Config (line 12) | type Config struct
  function InitConfig (line 17) | func InitConfig() error {

FILE: storage/db/elasticsearch/elasticsearch.go
  function init (line 3) | func init() {

FILE: storage/logger/logger.go
  function Info (line 5) | func Info(v ...interface{}) {
  function Debug (line 9) | func Debug(v ...interface{}) {
  function Warn (line 13) | func Warn(v ...interface{}) {
  function Error (line 17) | func Error(v ...interface{}) {

FILE: storage/main.go
  function main (line 8) | func main() {