Repository: icpd/subscribe2clash Branch: main Commit: 41a0dfdfc9b4 Files: 117 Total size: 784.3 KB Directory structure: gitextract_x32qht1_/ ├── .gitattributes ├── .gitignore ├── README.md ├── crawler/ │ ├── config/ │ │ └── config.go │ ├── engine/ │ │ ├── concurrent.go │ │ ├── simple.go │ │ ├── types.go │ │ └── worker.go │ ├── fetcher/ │ │ └── fetcher.go │ ├── frontend/ │ │ ├── controller/ │ │ │ └── searchresult.go │ │ ├── model/ │ │ │ └── page.go │ │ ├── starter.go │ │ └── view/ │ │ ├── css/ │ │ │ └── style.css │ │ ├── index.html │ │ ├── js/ │ │ │ └── index.js │ │ ├── searchresult.go │ │ ├── searchresult_test.go │ │ └── template.html │ ├── main.go │ ├── model/ │ │ ├── car.go │ │ └── profile.go │ ├── persist/ │ │ ├── itemsaver.go │ │ └── itemsaver_test.go │ ├── scheduler/ │ │ ├── queued.go │ │ └── simple.go │ ├── xcar/ │ │ └── parser/ │ │ ├── cardetail.go │ │ ├── cardetail_test.go │ │ ├── cardetail_test_data.html │ │ ├── carlist.go │ │ ├── carlist_test.go │ │ ├── carlist_test_data.html │ │ ├── carmodel.go │ │ ├── carmodel_test.go │ │ └── carmodel_test_data.html │ └── zhenai/ │ └── parser/ │ ├── city.go │ ├── citylist.go │ ├── citylist_test.go │ ├── citylist_test_data.html │ ├── profile.go │ ├── profile_test.go │ └── profile_test_data.html ├── crawler_distributed/ │ ├── config/ │ │ └── config.go │ ├── main.go │ ├── persist/ │ │ ├── client/ │ │ │ └── itemsaver.go │ │ ├── rpc.go │ │ └── server/ │ │ ├── client_test.go │ │ └── itemsaver.go │ ├── rpcsupport/ │ │ └── rpc.go │ └── worker/ │ ├── client/ │ │ └── worker.go │ ├── rpc.go │ ├── server/ │ │ ├── client_test.go │ │ └── worker.go │ └── types.go ├── go.mod ├── go.sum ├── lang/ │ ├── basic/ │ │ ├── atomic/ │ │ │ └── atomic.go │ │ ├── basic/ │ │ │ ├── basic.go │ │ │ └── triangle_test.go │ │ ├── branch/ │ │ │ ├── abc.txt │ │ │ └── branch.go │ │ ├── func/ │ │ │ └── func.go │ │ ├── loop/ │ │ │ └── loop.go │ │ └── regex/ │ │ └── regex.go │ ├── channel/ │ │ ├── channel.go │ │ ├── done/ │ │ │ └── done.go │ │ ├── pattern/ │ │ │ └── main.go │ │ └── select/ │ │ └── select.go │ ├── container/ │ │ ├── arrays/ │ │ │ └── arrays.go │ │ ├── maps/ │ │ │ └── maps.go │ │ ├── nonrepeatingsubstr/ │ │ │ ├── nonrepeating.go │ │ │ └── nonrepeating_test.go │ │ ├── slices/ │ │ │ ├── sliceops.go │ │ │ └── slices.go │ │ └── strings/ │ │ └── strings.go │ ├── errhandling/ │ │ ├── defer/ │ │ │ └── defer.go │ │ ├── filelistingserver/ │ │ │ ├── errwrapper_test.go │ │ │ ├── filelisting/ │ │ │ │ └── handler.go │ │ │ └── web.go │ │ └── recover/ │ │ └── recover.go │ ├── functional/ │ │ ├── adder/ │ │ │ └── adder.go │ │ ├── fib/ │ │ │ └── fib.go │ │ └── main.go │ ├── goroutine/ │ │ └── goroutine.go │ ├── http/ │ │ ├── client.go │ │ └── gindemo/ │ │ └── ginserver.go │ ├── json/ │ │ └── main.go │ ├── maze/ │ │ ├── maze.go │ │ └── maze.in │ ├── queue/ │ │ ├── queue.go │ │ ├── queue_test.go │ │ └── queueentry/ │ │ └── main.go │ ├── retriever/ │ │ ├── main.go │ │ ├── mock/ │ │ │ └── mockretriever.go │ │ └── real/ │ │ └── retriever.go │ ├── rpc/ │ │ ├── client/ │ │ │ └── main.go │ │ ├── rpc.go │ │ └── server/ │ │ └── main.go │ └── tree/ │ ├── node.go │ ├── traversal.go │ ├── treeentry/ │ │ └── entry.go │ └── treeentry_embedded/ │ └── entry.go └── mockserver/ ├── config/ │ └── config.go ├── generator/ │ ├── city/ │ │ ├── city.go │ │ ├── city_test.go │ │ └── city_tmpl.html │ ├── citylist/ │ │ ├── citylist.go │ │ ├── citylist_test.go │ │ └── citylist_tmpl.html │ └── profile/ │ ├── profile.go │ ├── profile_test.go │ └── profile_tmpl.html ├── main.go ├── recommendation/ │ ├── rcmd.go │ └── rcmd_test.go └── static/ ├── css/ │ └── blog.css ├── index.html └── instructions.html ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ *.html linguist-language=go *.css linguist-language=go *.js linguist-language=go ================================================ FILE: .gitignore ================================================ ================================================ FILE: README.md ================================================ Interesting things start from viewing forks. ================================================ FILE: crawler/config/config.go ================================================ package config const ( // Parser names ParseCity = "ParseCity" ParseCityList = "ParseCityList" ParseProfile = "ParseProfile" ParseCarDetail = "ParseCarDetail" ParseCarList = "ParseCarList" ParseCarModel = "ParseCarModel" NilParser = "NilParser" // ElasticSearch ElasticIndex = "car_profile" // Rate limiting Qps = 2 ) ================================================ FILE: crawler/engine/concurrent.go ================================================ package engine type ConcurrentEngine struct { Scheduler Scheduler WorkerCount int ItemChan chan Item RequestProcessor Processor } type Processor func(Request) (ParseResult, error) type Scheduler interface { ReadyNotifier Submit(Request) WorkerChan() chan Request Run() } type ReadyNotifier interface { WorkerReady(chan Request) } func (e *ConcurrentEngine) Run(seeds ...Request) { out := make(chan ParseResult) e.Scheduler.Run() for i := 0; i < e.WorkerCount; i++ { e.createWorker(e.Scheduler.WorkerChan(), out, e.Scheduler) } for _, r := range seeds { if isDuplicate(r.Url) { continue } e.Scheduler.Submit(r) } for { result := <-out for _, item := range result.Items { go func(i Item) { e.ItemChan <- i }(item) } for _, request := range result.Requests { if isDuplicate(request.Url) { continue } e.Scheduler.Submit(request) } } } func (e *ConcurrentEngine) createWorker( in chan Request, out chan ParseResult, ready ReadyNotifier) { go func() { for { ready.WorkerReady(in) request := <-in result, err := e.RequestProcessor( request) if err != nil { continue } out <- result } }() } var visitedUrls = make(map[string]bool) func isDuplicate(url string) bool { if visitedUrls[url] { return true } visitedUrls[url] = true return false } ================================================ FILE: crawler/engine/simple.go ================================================ package engine import ( "log" ) type SimpleEngine struct{} func (e SimpleEngine) Run(seeds ...Request) { var requests []Request for _, r := range seeds { requests = append(requests, r) } for len(requests) > 0 { r := requests[0] requests = requests[1:] parseResult, err := Worker(r) if err != nil { continue } requests = append(requests, parseResult.Requests...) for _, item := range parseResult.Items { log.Printf("Got item: %v", item) } } } ================================================ FILE: crawler/engine/types.go ================================================ package engine import "imooc.com/ccmouse/learngo/crawler/config" type ParserFunc func( contents []byte, url string) ParseResult type Parser interface { Parse(contents []byte, url string) ParseResult Serialize() (name string, args interface{}) } type Request struct { Url string Parser Parser } type ParseResult struct { Requests []Request Items []Item } type Item struct { Url string Type string Id string Payload interface{} } type NilParser struct{} func (NilParser) Parse( _ []byte, _ string) ParseResult { return ParseResult{} } func (NilParser) Serialize() ( name string, args interface{}) { return config.NilParser, nil } type FuncParser struct { parser ParserFunc name string } func (f *FuncParser) Parse( contents []byte, url string) ParseResult { return f.parser(contents, url) } func (f *FuncParser) Serialize() ( name string, args interface{}) { return f.name, nil } func NewFuncParser( p ParserFunc, name string) *FuncParser { return &FuncParser{ parser: p, name: name, } } ================================================ FILE: crawler/engine/worker.go ================================================ package engine import ( "log" "imooc.com/ccmouse/learngo/crawler/fetcher" ) func Worker(r Request) (ParseResult, error) { body, err := fetcher.Fetch(r.Url) if err != nil { log.Printf("Fetcher: error "+ "fetching url %s: %v", r.Url, err) return ParseResult{}, err } return r.Parser.Parse(body, r.Url), nil } ================================================ FILE: crawler/fetcher/fetcher.go ================================================ package fetcher import ( "bufio" "fmt" "io/ioutil" "net/http" "log" "time" "golang.org/x/net/html/charset" "golang.org/x/text/encoding" "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" "imooc.com/ccmouse/learngo/crawler/config" ) var ( rateLimiter = time.Tick( time.Second / config.Qps) verboseLogging = false ) func SetVerboseLogging() { verboseLogging = true } func Fetch(url string) ([]byte, error) { <-rateLimiter if verboseLogging { log.Printf("Fetching url %s", url) } resp, err := http.Get(url) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode) } bodyReader := bufio.NewReader(resp.Body) e := determineEncoding(bodyReader) utf8Reader := transform.NewReader(bodyReader, e.NewDecoder()) return ioutil.ReadAll(utf8Reader) } func determineEncoding( r *bufio.Reader) encoding.Encoding { bytes, err := r.Peek(1024) if err != nil { log.Printf("Fetcher error: %v", err) return unicode.UTF8 } e, _, _ := charset.DetermineEncoding( bytes, "") return e } ================================================ FILE: crawler/frontend/controller/searchresult.go ================================================ package controller import ( "context" "net/http" "reflect" "regexp" "strconv" "strings" "github.com/olivere/elastic/v7" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/frontend/model" "imooc.com/ccmouse/learngo/crawler/frontend/view" ) type SearchResultHandler struct { view view.SearchResultView client *elastic.Client } func CreateSearchResultHandler( template string) SearchResultHandler { client, err := elastic.NewClient( elastic.SetSniff(false)) if err != nil { panic(err) } return SearchResultHandler{ view: view.CreateSearchResultView( template), client: client, } } func (h SearchResultHandler) ServeHTTP( w http.ResponseWriter, req *http.Request) { q := strings.TrimSpace(req.FormValue("q")) from, err := strconv.Atoi( req.FormValue("from")) if err != nil { from = 0 } page, err := h.getSearchResult(q, from) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } err = h.view.Render(w, page) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } const pageSize = 10 func (h SearchResultHandler) getSearchResult( q string, from int) (model.SearchResult, error) { var result model.SearchResult result.Query = q resp, err := h.client. Search(config.ElasticIndex). Query(elastic.NewQueryStringQuery( rewriteQueryString(q))). From(from). Do(context.Background()) if err != nil { return result, err } result.Hits = resp.TotalHits() result.Start = from result.Items = resp.Each( reflect.TypeOf(engine.Item{})) if result.Start == 0 { result.PrevFrom = -1 } else { result.PrevFrom = (result.Start - 1) / pageSize * pageSize } result.NextFrom = result.Start + len(result.Items) return result, nil } // Rewrites query string. Replaces field names // like "Age" to "Payload.Age" func rewriteQueryString(q string) string { re := regexp.MustCompile(`([A-Z][a-z]*):`) return re.ReplaceAllString(q, "Payload.$1:") } ================================================ FILE: crawler/frontend/model/page.go ================================================ package model type SearchResult struct { Hits int64 Start int Query string PrevFrom int NextFrom int Items []interface{} } ================================================ FILE: crawler/frontend/starter.go ================================================ package main import ( "net/http" "imooc.com/ccmouse/learngo/crawler/frontend/controller" ) func main() { http.Handle("/", http.FileServer( http.Dir("crawler/frontend/view"))) http.Handle("/search", controller.CreateSearchResultHandler( "crawler/frontend/view/template.html")) err := http.ListenAndServe(":8888", nil) if err != nil { panic(err) } } ================================================ FILE: crawler/frontend/view/css/style.css ================================================ /* -- import Roboto Font ---------------------------- */ @import "https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic&subset=latin,cyrillic"; /* -- You can use this tables in Bootstrap (v3) projects. -- */ /* -- Box model ------------------------------- */ *, *:after, *:before { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; } /* -- Demo style ------------------------------- */ html, body { position: relative; min-height: 100%; height: 100%; } html { position: relative; overflow-x: hidden; margin: 16px; padding: 0; min-height: 100%; font-size: 62.5%; } body { font-family: 'RobotoDraft', 'Roboto', 'Helvetica Neue, Helvetica, Arial', sans-serif; font-style: normal; font-weight: 300; font-size: 1.4rem; line-height: 2rem; letter-spacing: 0.01rem; color: #212121; background-color: #f5f5f5; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; text-rendering: optimizeLegibility; } #demo { margin: 20px auto; max-width: 1200px; } #demo h1 { font-size: 2.4rem; line-height: 3.2rem; letter-spacing: 0; font-weight: 300; color: #212121; text-transform: inherit; margin-bottom: 1rem; text-align: center; } #demo h2 { font-size: 1.5rem; line-height: 2.8rem; letter-spacing: 0.01rem; font-weight: 400; color: #212121; text-align: center; } .shadow-z-1 { -webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); -moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); } /* -- Material Design Table style -------------- */ .table { width: 100%; max-width: 100%; margin-bottom: 2rem; background-color: #fff; } .table > thead > tr, .table > tbody > tr, .table > tfoot > tr { -webkit-transition: all 0.3s ease; -o-transition: all 0.3s ease; transition: all 0.3s ease; } .table > thead > tr > th, .table > tbody > tr > th, .table > tfoot > tr > th, .table > thead > tr > td, .table > tbody > tr > td, .table > tfoot > tr > td { text-align: left; padding: 1.6rem; vertical-align: top; border-top: 0; -webkit-transition: all 0.3s ease; -o-transition: all 0.3s ease; transition: all 0.3s ease; } .table > thead > tr > th { font-weight: 400; color: #757575; vertical-align: bottom; border-bottom: 1px solid rgba(0, 0, 0, 0.12); } .table > caption + thead > tr:first-child > th, .table > colgroup + thead > tr:first-child > th, .table > thead:first-child > tr:first-child > th, .table > caption + thead > tr:first-child > td, .table > colgroup + thead > tr:first-child > td, .table > thead:first-child > tr:first-child > td { border-top: 0; } .table > tbody + tbody { border-top: 1px solid rgba(0, 0, 0, 0.12); } .table .table { background-color: #fff; } .table .no-border { border: 0; } .table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td { padding: 0.8rem; } .table-bordered { border: 0; } .table-bordered > thead > tr > th, .table-bordered > tbody > tr > th, .table-bordered > tfoot > tr > th, .table-bordered > thead > tr > td, .table-bordered > tbody > tr > td, .table-bordered > tfoot > tr > td { border: 0; border-bottom: 1px solid #e0e0e0; } .table-bordered > thead > tr > th, .table-bordered > thead > tr > td { border-bottom-width: 2px; } .table-striped > tbody > tr:nth-child(odd) > td, .table-striped > tbody > tr:nth-child(odd) > th { background-color: #f5f5f5; } .table-hover > tbody > tr:hover > td, .table-hover > tbody > tr:hover > th { background-color: rgba(0, 0, 0, 0.12); } @media screen and (max-width: 768px) { .table-responsive-vertical > .table { margin-bottom: 0; background-color: transparent; } .table-responsive-vertical > .table > thead, .table-responsive-vertical > .table > tfoot { display: none; } .table-responsive-vertical > .table > tbody { display: block; } .table-responsive-vertical > .table > tbody > tr { display: block; border: 1px solid #e0e0e0; border-radius: 2px; margin-bottom: 1.6rem; } .table-responsive-vertical > .table > tbody > tr > td { background-color: #fff; display: block; vertical-align: middle; text-align: right; } .table-responsive-vertical > .table > tbody > tr > td[data-title]:before { content: attr(data-title); float: left; font-size: inherit; font-weight: 400; color: #757575; } .table-responsive-vertical.shadow-z-1 { -webkit-box-shadow: none; -moz-box-shadow: none; box-shadow: none; } .table-responsive-vertical.shadow-z-1 > .table > tbody > tr { border: none; -webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); -moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); } .table-responsive-vertical > .table-bordered { border: 0; } .table-responsive-vertical > .table-bordered > tbody > tr > td { border: 0; border-bottom: 1px solid #e0e0e0; } .table-responsive-vertical > .table-bordered > tbody > tr > td:last-child { border-bottom: 0; } .table-responsive-vertical > .table-striped > tbody > tr > td, .table-responsive-vertical > .table-striped > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical > .table-striped > tbody > tr > td:nth-child(odd) { background-color: #f5f5f5; } .table-responsive-vertical > .table-hover > tbody > tr:hover > td, .table-responsive-vertical > .table-hover > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical > .table-hover > tbody > tr > td:hover { background-color: rgba(0, 0, 0, 0.12); } } .table-striped.table-mc-red > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-red > tbody > tr:nth-child(odd) > th { background-color: #fde0dc; } .table-hover.table-mc-red > tbody > tr:hover > td, .table-hover.table-mc-red > tbody > tr:hover > th { background-color: #f9bdbb; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-red > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td:nth-child(odd) { background-color: #fde0dc; } .table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-red > tbody > tr > td:hover { background-color: #f9bdbb; } } .table-striped.table-mc-pink > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-pink > tbody > tr:nth-child(odd) > th { background-color: #fce4ec; } .table-hover.table-mc-pink > tbody > tr:hover > td, .table-hover.table-mc-pink > tbody > tr:hover > th { background-color: #f8bbd0; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td:nth-child(odd) { background-color: #fce4ec; } .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr > td:hover { background-color: #f8bbd0; } } .table-striped.table-mc-purple > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-purple > tbody > tr:nth-child(odd) > th { background-color: #f3e5f5; } .table-hover.table-mc-purple > tbody > tr:hover > td, .table-hover.table-mc-purple > tbody > tr:hover > th { background-color: #e1bee7; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td:nth-child(odd) { background-color: #f3e5f5; } .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr > td:hover { background-color: #e1bee7; } } .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > th { background-color: #ede7f6; } .table-hover.table-mc-deep-purple > tbody > tr:hover > td, .table-hover.table-mc-deep-purple > tbody > tr:hover > th { background-color: #d1c4e9; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td:nth-child(odd) { background-color: #ede7f6; } .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr > td:hover { background-color: #d1c4e9; } } .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > th { background-color: #e8eaf6; } .table-hover.table-mc-indigo > tbody > tr:hover > td, .table-hover.table-mc-indigo > tbody > tr:hover > th { background-color: #c5cae9; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td:nth-child(odd) { background-color: #e8eaf6; } .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr > td:hover { background-color: #c5cae9; } } .table-striped.table-mc-blue > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-blue > tbody > tr:nth-child(odd) > th { background-color: #e7e9fd; } .table-hover.table-mc-blue > tbody > tr:hover > td, .table-hover.table-mc-blue > tbody > tr:hover > th { background-color: #d0d9ff; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td:nth-child(odd) { background-color: #e7e9fd; } .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr > td:hover { background-color: #d0d9ff; } } .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > th { background-color: #e1f5fe; } .table-hover.table-mc-light-blue > tbody > tr:hover > td, .table-hover.table-mc-light-blue > tbody > tr:hover > th { background-color: #b3e5fc; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td:nth-child(odd) { background-color: #e1f5fe; } .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr > td:hover { background-color: #b3e5fc; } } .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > th { background-color: #e0f7fa; } .table-hover.table-mc-cyan > tbody > tr:hover > td, .table-hover.table-mc-cyan > tbody > tr:hover > th { background-color: #b2ebf2; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td:nth-child(odd) { background-color: #e0f7fa; } .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr > td:hover { background-color: #b2ebf2; } } .table-striped.table-mc-teal > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-teal > tbody > tr:nth-child(odd) > th { background-color: #e0f2f1; } .table-hover.table-mc-teal > tbody > tr:hover > td, .table-hover.table-mc-teal > tbody > tr:hover > th { background-color: #b2dfdb; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td:nth-child(odd) { background-color: #e0f2f1; } .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr > td:hover { background-color: #b2dfdb; } } .table-striped.table-mc-green > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-green > tbody > tr:nth-child(odd) > th { background-color: #d0f8ce; } .table-hover.table-mc-green > tbody > tr:hover > td, .table-hover.table-mc-green > tbody > tr:hover > th { background-color: #a3e9a4; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-green > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td:nth-child(odd) { background-color: #d0f8ce; } .table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-green > tbody > tr > td:hover { background-color: #a3e9a4; } } .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > th { background-color: #f1f8e9; } .table-hover.table-mc-light-green > tbody > tr:hover > td, .table-hover.table-mc-light-green > tbody > tr:hover > th { background-color: #dcedc8; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td:nth-child(odd) { background-color: #f1f8e9; } .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr > td:hover { background-color: #dcedc8; } } .table-striped.table-mc-lime > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-lime > tbody > tr:nth-child(odd) > th { background-color: #f9fbe7; } .table-hover.table-mc-lime > tbody > tr:hover > td, .table-hover.table-mc-lime > tbody > tr:hover > th { background-color: #f0f4c3; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td:nth-child(odd) { background-color: #f9fbe7; } .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr > td:hover { background-color: #f0f4c3; } } .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > th { background-color: #fffde7; } .table-hover.table-mc-yellow > tbody > tr:hover > td, .table-hover.table-mc-yellow > tbody > tr:hover > th { background-color: #fff9c4; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td:nth-child(odd) { background-color: #fffde7; } .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr > td:hover { background-color: #fff9c4; } } .table-striped.table-mc-amber > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-amber > tbody > tr:nth-child(odd) > th { background-color: #fff8e1; } .table-hover.table-mc-amber > tbody > tr:hover > td, .table-hover.table-mc-amber > tbody > tr:hover > th { background-color: #ffecb3; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td:nth-child(odd) { background-color: #fff8e1; } .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr > td:hover { background-color: #ffecb3; } } .table-striped.table-mc-orange > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-orange > tbody > tr:nth-child(odd) > th { background-color: #fff3e0; } .table-hover.table-mc-orange > tbody > tr:hover > td, .table-hover.table-mc-orange > tbody > tr:hover > th { background-color: #ffe0b2; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td:nth-child(odd) { background-color: #fff3e0; } .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr > td:hover { background-color: #ffe0b2; } } .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > td, .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > th { background-color: #fbe9e7; } .table-hover.table-mc-deep-orange > tbody > tr:hover > td, .table-hover.table-mc-deep-orange > tbody > tr:hover > th { background-color: #ffccbc; } @media screen and (max-width: 767px) { .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td, .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) { background-color: #fff; } .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td:nth-child(odd) { background-color: #fbe9e7; } .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover > td, .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover { background-color: #fff; } .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr > td:hover { background-color: #ffccbc; } } ================================================ FILE: crawler/frontend/view/index.html ================================================
================================================ FILE: crawler/frontend/view/js/index.js ================================================ /** * Created by Kupletsky Sergey on 05.11.14. * * Material Design Responsive Table * Tested on Win8.1 with browsers: Chrome 37, Firefox 32, Opera 25, IE 11, Safari 5.1.7 * You can use this table in Bootstrap (v3) projects. Material Design Responsive Table CSS-style will override basic bootstrap style. * JS used only for table constructor: you don't need it in your project */ $(document).ready(function() { var table = $('#table'); // Table bordered $('#table-bordered').change(function() { var value = $( this ).val(); table.removeClass('table-bordered').addClass(value); }); // Table striped $('#table-striped').change(function() { var value = $( this ).val(); table.removeClass('table-striped').addClass(value); }); // Table hover $('#table-hover').change(function() { var value = $( this ).val(); table.removeClass('table-hover').addClass(value); }); // Table color $('#table-color').change(function() { var value = $(this).val(); table.removeClass(/^table-mc-/).addClass(value); }); }); // jQuery’s hasClass and removeClass on steroids // by Nikita Vasilyev // https://github.com/NV/jquery-regexp-classes (function(removeClass) { jQuery.fn.removeClass = function( value ) { if ( value && typeof value.test === "function" ) { for ( var i = 0, l = this.length; i < l; i++ ) { var elem = this[i]; if ( elem.nodeType === 1 && elem.className ) { var classNames = elem.className.split( /\s+/ ); for ( var n = classNames.length; n--; ) { if ( value.test(classNames[n]) ) { classNames.splice(n, 1); } } elem.className = jQuery.trim( classNames.join(" ") ); } } } else { removeClass.call(this, value); } return this; } })(jQuery.fn.removeClass); ================================================ FILE: crawler/frontend/view/searchresult.go ================================================ package view import ( "html/template" "io" "imooc.com/ccmouse/learngo/crawler/frontend/model" ) type SearchResultView struct { template *template.Template } func CreateSearchResultView( filename string) SearchResultView { return SearchResultView{ template: template.Must( template.ParseFiles(filename)), } } func (s SearchResultView) Render( w io.Writer, data model.SearchResult) error { return s.template.Execute(w, data) } ================================================ FILE: crawler/frontend/view/searchresult_test.go ================================================ package view import ( "os" "testing" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/frontend/model" common "imooc.com/ccmouse/learngo/crawler/model" ) func TestSearchResultView_Render(t *testing.T) { view := CreateSearchResultView( "template.html") out, err := os.Create("template.test.html") if err != nil { panic(err) } defer out.Close() page := model.SearchResult{} page.Hits = 123 item := engine.Item{ Url: "http://album.zhenai.com/u/108906739", Type: "zhenai", Id: "108906739", Payload: common.Profile{ Age: 34, Height: 162, Weight: 57, Income: "3001-5000元", Gender: "女", Name: "安静的雪", Xinzuo: "牡羊座", Occupation: "人事/行政", Marriage: "离异", House: "已购房", Hokou: "山东菏泽", Education: "大学本科", Car: "未购车", }, } for i := 0; i < 10; i++ { page.Items = append(page.Items, item) } err = view.Render(out, page) if err != nil { t.Error(err) } // TODO: verify contents in template.test.html } ================================================ FILE: crawler/frontend/view/template.html ================================================

共为你找到相关结果约为{{.Hits}}个。显示从{{.Start}}起共{{len .Items}}个。

{{range .Items}} {{if eq .Type "xcar"}} {{with .Payload}} {{end}} {{end}} {{if eq .Type "zhenai"}} {{with .Payload}} {{end}} {{end}} {{else}}
没有找到相关用户
{{end}}
{{.Payload.Name}}{{.Name}} {{.Price}}万元 {{.Size}} {{.Fuel}}L/100km {{.Transmission}} {{.Engine}} {{.Displacement}}L {{.MaxSpeed}}km/h {{.Acceleration}}s,0-100km/h
{{.Payload.Name}}{{.Gender}} {{.Age}} {{.Height}}cm {{.Weight}}kg {{.Income}} {{.Education}} {{.Occupation}} {{.Hokou}} {{.Xinzuo}} {{.House}} {{.Car}}
{{if ge .PrevFrom 0}} 上一页 {{end}} 下一页
================================================ FILE: crawler/main.go ================================================ package main import ( "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/persist" "imooc.com/ccmouse/learngo/crawler/scheduler" "imooc.com/ccmouse/learngo/crawler/zhenai/parser" ) func main() { itemChan, err := persist.ItemSaver( config.ElasticIndex) if err != nil { panic(err) } e := engine.ConcurrentEngine{ Scheduler: &scheduler.QueuedScheduler{}, WorkerCount: 100, ItemChan: itemChan, RequestProcessor: engine.Worker, } e.Run(engine.Request{ Url: "http://localhost:8080/mock/www.zhenai.com/zhenghun", Parser: engine.NewFuncParser( parser.ParseCityList, config.ParseCityList), }) } ================================================ FILE: crawler/model/car.go ================================================ package model type Car struct { Name string Price float64 ImageURL string Size string Fuel float64 Transmission string Engine string Displacement float64 // 排量 MaxSpeed float64 Acceleration float64 } ================================================ FILE: crawler/model/profile.go ================================================ package model import "encoding/json" type Profile struct { Name string Gender string Age int Height int Weight int Income string Marriage string Education string Occupation string Hokou string Xinzuo string House string Car string } func FromJsonObj(o interface{}) (Profile, error) { var profile Profile s, err := json.Marshal(o) if err != nil { return profile, err } err = json.Unmarshal(s, &profile) return profile, err } ================================================ FILE: crawler/persist/itemsaver.go ================================================ package persist import ( "context" "errors" "log" "github.com/olivere/elastic/v7" "imooc.com/ccmouse/learngo/crawler/engine" ) func ItemSaver( index string) (chan engine.Item, error) { client, err := elastic.NewClient( // Must turn off sniff in docker elastic.SetSniff(false)) if err != nil { return nil, err } out := make(chan engine.Item) go func() { itemCount := 0 for { item := <-out log.Printf("Item Saver: got item "+ "#%d: %v", itemCount, item) itemCount++ err := Save(client, index, item) if err != nil { log.Printf("Item Saver: error "+ "saving item %v: %v", item, err) } } }() return out, nil } func Save( client *elastic.Client, index string, item engine.Item) error { if item.Type == "" { return errors.New("must supply Type") } indexService := client.Index(). Index(index). Type(item.Type). BodyJson(item) if item.Id != "" { indexService.Id(item.Id) } _, err := indexService. Do(context.Background()) return err } ================================================ FILE: crawler/persist/itemsaver_test.go ================================================ package persist import ( "context" "encoding/json" "testing" "github.com/olivere/elastic/v7" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" ) func TestSave(t *testing.T) { expected := engine.Item{ Url: "http://album.zhenai.com/u/108906739", Type: "zhenai", Id: "108906739", Payload: model.Profile{ Age: 34, Height: 162, Weight: 57, Income: "3001-5000元", Gender: "女", Name: "安静的雪", Xinzuo: "牡羊座", Occupation: "人事/行政", Marriage: "离异", House: "已购房", Hokou: "山东菏泽", Education: "大学本科", Car: "未购车", }, } // TODO: Try to start up elastic search // here using docker go client. client, err := elastic.NewClient( elastic.SetSniff(false)) if err != nil { panic(err) } const index = "dating_test" // Save expected item err = Save(client, index, expected) if err != nil { panic(err) } // Fetch saved item resp, err := client.Get(). Index(index). Type(expected.Type). Id(expected.Id). Do(context.Background()) if err != nil { panic(err) } t.Logf("%s", resp.Source) var actual engine.Item json.Unmarshal(resp.Source, &actual) actualProfile, _ := model.FromJsonObj( actual.Payload) actual.Payload = actualProfile // Verify result if actual != expected { t.Errorf("got %v; expected %v", actual, expected) } } ================================================ FILE: crawler/scheduler/queued.go ================================================ package scheduler import "imooc.com/ccmouse/learngo/crawler/engine" type QueuedScheduler struct { requestChan chan engine.Request workerChan chan chan engine.Request } func (s *QueuedScheduler) WorkerChan() chan engine.Request { return make(chan engine.Request) } func (s *QueuedScheduler) Submit(r engine.Request) { s.requestChan <- r } func (s *QueuedScheduler) WorkerReady( w chan engine.Request) { s.workerChan <- w } func (s *QueuedScheduler) Run() { s.workerChan = make(chan chan engine.Request) s.requestChan = make(chan engine.Request) go func() { var requestQ []engine.Request var workerQ []chan engine.Request for { var activeRequest engine.Request var activeWorker chan engine.Request if len(requestQ) > 0 && len(workerQ) > 0 { activeWorker = workerQ[0] activeRequest = requestQ[0] } select { case r := <-s.requestChan: requestQ = append(requestQ, r) case w := <-s.workerChan: workerQ = append(workerQ, w) case activeWorker <- activeRequest: workerQ = workerQ[1:] requestQ = requestQ[1:] } } }() } ================================================ FILE: crawler/scheduler/simple.go ================================================ package scheduler import "imooc.com/ccmouse/learngo/crawler/engine" type SimpleScheduler struct { workerChan chan engine.Request } func (s *SimpleScheduler) WorkerChan() chan engine.Request { return s.workerChan } func (s *SimpleScheduler) WorkerReady(chan engine.Request) { } func (s *SimpleScheduler) Run() { s.workerChan = make(chan engine.Request) } func (s *SimpleScheduler) Submit( r engine.Request) { go func() { s.workerChan <- r }() } ================================================ FILE: crawler/xcar/parser/cardetail.go ================================================ package parser import ( "fmt" "regexp" "strconv" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" ) var priceReTmpl = `(\d+\.\d+)` var nameRe = regexp.MustCompile(`【(.*)报价_图片_参数】.*`) var carImageRe = regexp.MustCompile(`(\d+[^\d]\d+[^\d]\d+mm)`) var fuelRe = regexp.MustCompile(`(\d+\.\d+)L/100km`) var transmissionRe = regexp.MustCompile(`(.+)`) var engineRe = regexp.MustCompile(`发\s*动\s*机.*\s*.*<.*>(\d+kW[^<]*)<`) var displacementRe = regexp.MustCompile(`]*>(\d+)`) var accelRe = regexp.MustCompile(`]*>([\d\.]+)`) var urlRe = regexp.MustCompile(`http://newcar.xcar.com.cn/(m\d+)/`) func ParseCarDetail(contents []byte, url string) engine.ParseResult { id := extractString([]byte(url), urlRe) car := model.Car{ Name: extractString(contents, nameRe), ImageURL: "http:" + extractString(contents, carImageRe), Size: extractString(contents, sizeRe), Fuel: extractFloat(contents, fuelRe), Transmission: extractString(contents, transmissionRe), Engine: extractString(contents, engineRe), Displacement: extractFloat(contents, displacementRe), MaxSpeed: extractFloat(contents, maxSpeedRe), Acceleration: extractFloat(contents, accelRe), } priceRe, err := regexp.Compile( fmt.Sprintf(priceReTmpl, regexp.QuoteMeta(id))) if err == nil { car.Price = extractFloat(contents, priceRe) } result := engine.ParseResult{ Items: []engine.Item{ { Id: id, Url: url, Type: "xcar", Payload: car, }, }, } carModelResult := ParseCarModel(contents, url) result.Requests = carModelResult.Requests return result } func extractString( contents []byte, re *regexp.Regexp) string { match := re.FindSubmatch(contents) if len(match) >= 2 { return string(match[1]) } else { return "" } } func extractFloat(contents []byte, re *regexp.Regexp) float64 { f, err := strconv.ParseFloat(extractString(contents, re), 64) if err != nil { return 0 } return f } ================================================ FILE: crawler/xcar/parser/cardetail_test.go ================================================ package parser import ( "io/ioutil" "testing" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" ) func TestParseCarDetail(t *testing.T) { contents, err := ioutil.ReadFile( "cardetail_test_data.html") if err != nil { panic(err) } expectedItem := engine.Item{ Url: "http://newcar.xcar.com.cn/m35001/", Type: "xcar", Id: "m35001", Payload: model.Car{ Name: "奥迪TT双门2017款45 TFSI", Price: 47.18, ImageURL: "http://img1.xcarimg.com/b63/s8386/m_20170616000036181753843373443.jpg-280x210.jpg", Size: "4191×1832×1353mm", Fuel: 16.7, Transmission: "6挡双离合", Engine: "169kW(2.0L涡轮增压)", Displacement: 2, MaxSpeed: 250, Acceleration: 5.9, }, } result := ParseCarDetail(contents, "http://newcar.xcar.com.cn/m35001/") if len(result.Items) != 1 { t.Errorf("Must only return one item, but %d returned", len(result.Items)) } actualItem := result.Items[0] if actualItem != expectedItem { t.Errorf("expected item: %+v, but was %+v", expectedItem, actualItem) } const resultSize = 8 expectedUrls := []string{ "http://newcar.xcar.com.cn/m45776/", "http://newcar.xcar.com.cn/m45776/", "http://newcar.xcar.com.cn/m32946/", } if len(result.Requests) != resultSize { t.Errorf("result should have %d "+ "requests; but had %d", resultSize, len(result.Requests)) } for i, url := range expectedUrls { if result.Requests[i].Url != url { t.Errorf("expected url #%d: %s; but "+ "was %s", i, url, result.Requests[i].Url) } } } ================================================ FILE: crawler/xcar/parser/cardetail_test_data.html ================================================ 【奥迪TT双门2017款45 TFSI报价_图片_参数】_奥迪(进口)奥迪TT双门怎么样_爱卡汽车
搜索
本地
参考价
厂商
指导价
47.18
  • 车身尺寸:4191×1832×1353mm
  • 工信部油耗:16.7L/100km
  • 变 速 箱:6挡双离合
  • 发 动 机:169kW(2.0L涡轮增压)
  • 驱动方式:前置前驱
  • 排  量:2.0L
基本参数
-无● 有○ 可选▲ 待查
级别 跑车 发动机 169kW(2.0L涡轮增压)
动力类型 汽油机 变速箱 6挡双离合
长×宽×高(mm) 4191×1832×1353 车身结构 2门 4座 跑车
上市年份 2017 最高车速(km/h) 250
0-100加速时间(s) 5.9 工信部油耗(L/100km) 6.7
车身参数
车长(mm) 4191 车宽(mm) 1832
车高(mm) 1353 轴距(mm) 2502
整备质量(kg) 1350 最小离地间隙(mm) -
前轮距(mm) 1572 后轮距(mm) 1552
车门数 2 座位数 4
油箱容积(L) 50 行李厢最小容积(L) 305

查看全部配置

本地经销商报价

切换城市

爱卡汽车提醒您:过低的报价在实际交易中可能存在附加条件

全款购车

裸车价格:471800元
  • + 购置税:40325 元
  • + 交强险:950 元
  • + 商业险:约13386 元
  • + 车船税: 480 元
  • + 上牌费: 500 元
参考成交价:约527,400元

注:此结果仅供参考,实际费用以当地缴费为准

奥迪还有什么车

更多
奥迪A6L
指导价: 40.98-65.08
关注度:
奥迪Q5L
指导价: 38.28-49.80
关注度:
奥迪A4L
指导价: 28.68-40.18
关注度:
奥迪Q8
指导价: 76.88-109.76
关注度:
奥迪Q3
指导价: 23.42-35.98
关注度:
车型对比
分享
================================================ FILE: crawler/xcar/parser/carlist.go ================================================ package parser import ( "regexp" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" ) const host = "http://newcar.xcar.com.cn" var carModelRe = regexp.MustCompile(``) var carListRe = regexp.MustCompile(` 【图】买什么车好_最新上市汽车车型推荐-爱卡汽车网

选车条件

条件重置
================================================ FILE: crawler/xcar/parser/carmodel.go ================================================ package parser import ( "regexp" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" ) var carDetailRe = regexp.MustCompile(` 【奥迪TT双门】新奥迪_奥迪TT双门报价|图片_2019奥迪TT双门怎么样_爱卡汽车
当前位置:爱卡汽车>跑车>奥迪TT双门>车系首页
搜索
厂商指导价:45.38-49.68
切换城市
降价信息
1.8T 132kW 关注度 指导价 本地参考价 相关信息

2017款 40 TFSI

前置前驱6挡双离合

45.38万

图片|配置对比

2.0T 169kW 关注度 指导价 本地参考价 相关信息

2017款 45 TFSI

前置前驱6挡双离合

47.18万

图片|配置对比

2017款 45 TFSI quattro

前置四驱6挡双离合

49.68万

图片|配置对比

关注该车系的还关注

718 54.50 万起

雷克萨斯RC 44.60 万起

科迈罗 39.99 万起

奔驰SLC 50.68 万起

Mustang 36.98 万起

坐进前排舒服吗?
适中 空间刚刚好
奥迪A6L
指导价:40.98-65.08
关注度:
奥迪Q5L
指导价:38.28-49.80
关注度:
奥迪A4L
指导价:28.68-40.18
关注度:
奥迪Q8
指导价:76.88-109.76
关注度:
奥迪Q3
指导价:23.42-35.98
关注度:
更 多
热门推荐
车型对比
分享
================================================ FILE: crawler/zhenai/parser/city.go ================================================ package parser import ( "regexp" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" ) var ( profileRe = regexp.MustCompile( `]*>([^<]+)`) cityUrlRe = regexp.MustCompile( `href="(.*www\.zhenai\.com/zhenghun/[^"]+)"`) ) func ParseCity( contents []byte, _ string) engine.ParseResult { matches := profileRe.FindAllSubmatch( contents, -1) result := engine.ParseResult{} for _, m := range matches { result.Requests = append( result.Requests, engine.Request{ Url: string(m[1]), Parser: NewProfileParser( string(m[2])), }) } matches = cityUrlRe.FindAllSubmatch( contents, -1) for _, m := range matches { result.Requests = append(result.Requests, engine.Request{ Url: string(m[1]), Parser: engine.NewFuncParser( ParseCity, config.ParseCity), }) } return result } ================================================ FILE: crawler/zhenai/parser/citylist.go ================================================ package parser import ( "regexp" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" ) const cityListRe = `]*>([^<]+)` func ParseCityList( contents []byte, _ string) engine.ParseResult { re := regexp.MustCompile(cityListRe) matches := re.FindAllSubmatch(contents, -1) result := engine.ParseResult{} for _, m := range matches { result.Requests = append( result.Requests, engine.Request{ Url: string(m[1]), Parser: engine.NewFuncParser( ParseCity, config.ParseCity), }) } return result } ================================================ FILE: crawler/zhenai/parser/citylist_test.go ================================================ package parser import ( "testing" "io/ioutil" ) func TestParseCityList(t *testing.T) { contents, err := ioutil.ReadFile( "citylist_test_data.html") if err != nil { panic(err) } result := ParseCityList(contents, "") const resultSize = 470 expectedUrls := []string{ "http://www.zhenai.com/zhenghun/aba", "http://www.zhenai.com/zhenghun/akesu", "http://www.zhenai.com/zhenghun/alashanmeng", } if len(result.Requests) != resultSize { t.Errorf("result should have %d "+ "requests; but had %d", resultSize, len(result.Requests)) } for i, url := range expectedUrls { if result.Requests[i].Url != url { t.Errorf("expected url #%d: %s; but "+ "was %s", i, url, result.Requests[i].Url) } } } ================================================ FILE: crawler/zhenai/parser/citylist_test_data.html ================================================ 珍爱征婚网_征婚交友信息_同城征婚网站

相亲无难事,珍爱有红娘

A
阿坝 阿克苏 阿拉善盟 阿勒泰 阿里 安徽 安康 安庆 鞍山 安顺 安阳 澳门
B
白城 百色 白山 白银 巴南 蚌埠 宝坻 保定 宝鸡 宝山 保山 包头 巴彦淖尔 巴音郭楞 巴中 北碚 北辰 北海 北京 本溪 毕节 滨海新 滨州 璧山 博尔塔拉 亳州
C
沧州 长春 常德 昌都 昌吉 长宁 昌平 长沙 长寿 长治 常州 朝阳 潮州 承德 成都 城口 郴州 赤峰 池州 崇明 重庆 崇左 楚雄 滁州
D
大渡口 大理 大连 丹东 大庆 大同 大兴 大兴安岭 达州 大足 德宏 德阳 德州 垫江 定西 迪庆 东城 东莞 东丽 东营
E
鄂尔多斯 恩施 鄂州
F
防城港 房山 丰都 奉节 丰台 奉贤 佛山 福建 涪陵 抚顺 阜新 阜阳 福州 抚州
G
甘南 甘肃 赣州 甘孜 高雄 广安 广东 广西 广元 广州 贵港 桂林 贵阳 贵州 果洛 固原
H
哈尔滨 海北 海淀 海东地 海口 海南 海南藏族自治州 海西 哈密 邯郸 杭州 汉中 河北 鹤壁 河池 合川 河东 合肥 鹤岗 黑河 黑龙江 河南 衡水 衡阳 和平 和田 河西 河源 菏泽 贺州 红河 虹口 红桥 淮安 淮北 怀化 淮南 怀柔 花莲 黄冈 黄南 黄浦 黄山 黄石 湖北 呼和浩特 惠州 葫芦岛 呼伦贝尔 湖南 湖州
J
嘉定 佳木斯 吉安 江北 江津 江门 江苏 江西 焦作 嘉兴 嘉义 嘉峪关 揭阳 吉林 基隆 济南 金昌 晋城 静安 景德镇 静海 荆门 荆州 金华 济宁 津南 金山 晋中 锦州 九江 九龙 九龙坡 酒泉 鸡西 蓟县
K
开封 开县 喀什 克拉玛依 克孜勒苏柯尔克孜 昆明
L
来宾 莱芜 廊坊 兰州 拉萨 乐山 梁平 凉山 连云港 聊城 辽宁 辽阳 辽源 丽江 临沧 临汾 临夏 临沂 林芝 丽水 六安 六盘水 柳州 陇南 龙岩 娄底 漯河 洛阳 泸州 吕梁
M
马鞍山 茂名 眉山 梅州 门头沟 绵阳 苗栗 闵行 密云 牡丹江
N
南岸 南昌 南充 南川 南京 南开 南宁 南平 南通 南投 南阳 那曲 内江 内蒙古 宁波 宁德 宁河 宁夏 怒江
P
盘锦 攀枝花 澎湖 彭水 平顶山 屏东 平谷 平凉 萍乡 浦东新 普洱 莆田 普陀 濮阳
Q
黔东 黔江 黔南 黔西 綦江 青岛 青海 青浦 庆阳 清远 秦皇岛 钦州 齐齐哈尔 七台河 泉州 曲靖 衢州
R
日喀则 日照 荣昌
S
三门峡 三明 三亚 山东 上海 商洛 商丘 上饶 山南 汕头 汕尾 山西 陕西 韶关 绍兴 邵阳 沙坪坝 沈阳 深圳 石家庄 石景山 十堰 石柱 石嘴山 双鸭山 顺义 朔州 四川 四平 松江 松原 绥化 遂宁 随州 宿迁 苏州 宿州
T
塔城 泰安 台北 台东 台南 台湾 太原 台中 泰州 台州 唐山 桃园 天津 天水 铁岭 铜川 通化 铜梁 通辽 铜陵 潼南 铜仁 通州 吐鲁番
W
万州 潍坊 威海 渭南 文山 温州 乌海 武汉 芜湖 乌兰察布 武隆 乌鲁木齐 武清 巫山 武威 无锡 巫溪 吴忠 梧州
X
厦门 西安 香港 湘潭 湘西 襄阳 咸宁 咸阳 孝感 西城 锡林郭勒盟 新北 兴安 邢台 西宁 新疆 新界 新乡 信阳 新余 忻州 新竹 西青 西双版纳 秀山 西藏 宣城 许昌 徐汇 徐州
Y
雅安 延安 延边 盐城 阳江 杨浦 阳泉 扬州 延庆 烟台 宜宾 宜昌 宜春 伊春 宜兰 伊犁 银川 营口 鹰潭 益阳 永川 永州 酉阳 渝北 岳阳 榆林 玉林 运城 云浮 云林 云南 云阳 玉树 玉溪 渝中
Z
枣庄 闸北 彰化 张家界 张家口 张掖 漳州 湛江 肇庆 昭通 浙江 郑州 镇江 中山 中卫 忠县 周口 舟山 珠海 驻马店 株洲 淄博 自贡 资阳 遵义
> ================================================ FILE: crawler/zhenai/parser/profile.go ================================================ package parser import ( "regexp" "strconv" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" ) var ageRe = regexp.MustCompile( `年龄:(\d+)岁`) var heightRe = regexp.MustCompile( `身高:(\d+)CM`) var incomeRe = regexp.MustCompile( `月收入:([^<]+)`) var weightRe = regexp.MustCompile( `体重:(\d+)KG`) var genderRe = regexp.MustCompile( `性别:([^<]+)`) var xinzuoRe = regexp.MustCompile( `星座:([^<]+)`) var marriageRe = regexp.MustCompile( `婚况:([^<]+)`) var educationRe = regexp.MustCompile( `学历:([^<]+)`) var occupationRe = regexp.MustCompile( `职业:([^<]+)`) var hokouRe = regexp.MustCompile( `籍贯:([^<]+)`) var houseRe = regexp.MustCompile( `住房条件:([^<]+)`) var carRe = regexp.MustCompile( `是否购车:([^<]+)`) var guessRe = regexp.MustCompile( `]*href="(.*album\.zhenai\.com/u/[\d]+)">([^<]+)`) var idUrlRe = regexp.MustCompile( `.*album\.zhenai\.com/u/([\d]+)`) func parseProfile( contents []byte, url string, name string) engine.ParseResult { profile := model.Profile{} profile.Name = name age, err := strconv.Atoi( extractString(contents, ageRe)) if err == nil { profile.Age = age } height, err := strconv.Atoi( extractString(contents, heightRe)) if err == nil { profile.Height = height } weight, err := strconv.Atoi( extractString(contents, weightRe)) if err == nil { profile.Weight = weight } profile.Income = extractString( contents, incomeRe) profile.Gender = extractString( contents, genderRe) profile.Car = extractString( contents, carRe) profile.Education = extractString( contents, educationRe) profile.Hokou = extractString( contents, hokouRe) profile.House = extractString( contents, houseRe) profile.Marriage = extractString( contents, marriageRe) profile.Occupation = extractString( contents, occupationRe) profile.Xinzuo = extractString( contents, xinzuoRe) result := engine.ParseResult{ Items: []engine.Item{ { Url: url, Type: "zhenai", Id: extractString( []byte(url), idUrlRe), Payload: profile, }, }, } matches := guessRe.FindAllSubmatch( contents, -1) for _, m := range matches { result.Requests = append(result.Requests, engine.Request{ Url: string(m[1]), Parser: NewProfileParser( string(m[2])), }) } return result } func extractString( contents []byte, re *regexp.Regexp) string { match := re.FindSubmatch(contents) if len(match) >= 2 { return string(match[1]) } else { return "" } } type ProfileParser struct { userName string } func (p *ProfileParser) Parse( contents []byte, url string) engine.ParseResult { return parseProfile(contents, url, p.userName) } func (p *ProfileParser) Serialize() ( name string, args interface{}) { return config.ParseProfile, p.userName } func NewProfileParser( name string) *ProfileParser { return &ProfileParser{ userName: name, } } ================================================ FILE: crawler/zhenai/parser/profile_test.go ================================================ package parser import ( "io/ioutil" "testing" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" ) func TestParseProfile(t *testing.T) { contents, err := ioutil.ReadFile( "profile_test_data.html") if err != nil { panic(err) } result := parseProfile(contents, "http://album.zhenai.com/u/108906739", "安静的雪") if len(result.Items) != 1 { t.Errorf("Items should contain 1 "+ "element; but was %v", result.Items) } actual := result.Items[0] expected := engine.Item{ Url: "http://album.zhenai.com/u/108906739", Type: "zhenai", Id: "108906739", Payload: model.Profile{ Age: 34, Height: 162, Weight: 57, Income: "3001-5000元", Gender: "女", Name: "安静的雪", Xinzuo: "牡羊座", Occupation: "人事/行政", Marriage: "离异", House: "已购房", Hokou: "山东菏泽", Education: "大学本科", Car: "未购车", }, } if actual != expected { t.Errorf("expected %v; but was %v", expected, actual) } } ================================================ FILE: crawler/zhenai/parser/profile_test_data.html ================================================ 安静的雪资料照片_新疆阿勒泰征婚交友_珍爱网

相亲无难事,珍爱有红娘

安静的雪照片

安静的雪

34岁,新疆阿勒泰,162cm,3001-5000元

发邮件 打招呼 问问题 委托红娘
安静的雪照片

安静的雪 698分

ID:108906739 诚信值:15

年龄:34岁 身高:162CM 月收入:3001-5000元
婚况:离异 学历:大学本科 工作地:新疆阿勒泰
职业: 人事/行政 有无孩子:有,我们住在一起 籍贯:山东菏泽

不完美又何妨?万物皆有裂隙,那是光进来的地方。随缘

想要听听TA对自己的想法?来邀请TA补充自我描述

详细资料

性别: 生肖:--
身高:162CM 星座:牡羊座
体重:57KG 血型:--
体型:-- 职业:人事/行政
民族:汉族 公司:--
信仰:--

生活状况

住房条件:已购房 想何时结婚:--
是否购车:未购车 婚后与父母住吗:--
是否吸烟:不吸烟 与对方父母同住:--
是否喝酒:不喝酒 较大的消费:--
厨艺:-- 喜欢怎样的约会:--
家务:--

兴趣爱好

喜欢的活动:-- 喜欢的食物:--
喜欢的体育运动:-- 喜欢的地方:--
喜欢的音乐:-- 喜欢的宠物:--
喜欢的影视节目:--

择偶条件

性别: 体型:不限
年龄:35 - 43岁 职业:不限
身高:173 - 185厘米 是否抽烟:不吸烟
学历:不限 是否喝酒:不限
月收入:5000元以上 有没有孩子:不限
婚况: 不限 是否想要孩子:不限
工作地区:新疆阿勒泰 是否有照片:不限
================================================ FILE: crawler_distributed/config/config.go ================================================ package config const ( ItemSaverRpc = "ItemSaverService.Save" CrawlServiceRpc = "CrawlService.Process" ) ================================================ FILE: crawler_distributed/main.go ================================================ package main import ( "errors" "net/rpc" "log" "flag" "strings" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/scheduler" "imooc.com/ccmouse/learngo/crawler/zhenai/parser" itemsaver "imooc.com/ccmouse/learngo/crawler_distributed/persist/client" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" worker "imooc.com/ccmouse/learngo/crawler_distributed/worker/client" ) var ( itemSaverHost = flag.String( "itemsaver_host", "", "itemsaver host") workerHosts = flag.String( "worker_hosts", "", "worker hosts (comma separated)") ) func main() { flag.Parse() itemChan, err := itemsaver.ItemSaver( *itemSaverHost) if err != nil { panic(err) } pool, err := createClientPool( strings.Split(*workerHosts, ",")) if err != nil { panic(err) } processor := worker.CreateProcessor(pool) e := engine.ConcurrentEngine{ Scheduler: &scheduler.QueuedScheduler{}, WorkerCount: 100, ItemChan: itemChan, RequestProcessor: processor, } e.Run(engine.Request{ Url: "http://localhost:8080/mock/www.zhenai.com/zhenghun", Parser: engine.NewFuncParser( parser.ParseCityList, config.ParseCityList), }) } func createClientPool( hosts []string) (chan *rpc.Client, error) { var clients []*rpc.Client for _, h := range hosts { client, err := rpcsupport.NewClient(h) if err == nil { clients = append(clients, client) log.Printf("Connected to %s", h) } else { log.Printf( "Error connecting to %s: %v", h, err) } } if len(clients) == 0 { return nil, errors.New( "no connections available") } out := make(chan *rpc.Client) go func() { for { for _, client := range clients { out <- client } } }() return out, nil } ================================================ FILE: crawler_distributed/persist/client/itemsaver.go ================================================ package client import ( "log" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler_distributed/config" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" ) func ItemSaver( host string) (chan engine.Item, error) { client, err := rpcsupport.NewClient(host) if err != nil { return nil, err } out := make(chan engine.Item) go func() { itemCount := 0 for { item := <-out log.Printf("Item Saver: got item "+ "#%d: %v", itemCount, item) itemCount++ // Call RPC to save item result := "" err := client.Call( config.ItemSaverRpc, item, &result) if err != nil { log.Printf("Item Saver: error "+ "saving item %v: %v", item, err) } } }() return out, nil } ================================================ FILE: crawler_distributed/persist/rpc.go ================================================ package persist import ( "log" "github.com/olivere/elastic/v7" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/persist" ) type ItemSaverService struct { Client *elastic.Client Index string } func (s *ItemSaverService) Save( item engine.Item, result *string) error { err := persist.Save(s.Client, s.Index, item) log.Printf("Item %v saved.", item) if err == nil { *result = "ok" } else { log.Printf("Error saving item %v: %v", item, err) } return err } ================================================ FILE: crawler_distributed/persist/server/client_test.go ================================================ package main import ( "testing" "time" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" "imooc.com/ccmouse/learngo/crawler_distributed/config" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" ) func TestItemSaver(t *testing.T) { const host = ":1234" // start ItemSaverServer go serveRpc(host, "test1") time.Sleep(time.Second) // start ItemSaverClient client, err := rpcsupport.NewClient(host) if err != nil { panic(err) } // Call save item := engine.Item{ Url: "http://album.zhenai.com/u/108906739", Type: "zhenai", Id: "108906739", Payload: model.Profile{ Age: 34, Height: 162, Weight: 57, Income: "3001-5000元", Gender: "女", Name: "安静的雪", Xinzuo: "牡羊座", Occupation: "人事/行政", Marriage: "离异", House: "已购房", Hokou: "山东菏泽", Education: "大学本科", Car: "未购车", }, } result := "" err = client.Call(config.ItemSaverRpc, item, &result) if err != nil || result != "ok" { t.Errorf("result: %s; err: %s", result, err) } } ================================================ FILE: crawler_distributed/persist/server/itemsaver.go ================================================ package main import ( "flag" "fmt" "log" "github.com/olivere/elastic/v7" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler_distributed/persist" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" ) var port = flag.Int("port", 0, "the port for me to listen on") func main() { flag.Parse() if *port == 0 { fmt.Println("must specify a port") return } log.Fatal(serveRpc( fmt.Sprintf(":%d", *port), config.ElasticIndex)) } func serveRpc(host, index string) error { client, err := elastic.NewClient( elastic.SetSniff(false)) if err != nil { return err } return rpcsupport.ServeRpc(host, &persist.ItemSaverService{ Client: client, Index: index, }) } ================================================ FILE: crawler_distributed/rpcsupport/rpc.go ================================================ package rpcsupport import ( "log" "net" "net/rpc" "net/rpc/jsonrpc" ) func ServeRpc( host string, service interface{}) error { rpc.Register(service) listener, err := net.Listen("tcp", host) if err != nil { return err } log.Printf("Listening on %s", host) for { conn, err := listener.Accept() if err != nil { log.Printf("accept error: %v", err) continue } go jsonrpc.ServeConn(conn) } } func NewClient(host string) (*rpc.Client, error) { conn, err := net.Dial("tcp", host) if err != nil { return nil, err } return jsonrpc.NewClient(conn), nil } ================================================ FILE: crawler_distributed/worker/client/worker.go ================================================ package client import ( "net/rpc" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler_distributed/config" "imooc.com/ccmouse/learngo/crawler_distributed/worker" ) func CreateProcessor( clientChan chan *rpc.Client) engine.Processor { return func( req engine.Request) ( engine.ParseResult, error) { sReq := worker.SerializeRequest(req) var sResult worker.ParseResult c := <-clientChan err := c.Call(config.CrawlServiceRpc, sReq, &sResult) if err != nil { return engine.ParseResult{}, err } return worker.DeserializeResult(sResult), nil } } ================================================ FILE: crawler_distributed/worker/rpc.go ================================================ package worker import "imooc.com/ccmouse/learngo/crawler/engine" type CrawlService struct{} func (CrawlService) Process( req Request, result *ParseResult) error { engineReq, err := DeserializeRequest(req) if err != nil { return err } engineResult, err := engine.Worker(engineReq) if err != nil { return err } *result = SerializeResult(engineResult) return nil } ================================================ FILE: crawler_distributed/worker/server/client_test.go ================================================ package main import ( "fmt" "testing" "time" "imooc.com/ccmouse/learngo/crawler/config" rpcnames "imooc.com/ccmouse/learngo/crawler_distributed/config" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" "imooc.com/ccmouse/learngo/crawler_distributed/worker" ) func TestCrawlService(t *testing.T) { const host = ":9000" go rpcsupport.ServeRpc( host, worker.CrawlService{}) time.Sleep(time.Second) client, err := rpcsupport.NewClient(host) if err != nil { panic(err) } // TODO: Use a fake fetcher to handle the url. // So we don't get data from zhenai.com req := worker.Request{ Url: "http://album.zhenai.com/u/108906739", Parser: worker.SerializedParser{ Name: config.ParseProfile, Args: "安静的雪", }, } var result worker.ParseResult err = client.Call( rpcnames.CrawlServiceRpc, req, &result) if err != nil { t.Error(err) } else { fmt.Println(result) } // TODO: Verify results } ================================================ FILE: crawler_distributed/worker/server/worker.go ================================================ package main import ( "fmt" "log" "flag" "imooc.com/ccmouse/learngo/crawler/fetcher" "imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport" "imooc.com/ccmouse/learngo/crawler_distributed/worker" ) var port = flag.Int("port", 0, "the port for me to listen on") func main() { flag.Parse() fetcher.SetVerboseLogging() if *port == 0 { fmt.Println("must specify a port") return } log.Fatal(rpcsupport.ServeRpc( fmt.Sprintf(":%d", *port), worker.CrawlService{})) } ================================================ FILE: crawler_distributed/worker/types.go ================================================ package worker import ( "errors" "fmt" "log" "imooc.com/ccmouse/learngo/crawler/config" "imooc.com/ccmouse/learngo/crawler/engine" xcar "imooc.com/ccmouse/learngo/crawler/xcar/parser" zhenai "imooc.com/ccmouse/learngo/crawler/zhenai/parser" ) type SerializedParser struct { Name string Args interface{} } type Request struct { Url string Parser SerializedParser } type ParseResult struct { Items []engine.Item Requests []Request } func SerializeRequest(r engine.Request) Request { name, args := r.Parser.Serialize() return Request{ Url: r.Url, Parser: SerializedParser{ Name: name, Args: args, }, } } func SerializeResult( r engine.ParseResult) ParseResult { result := ParseResult{ Items: r.Items, } for _, req := range r.Requests { result.Requests = append(result.Requests, SerializeRequest(req)) } return result } func DeserializeRequest( r Request) (engine.Request, error) { parser, err := deserializeParser(r.Parser) if err != nil { return engine.Request{}, err } return engine.Request{ Url: r.Url, Parser: parser, }, nil } func DeserializeResult( r ParseResult) engine.ParseResult { result := engine.ParseResult{ Items: r.Items, } for _, req := range r.Requests { engineReq, err := DeserializeRequest(req) if err != nil { log.Printf("error deserializing "+ "request: %v", err) continue } result.Requests = append(result.Requests, engineReq) } return result } func deserializeParser( p SerializedParser) (engine.Parser, error) { switch p.Name { case config.ParseCityList: return engine.NewFuncParser( zhenai.ParseCityList, config.ParseCityList), nil case config.ParseCity: return engine.NewFuncParser( zhenai.ParseCity, config.ParseCity), nil case config.ParseProfile: if userName, ok := p.Args.(string); ok { return zhenai.NewProfileParser( userName), nil } else { return nil, fmt.Errorf("invalid "+ "arg: %v", p.Args) } case config.ParseCarDetail: return engine.NewFuncParser( xcar.ParseCarDetail, config.ParseCarDetail), nil case config.ParseCarModel: return engine.NewFuncParser( xcar.ParseCarModel, config.ParseCarModel), nil case config.ParseCarList: return engine.NewFuncParser( xcar.ParseCarList, config.ParseCarList), nil case config.NilParser: return engine.NilParser{}, nil default: return nil, errors.New( "unknown parser name") } } ================================================ FILE: go.mod ================================================ module imooc.com/ccmouse/learngo go 1.13 require ( github.com/gin-gonic/gin v1.9.1 github.com/google/go-cmp v0.6.0 github.com/olivere/elastic/v7 v7.0.8 go.uber.org/zap v1.12.0 golang.org/x/net v0.38.0 golang.org/x/text v0.23.0 ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aws/aws-sdk-go v1.19.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olivere/elastic v6.2.23+incompatible h1:oRGUA/8fKcnkDcqLuwGb5YCzgbgEBo+Y9gamsWqZ0qU= github.com/olivere/elastic v6.2.23+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/olivere/elastic/v7 v7.0.8 h1:tp9BHGFilpoH7O7fQOwWiXJQFJkl9PZJvUwO74OvfKc= github.com/olivere/elastic/v7 v7.0.8/go.mod h1:UcXCjbh5xfX9uMB1VCcIYgGJBItbd4uRBdYRsBnnXHo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= ================================================ FILE: lang/basic/atomic/atomic.go ================================================ package main import ( "fmt" "sync" "time" ) type atomicInt struct { value int lock sync.Mutex } func (a *atomicInt) increment() { fmt.Println("safe increment") func() { a.lock.Lock() defer a.lock.Unlock() a.value++ }() } func (a *atomicInt) get() int { a.lock.Lock() defer a.lock.Unlock() return a.value } func main() { var a atomicInt a.increment() go func() { a.increment() }() time.Sleep(time.Millisecond) fmt.Println(a.get()) } ================================================ FILE: lang/basic/basic/basic.go ================================================ package main import ( "fmt" "math" "math/cmplx" ) var ( aa = 3 ss = "kkk" bb = true ) func variableZeroValue() { var a int var s string fmt.Printf("%d %q\n", a, s) } func variableInitialValue() { var a, b int = 3, 4 var s string = "abc" fmt.Println(a, b, s) } func variableTypeDeduction() { var a, b, c, s = 3, 4, true, "def" fmt.Println(a, b, c, s) } func variableShorter() { a, b, c, s := 3, 4, true, "def" b = 5 fmt.Println(a, b, c, s) } func euler() { fmt.Printf("%.3f\n", cmplx.Exp(1i*math.Pi)+1) } func triangle() { var a, b int = 3, 4 fmt.Println(calcTriangle(a, b)) } func calcTriangle(a, b int) int { var c int c = int(math.Sqrt(float64(a*a + b*b))) return c } func consts() { const ( filename = "abc.txt" a, b = 3, 4 ) var c int c = int(math.Sqrt(a*a + b*b)) fmt.Println(filename, c) } func enums() { const ( cpp = iota _ python golang javascript ) const ( b = 1 << (10 * iota) kb mb gb tb pb ) fmt.Println(cpp, javascript, python, golang) fmt.Println(b, kb, mb, gb, tb, pb) } func main() { fmt.Println("Hello world") variableZeroValue() variableInitialValue() variableTypeDeduction() variableShorter() fmt.Println(aa, ss, bb) euler() triangle() consts() enums() } ================================================ FILE: lang/basic/basic/triangle_test.go ================================================ package main import "testing" func TestTriangle(t *testing.T) { tests := []struct{ a, b, c int }{ {3, 4, 5}, {5, 12, 13}, {8, 15, 17}, {12, 35, 37}, {30000, 40000, 50000}, } for _, tt := range tests { if actual := calcTriangle(tt.a, tt.b); actual != tt.c { t.Errorf("calcTriangle(%d, %d); "+ "got %d; expected %d", tt.a, tt.b, actual, tt.c) } } } ================================================ FILE: lang/basic/branch/abc.txt ================================================ abcde 12345 hello again ================================================ FILE: lang/basic/branch/branch.go ================================================ package main import ( "fmt" "io/ioutil" ) func grade(score int) string { g := "" switch { case score < 0 || score > 100: panic(fmt.Sprintf( "Wrong score: %d", score)) case score < 60: g = "F" case score < 80: g = "C" case score < 90: g = "B" case score <= 100: g = "A" } return g } func main() { // If "abc.txt" is not found, // please check what current directory is, // and change filename accordingly. const filename = "abc.txt" if contents, err := ioutil.ReadFile(filename); err != nil { fmt.Println(err) } else { fmt.Printf("%s\n", contents) } fmt.Println( grade(0), grade(59), grade(60), grade(82), grade(99), grade(100), // Uncomment to see it panics. // grade(-3), ) } ================================================ FILE: lang/basic/func/func.go ================================================ package main import ( "fmt" "math" "reflect" "runtime" ) func eval(a, b int, op string) (int, error) { switch op { case "+": return a + b, nil case "-": return a - b, nil case "*": return a * b, nil case "/": q, _ := div(a, b) return q, nil default: return 0, fmt.Errorf( "unsupported operation: %s", op) } } func div(a, b int) (q, r int) { return a / b, a % b } func apply(op func(int, int) int, a, b int) int { p := reflect.ValueOf(op).Pointer() opName := runtime.FuncForPC(p).Name() fmt.Printf("Calling function %s with args "+ "(%d, %d)\n", opName, a, b) return op(a, b) } func sum(numbers ...int) int { s := 0 for i := range numbers { s += numbers[i] } return s } func swap(a, b int) (int, int) { return b, a } func main() { fmt.Println("Error handling") if result, err := eval(3, 4, "x"); err != nil { fmt.Println("Error:", err) } else { fmt.Println(result) } q, r := div(13, 3) fmt.Printf("13 div 3 is %d mod %d\n", q, r) fmt.Println("pow(3, 4) is:", apply( func(a int, b int) int { return int(math.Pow( float64(a), float64(b))) }, 3, 4)) fmt.Println("1+2+...+5 =", sum(1, 2, 3, 4, 5)) a, b := 3, 4 a, b = swap(a, b) fmt.Println("a, b after swap is:", a, b) } ================================================ FILE: lang/basic/loop/loop.go ================================================ package main import ( "bufio" "fmt" "io" "os" "strconv" "strings" ) func convertToBin(n int) string { result := "" for ; n > 0; n /= 2 { lsb := n % 2 result = strconv.Itoa(lsb) + result } return result } func printFile(filename string) { file, err := os.Open(filename) if err != nil { panic(err) } printFileContents(file) } func printFileContents(reader io.Reader) { scanner := bufio.NewScanner(reader) for scanner.Scan() { fmt.Println(scanner.Text()) } } func forever() { for { fmt.Println("abc") } } func main() { fmt.Println("convertToBin results:") fmt.Println( convertToBin(5), // 101 convertToBin(13), // 1101 convertToBin(72387885), convertToBin(0), ) fmt.Println("abc.txt contents:") printFile("lang/basic/branch/abc.txt") fmt.Println("printing a string:") s := `abc"d" kkkk 123 p` printFileContents(strings.NewReader(s)) // Uncomment to see it runs forever // forever() } ================================================ FILE: lang/basic/regex/regex.go ================================================ package main import ( "fmt" "regexp" ) const text = ` my email is ccmouse@gmail.com@abc.com email1 is abc@def.org email2 is kkk@qq.com email3 is ddd@abc.com.cn ` func main() { re := regexp.MustCompile( `([a-zA-Z0-9]+)@([a-zA-Z0-9]+)(\.[a-zA-Z0-9.]+)`) match := re.FindAllStringSubmatch(text, -1) for _, m := range match { fmt.Println(m) } } ================================================ FILE: lang/channel/channel.go ================================================ package main import ( "fmt" "time" ) func worker(id int, c chan int) { for n := range c { fmt.Printf("Worker %d received %c\n", id, n) } } func createWorker(id int) chan<- int { c := make(chan int) go worker(id, c) return c } func chanDemo() { var channels [10]chan<- int for i := 0; i < 10; i++ { channels[i] = createWorker(i) } for i := 0; i < 10; i++ { channels[i] <- 'a' + i } for i := 0; i < 10; i++ { channels[i] <- 'A' + i } time.Sleep(time.Millisecond) } func bufferedChannel() { c := make(chan int, 3) go worker(0, c) c <- 'a' c <- 'b' c <- 'c' c <- 'd' time.Sleep(time.Millisecond) } func channelClose() { c := make(chan int) go worker(0, c) c <- 'a' c <- 'b' c <- 'c' c <- 'd' close(c) time.Sleep(time.Millisecond) } func main() { fmt.Println("Channel as first-class citizen") chanDemo() fmt.Println("Buffered channel") bufferedChannel() fmt.Println("Channel close and range") channelClose() } ================================================ FILE: lang/channel/done/done.go ================================================ package main import ( "fmt" "sync" ) func doWork(id int, w worker) { for n := range w.in { fmt.Printf("Worker %d received %c\n", id, n) w.done() } } type worker struct { in chan int done func() } func createWorker( id int, wg *sync.WaitGroup) worker { w := worker{ in: make(chan int), done: func() { wg.Done() }, } go doWork(id, w) return w } func chanDemo() { var wg sync.WaitGroup var workers [10]worker for i := 0; i < 10; i++ { workers[i] = createWorker(i, &wg) } wg.Add(20) for i, worker := range workers { worker.in <- 'a' + i } for i, worker := range workers { worker.in <- 'A' + i } wg.Wait() } func main() { chanDemo() } ================================================ FILE: lang/channel/pattern/main.go ================================================ package main import ( "fmt" "math/rand" "time" ) func msgGen(name string) chan string { c := make(chan string) go func() { i := 0 for { time.Sleep(time.Duration(rand.Intn(2000)) * time.Millisecond) c <- fmt.Sprintf("service %s: message %d", name, i) i++ } }() return c } func fanIn(chs ...chan string) chan string { c := make(chan string) for _, ch := range chs { go func(in chan string) { for { c <- <-in } }(ch) } return c } func fanInBySelect(c1, c2 chan string) chan string { c := make(chan string) go func() { for { select { case m := <-c1: c <- m case m := <-c2: c <- m } } }() return c } func main() { m1 := msgGen("service1") m2 := msgGen("service2") m3 := msgGen("service3") m := fanIn(m1, m2, m3) for { fmt.Println(<-m) } } ================================================ FILE: lang/channel/select/select.go ================================================ package main import ( "fmt" "math/rand" "time" ) func generator() chan int { out := make(chan int) go func() { i := 0 for { time.Sleep( time.Duration(rand.Intn(1500)) * time.Millisecond) out <- i i++ } }() return out } func worker(id int, c chan int) { for n := range c { time.Sleep(time.Second) fmt.Printf("Worker %d received %d\n", id, n) } } func createWorker(id int) chan<- int { c := make(chan int) go worker(id, c) return c } func main() { var c1, c2 = generator(), generator() var worker = createWorker(0) var values []int tm := time.After(10 * time.Second) tick := time.Tick(time.Second) for { var activeWorker chan<- int var activeValue int if len(values) > 0 { activeWorker = worker activeValue = values[0] } select { case n := <-c1: values = append(values, n) case n := <-c2: values = append(values, n) case activeWorker <- activeValue: values = values[1:] case <-time.After(800 * time.Millisecond): fmt.Println("timeout") case <-tick: fmt.Println( "queue len =", len(values)) case <-tm: fmt.Println("bye") return } } } ================================================ FILE: lang/container/arrays/arrays.go ================================================ package main import "fmt" func printArray(arr [5]int) { arr[0] = 100 for i, v := range arr { fmt.Println(i, v) } } func main() { var arr1 [5]int arr2 := [3]int{1, 3, 5} arr3 := [...]int{2, 4, 6, 8, 10} var grid [4][5]int fmt.Println("array definitions:") fmt.Println(arr1, arr2, arr3) fmt.Println(grid) fmt.Println("printArray(arr1)") printArray(arr1) fmt.Println("printArray(arr3)") printArray(arr3) fmt.Println("arr1 and arr3") fmt.Println(arr1, arr3) } ================================================ FILE: lang/container/maps/maps.go ================================================ package main import "fmt" func main() { m := map[string]string{ "name": "ccmouse", "course": "golang", "site": "imooc", "quality": "notbad", } m2 := make(map[string]int) // m2 == empty map var m3 map[string]int // m3 == nil fmt.Println("m, m2, m3:") fmt.Println(m, m2, m3) fmt.Println("Traversing map m") for k, v := range m { fmt.Println(k, v) } fmt.Println("Getting values") courseName := m["course"] fmt.Println(`m["course"] =`, courseName) if causeName, ok := m["cause"]; ok { fmt.Println(causeName) } else { fmt.Println("key 'cause' does not exist") } fmt.Println("Deleting values") name, ok := m["name"] fmt.Printf("m[%q] before delete: %q, %v\n", "name", name, ok) delete(m, "name") name, ok = m["name"] fmt.Printf("m[%q] after delete: %q, %v\n", "name", name, ok) } ================================================ FILE: lang/container/nonrepeatingsubstr/nonrepeating.go ================================================ package main import ( "fmt" ) func lengthOfNonRepeatingSubStr(s string) int { lastOccurred := make(map[rune]int) start := 0 maxLength := 0 for i, ch := range []rune(s) { if lastI, ok := lastOccurred[ch]; ok && lastI >= start { start = lastI + 1 } if i-start+1 > maxLength { maxLength = i - start + 1 } lastOccurred[ch] = i } return maxLength } func main() { fmt.Println( lengthOfNonRepeatingSubStr("abcabcbb")) fmt.Println( lengthOfNonRepeatingSubStr("bbbbb")) fmt.Println( lengthOfNonRepeatingSubStr("pwwkew")) fmt.Println( lengthOfNonRepeatingSubStr("")) fmt.Println( lengthOfNonRepeatingSubStr("b")) fmt.Println( lengthOfNonRepeatingSubStr("abcdef")) fmt.Println( lengthOfNonRepeatingSubStr("这里是慕课网")) fmt.Println( lengthOfNonRepeatingSubStr("一二三二一")) fmt.Println( lengthOfNonRepeatingSubStr( "黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花")) } ================================================ FILE: lang/container/nonrepeatingsubstr/nonrepeating_test.go ================================================ package main import "testing" func TestSubstr(t *testing.T) { tests := []struct { s string ans int }{ // Normal cases {"abcabcbb", 3}, {"pwwkew", 3}, // Edge cases {"", 0}, {"b", 1}, {"bbbbbbbbb", 1}, {"abcabcabcd", 4}, // Chinese support {"这里是慕课网", 6}, {"一二三二一", 3}, {"黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花", 8}, } for _, tt := range tests { actual := lengthOfNonRepeatingSubStr(tt.s) if actual != tt.ans { t.Errorf("got %d for input %s; "+ "expected %d", actual, tt.s, tt.ans) } } } func BenchmarkSubstr(b *testing.B) { s := "黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花" for i := 0; i < 13; i++ { s = s + s } b.Logf("len(s) = %d", len(s)) ans := 8 b.ResetTimer() for i := 0; i < b.N; i++ { actual := lengthOfNonRepeatingSubStr(s) if actual != ans { b.Errorf("got %d for input %s; "+ "expected %d", actual, s, ans) } } } ================================================ FILE: lang/container/slices/sliceops.go ================================================ package main import "fmt" func printSlice(s []int) { fmt.Printf("%v, len=%d, cap=%d\n", s, len(s), cap(s)) } func sliceOps() { fmt.Println("Creating slice") var s []int // Zero value for slice is nil for i := 0; i < 100; i++ { printSlice(s) s = append(s, 2*i+1) } fmt.Println(s) s1 := []int{2, 4, 6, 8} printSlice(s1) s2 := make([]int, 16) s3 := make([]int, 10, 32) printSlice(s2) printSlice(s3) fmt.Println("Copying slice") copy(s2, s1) printSlice(s2) fmt.Println("Deleting elements from slice") s2 = append(s2[:3], s2[4:]...) printSlice(s2) fmt.Println("Popping from front") front := s2[0] s2 = s2[1:] fmt.Println(front) printSlice(s2) fmt.Println("Popping from back") tail := s2[len(s2)-1] s2 = s2[:len(s2)-1] fmt.Println(tail) printSlice(s2) } ================================================ FILE: lang/container/slices/slices.go ================================================ package main import "fmt" func updateSlice(s []int) { s[0] = 100 } func main() { arr := [...]int{0, 1, 2, 3, 4, 5, 6, 7} fmt.Println("arr[2:6] =", arr[2:6]) fmt.Println("arr[:6] =", arr[:6]) s1 := arr[2:] fmt.Println("s1 =", s1) s2 := arr[:] fmt.Println("s2 =", s2) fmt.Println("After updateSlice(s1)") updateSlice(s1) fmt.Println(s1) fmt.Println(arr) fmt.Println("After updateSlice(s2)") updateSlice(s2) fmt.Println(s2) fmt.Println(arr) fmt.Println("Reslice") fmt.Println(s2) s2 = s2[:5] fmt.Println(s2) s2 = s2[2:] fmt.Println(s2) fmt.Println("Extending slice") arr[0], arr[2] = 0, 2 fmt.Println("arr =", arr) s1 = arr[2:6] s2 = s1[3:5] // [s1[3], s1[4]] fmt.Printf("s1=%v, len(s1)=%d, cap(s1)=%d\n", s1, len(s1), cap(s1)) fmt.Printf("s2=%v, len(s2)=%d, cap(s2)=%d\n", s2, len(s2), cap(s2)) s3 := append(s2, 10) s4 := append(s3, 11) s5 := append(s4, 12) fmt.Println("s3, s4, s5 =", s3, s4, s5) // s4 and s5 no longer view arr. fmt.Println("arr =", arr) // Uncomment to run sliceOps demo. // If we see undefined: sliceOps // please try go run slices.go sliceops.go fmt.Println("Uncomment to see sliceOps demo") // sliceOps() } ================================================ FILE: lang/container/strings/strings.go ================================================ package main import ( "fmt" "unicode/utf8" ) func main() { s := "Yes我爱慕课网!" // UTF-8 fmt.Println(s) for _, b := range []byte(s) { fmt.Printf("%X ", b) } fmt.Println() for i, ch := range s { // ch is a rune fmt.Printf("(%d %X) ", i, ch) } fmt.Println() fmt.Println("Rune count:", utf8.RuneCountInString(s)) bytes := []byte(s) for len(bytes) > 0 { ch, size := utf8.DecodeRune(bytes) bytes = bytes[size:] fmt.Printf("%c ", ch) } fmt.Println() for i, ch := range []rune(s) { fmt.Printf("(%d %c) ", i, ch) } fmt.Println() } ================================================ FILE: lang/errhandling/defer/defer.go ================================================ package main import ( "fmt" "os" "bufio" "imooc.com/ccmouse/learngo/lang/functional/fib" ) func tryDefer() { for i := 0; i < 100; i++ { defer fmt.Println(i) if i == 30 { // Uncomment panic to see // how it works with defer // panic("printed too many") } } } func writeFile(filename string) { file, err := os.OpenFile(filename, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { if pathError, ok := err.(*os.PathError); !ok { panic(err) } else { fmt.Printf("%s, %s, %s\n", pathError.Op, pathError.Path, pathError.Err) } return } defer file.Close() writer := bufio.NewWriter(file) defer writer.Flush() f := fib.Fibonacci() for i := 0; i < 20; i++ { fmt.Fprintln(writer, f()) } } func main() { tryDefer() writeFile("fib.txt") } ================================================ FILE: lang/errhandling/filelistingserver/errwrapper_test.go ================================================ package main import ( "errors" "fmt" "io/ioutil" "net/http" "net/http/httptest" "os" "strings" "testing" ) func errPanic(_ http.ResponseWriter, _ *http.Request) error { panic(123) } type testingUserError string func (e testingUserError) Error() string { return e.Message() } func (e testingUserError) Message() string { return string(e) } func errUserError(_ http.ResponseWriter, _ *http.Request) error { return testingUserError("user error") } func errNotFound(_ http.ResponseWriter, _ *http.Request) error { return os.ErrNotExist } func errNoPermission(_ http.ResponseWriter, _ *http.Request) error { return os.ErrPermission } func errUnknown(_ http.ResponseWriter, _ *http.Request) error { return errors.New("unknown error") } func noError(writer http.ResponseWriter, _ *http.Request) error { fmt.Fprintln(writer, "no error") return nil } var tests = []struct { h appHandler code int message string }{ {errPanic, 500, "Internal Server Error"}, {errUserError, 400, "user error"}, {errNotFound, 404, "Not Found"}, {errNoPermission, 403, "Forbidden"}, {errUnknown, 500, "Internal Server Error"}, {noError, 200, "no error"}, } func TestErrWrapper(t *testing.T) { for _, tt := range tests { f := errWrapper(tt.h) response := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "http://www.imooc.com", nil) f(response, request) verifyResponse(response.Result(), tt.code, tt.message, t) } } func TestErrWrapperInServer(t *testing.T) { for _, tt := range tests { f := errWrapper(tt.h) server := httptest.NewServer( http.HandlerFunc(f)) resp, _ := http.Get(server.URL) verifyResponse( resp, tt.code, tt.message, t) } } func verifyResponse(resp *http.Response, expectedCode int, expectedMsg string, t *testing.T) { b, _ := ioutil.ReadAll(resp.Body) body := strings.Trim(string(b), "\n") if resp.StatusCode != expectedCode || body != expectedMsg { t.Errorf("expect (%d, %s); "+ "got (%d, %s)", expectedCode, expectedMsg, resp.StatusCode, body) } } ================================================ FILE: lang/errhandling/filelistingserver/filelisting/handler.go ================================================ package filelisting import ( "fmt" "io/ioutil" "net/http" "os" "strings" ) const prefix = "/list/" type userError string func (e userError) Error() string { return e.Message() } func (e userError) Message() string { return string(e) } func HandleFileList(writer http.ResponseWriter, request *http.Request) error { fmt.Println() if strings.Index( request.URL.Path, prefix) != 0 { return userError( fmt.Sprintf("path %s must start "+ "with %s", request.URL.Path, prefix)) } path := request.URL.Path[len(prefix):] file, err := os.Open(path) if err != nil { return err } defer file.Close() all, err := ioutil.ReadAll(file) if err != nil { return err } writer.Write(all) return nil } ================================================ FILE: lang/errhandling/filelistingserver/web.go ================================================ package main import ( "log" "net/http" _ "net/http/pprof" "os" "imooc.com/ccmouse/learngo/lang/errhandling/filelistingserver/filelisting" ) type appHandler func(writer http.ResponseWriter, request *http.Request) error func errWrapper( handler appHandler) func( http.ResponseWriter, *http.Request) { return func(writer http.ResponseWriter, request *http.Request) { // panic defer func() { if r := recover(); r != nil { log.Printf("Panic: %v", r) http.Error(writer, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } }() err := handler(writer, request) if err != nil { log.Printf("Error occurred "+ "handling request: %s", err.Error()) // user error if userErr, ok := err.(userError); ok { http.Error(writer, userErr.Message(), http.StatusBadRequest) return } // system error code := http.StatusOK switch { case os.IsNotExist(err): code = http.StatusNotFound case os.IsPermission(err): code = http.StatusForbidden default: code = http.StatusInternalServerError } http.Error(writer, http.StatusText(code), code) } } } type userError interface { error Message() string } func main() { http.HandleFunc("/", errWrapper(filelisting.HandleFileList)) err := http.ListenAndServe(":8888", nil) if err != nil { panic(err) } } ================================================ FILE: lang/errhandling/recover/recover.go ================================================ package main import ( "fmt" ) func tryRecover() { defer func() { r := recover() if r == nil { fmt.Println("Nothing to recover. " + "Please try uncomment errors " + "below.") return } if err, ok := r.(error); ok { fmt.Println("Error occurred:", err) } else { panic(fmt.Sprintf( "I don't know what to do: %v", r)) } }() // Uncomment each block to see different panic // scenarios. // Normal error //panic(errors.New("this is an error")) // Division by zero //b := 0 //a := 5 / b //fmt.Println(a) // Causes re-panic //panic(123) } func main() { tryRecover() } ================================================ FILE: lang/functional/adder/adder.go ================================================ package main import "fmt" func adder() func(int) int { sum := 0 return func(v int) int { sum += v return sum } } type iAdder func(int) (int, iAdder) func adder2(base int) iAdder { return func(v int) (int, iAdder) { return base + v, adder2(base + v) } } func main() { // a := adder() is trivial and also works. a := adder2(0) for i := 0; i < 10; i++ { var s int s, a = a(i) fmt.Printf("0 + 1 + ... + %d = %d\n", i, s) } } ================================================ FILE: lang/functional/fib/fib.go ================================================ package fib // 1, 1, 2, 3, 5, 8, 13, ... func Fibonacci() func() int { a, b := 0, 1 return func() int { a, b = b, a+b return a } } ================================================ FILE: lang/functional/main.go ================================================ package main import ( "bufio" "fmt" "io" "strings" "imooc.com/ccmouse/learngo/lang/functional/fib" ) type intGen func() int func (g intGen) Read( p []byte) (n int, err error) { next := g() if next > 10000 { return 0, io.EOF } s := fmt.Sprintf("%d\n", next) // TODO: incorrect if p is too small! return strings.NewReader(s).Read(p) } func printFileContents(reader io.Reader) { scanner := bufio.NewScanner(reader) for scanner.Scan() { fmt.Println(scanner.Text()) } } func main() { var f intGen = fib.Fibonacci() printFileContents(f) } ================================================ FILE: lang/goroutine/goroutine.go ================================================ package main import ( "fmt" "time" ) func main() { for i := 0; i < 1000; i++ { go func(i int) { for { fmt.Printf("Hello from "+ "goroutine %d\n", i) } }(i) } time.Sleep(time.Minute) } ================================================ FILE: lang/http/client.go ================================================ package main import ( "fmt" "net/http" "net/http/httputil" ) func main() { request, err := http.NewRequest( http.MethodGet, "http://www.imooc.com", nil) request.Header.Add("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1") client := http.Client{ CheckRedirect: func( req *http.Request, via []*http.Request) error { fmt.Println("Redirect:", req) return nil }, } resp, err := client.Do(request) if err != nil { panic(err) } defer resp.Body.Close() s, err := httputil.DumpResponse(resp, true) if err != nil { panic(err) } fmt.Printf("%s\n", s) } ================================================ FILE: lang/http/gindemo/ginserver.go ================================================ package main import ( "github.com/gin-gonic/gin" "go.uber.org/zap" "math/rand" "time" ) const keyRequestId = "requestId" func main() { r := gin.Default() logger, err := zap.NewProduction() if err != nil { panic(err) } r.Use(func(c *gin.Context) { s := time.Now() c.Next() logger.Info("incoming request", zap.String("path", c.Request.URL.Path), zap.Int("status", c.Writer.Status()), zap.Duration("elapsed", time.Now().Sub(s))) }, func(c *gin.Context) { c.Set(keyRequestId, rand.Int()) c.Next() }) r.GET("/ping", func(c *gin.Context) { h := gin.H{ "message": "pong", } if rid, exists := c.Get(keyRequestId); exists { h[keyRequestId] = rid } c.JSON(200, h) }) r.GET("/hello", func(c *gin.Context) { c.String(200, "hello") }) r.Run() } ================================================ FILE: lang/json/main.go ================================================ package main import ( "encoding/json" "fmt" ) type OrderItem struct { ID string `json:"id"` Name string `json:"name"` Price float64 `json:"price"` } type Order struct { ID string `json:"id"` Items []OrderItem `json:"items"` TotalPrice float64 `json:"total_price"` } func main() { parseNLP() } func marshal() { o := Order{ ID: "1234", TotalPrice: 20, Items: []OrderItem{ { ID: "item_1", Name: "learn go", Price: 15, }, { ID: "item_2", Name: "interview", Price: 10, }, }, } b, err := json.Marshal(o) if err != nil { panic(err) } fmt.Printf("%s\n", b) } func unmarshal() { s := `{"id":"1234","items":[{"id":"item_1","name":"learn go","price":15},{"id":"item_2","name":"interview","price":10}],"total_price":20}` var o Order err := json.Unmarshal([]byte(s), &o) if err != nil { panic(err) } fmt.Printf("%+v\n", o) } func parseNLP() { res := `{ "data": [ { "synonym":"", "weight":"0.6", "word": "真丝", "tag":"材质" }, { "synonym":"", "weight":"0.8", "word": "韩都衣舍", "tag":"品牌" }, { "synonym":"连身裙;联衣裙", "weight":"1.0", "word": "连衣裙", "tag":"品类" } ] }` m := struct { Data []struct { Synonym string `json:"synonym"` Tag string `json:"tag"` } `json:"data"` }{} err := json.Unmarshal([]byte(res), &m) if err != nil { panic(err) } fmt.Printf("%+v, %+v\n", m.Data[2].Synonym, m.Data[2].Tag) } ================================================ FILE: lang/maze/maze.go ================================================ package main import ( "fmt" "os" ) func readMaze(filename string) [][]int { file, err := os.Open(filename) if err != nil { panic(err) } var row, col int fmt.Fscanf(file, "%d %d", &row, &col) maze := make([][]int, row) for i := range maze { maze[i] = make([]int, col) for j := range maze[i] { fmt.Fscanf(file, "%d", &maze[i][j]) } } return maze } type point struct { i, j int } var dirs = [4]point{ {-1, 0}, {0, -1}, {1, 0}, {0, 1}} func (p point) add(r point) point { return point{p.i + r.i, p.j + r.j} } func (p point) at(grid [][]int) (int, bool) { if p.i < 0 || p.i >= len(grid) { return 0, false } if p.j < 0 || p.j >= len(grid[p.i]) { return 0, false } return grid[p.i][p.j], true } func walk(maze [][]int, start, end point) [][]int { steps := make([][]int, len(maze)) for i := range steps { steps[i] = make([]int, len(maze[i])) } Q := []point{start} for len(Q) > 0 { cur := Q[0] Q = Q[1:] if cur == end { break } for _, dir := range dirs { next := cur.add(dir) val, ok := next.at(maze) if !ok || val == 1 { continue } val, ok = next.at(steps) if !ok || val != 0 { continue } if next == start { continue } curSteps, _ := cur.at(steps) steps[next.i][next.j] = curSteps + 1 Q = append(Q, next) } } return steps } func main() { maze := readMaze("lang/maze/maze.in") steps := walk(maze, point{0, 0}, point{len(maze) - 1, len(maze[0]) - 1}) for _, row := range steps { for _, val := range row { fmt.Printf("%3d", val) } fmt.Println() } // TODO: construct path from steps } ================================================ FILE: lang/maze/maze.in ================================================ 6 5 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 1 1 1 0 0 0 1 0 0 1 0 1 0 0 0 ================================================ FILE: lang/queue/queue.go ================================================ package queue // A FIFO queue. type Queue []int // Pushes the element into the queue. // // e.g. q.Push(123) func (q *Queue) Push(v int) { *q = append(*q, v) } // Pops element from head. func (q *Queue) Pop() int { head := (*q)[0] *q = (*q)[1:] return head } // Returns if the queue is empty or not. func (q *Queue) IsEmpty() bool { return len(*q) == 0 } ================================================ FILE: lang/queue/queue_test.go ================================================ package queue import "fmt" func ExampleQueue_Pop() { q := Queue{1} q.Push(2) q.Push(3) fmt.Println(q.Pop()) fmt.Println(q.Pop()) fmt.Println(q.IsEmpty()) fmt.Println(q.Pop()) fmt.Println(q.IsEmpty()) // Output: // 1 // 2 // false // 3 // true } ================================================ FILE: lang/queue/queueentry/main.go ================================================ package main import ( "fmt" "imooc.com/ccmouse/learngo/lang/queue" ) func main() { q := queue.Queue{1} q.Push(2) q.Push(3) fmt.Println(q.Pop()) fmt.Println(q.Pop()) fmt.Println(q.IsEmpty()) fmt.Println(q.Pop()) fmt.Println(q.IsEmpty()) } ================================================ FILE: lang/retriever/main.go ================================================ package main import ( "fmt" "time" "imooc.com/ccmouse/learngo/lang/retriever/mock" "imooc.com/ccmouse/learngo/lang/retriever/real" ) type Retriever interface { Get(url string) string } type Poster interface { Post(url string, form map[string]string) string } const url = "http://www.imooc.com" func download(r Retriever) string { return r.Get(url) } func post(poster Poster) { poster.Post(url, map[string]string{ "name": "ccmouse", "course": "golang", }) } type RetrieverPoster interface { Retriever Poster } func session(s RetrieverPoster) string { s.Post(url, map[string]string{ "contents": "another faked imooc.com", }) return s.Get(url) } func main() { var r Retriever mockRetriever := mock.Retriever{ Contents: "this is a fake imooc.com"} r = &mockRetriever inspect(r) r = &real.Retriever{ UserAgent: "Mozilla/5.0", TimeOut: time.Minute, } inspect(r) // Type assertion if mockRetriever, ok := r.(*mock.Retriever); ok { fmt.Println(mockRetriever.Contents) } else { fmt.Println("r is not a mock retriever") } fmt.Println( "Try a session with mockRetriever") fmt.Println(session(&mockRetriever)) } func inspect(r Retriever) { fmt.Println("Inspecting", r) fmt.Printf(" > Type:%T Value:%v\n", r, r) fmt.Print(" > Type switch: ") switch v := r.(type) { case *mock.Retriever: fmt.Println("Contents:", v.Contents) case *real.Retriever: fmt.Println("UserAgent:", v.UserAgent) } fmt.Println() } ================================================ FILE: lang/retriever/mock/mockretriever.go ================================================ package mock import "fmt" type Retriever struct { Contents string } func (r *Retriever) String() string { return fmt.Sprintf( "Retriever: {Contents=%s}", r.Contents) } func (r *Retriever) Post(url string, form map[string]string) string { r.Contents = form["contents"] return "ok" } func (r *Retriever) Get(url string) string { return r.Contents } ================================================ FILE: lang/retriever/real/retriever.go ================================================ package real import ( "net/http" "net/http/httputil" "time" ) type Retriever struct { UserAgent string TimeOut time.Duration } func (r *Retriever) Get(url string) string { resp, err := http.Get(url) if err != nil { panic(err) } result, err := httputil.DumpResponse( resp, true) resp.Body.Close() if err != nil { panic(err) } return string(result) } ================================================ FILE: lang/rpc/client/main.go ================================================ package main import ( "fmt" "net" "net/rpc/jsonrpc" "imooc.com/ccmouse/learngo/lang/rpc" ) func main() { conn, err := net.Dial("tcp", ":1234") if err != nil { panic(err) } client := jsonrpc.NewClient(conn) var result float64 err = client.Call("DemoService.Div", rpcdemo.Args{10, 3}, &result) if err != nil { fmt.Println(err) } else { fmt.Println(result) } err = client.Call("DemoService.Div", rpcdemo.Args{10, 0}, &result) if err != nil { fmt.Println(err) } else { fmt.Println(result) } } ================================================ FILE: lang/rpc/rpc.go ================================================ package rpcdemo import "errors" type DemoService struct{} type Args struct { A, B int } func (DemoService) Div( args Args, result *float64) error { if args.B == 0 { return errors.New("division by zero") } *result = float64(args.A) / float64(args.B) return nil } ================================================ FILE: lang/rpc/server/main.go ================================================ package main import ( "log" "net" "net/rpc" "net/rpc/jsonrpc" "imooc.com/ccmouse/learngo/lang/rpc" ) func main() { rpc.Register(rpcdemo.DemoService{}) listener, err := net.Listen("tcp", ":1234") if err != nil { panic(err) } for { conn, err := listener.Accept() if err != nil { log.Printf("accept error: %v", err) continue } go jsonrpc.ServeConn(conn) } } ================================================ FILE: lang/tree/node.go ================================================ package tree import "fmt" type Node struct { Value int Left, Right *Node } func (node Node) Print() { fmt.Print(node.Value, " ") } func (node *Node) SetValue(value int) { if node == nil { fmt.Println("Setting Value to nil " + "node. Ignored.") return } node.Value = value } func CreateNode(value int) *Node { return &Node{Value: value} } ================================================ FILE: lang/tree/traversal.go ================================================ package tree import "fmt" func (node *Node) Traverse() { node.TraverseFunc(func(n *Node) { n.Print() }) fmt.Println() } func (node *Node) TraverseFunc(f func(*Node)) { if node == nil { return } node.Left.TraverseFunc(f) f(node) node.Right.TraverseFunc(f) } func (node *Node) TraverseWithChannel() chan *Node { out := make(chan *Node) go func() { node.TraverseFunc(func(node *Node) { out <- node }) close(out) }() return out } ================================================ FILE: lang/tree/treeentry/entry.go ================================================ package main import ( "fmt" "imooc.com/ccmouse/learngo/lang/tree" ) type myTreeNode struct { node *tree.Node } func (myNode *myTreeNode) postOrder() { if myNode == nil || myNode.node == nil { return } left := myTreeNode{myNode.node.Left} right := myTreeNode{myNode.node.Right} left.postOrder() right.postOrder() myNode.node.Print() } func main() { var root tree.Node root = tree.Node{Value: 3} root.Left = &tree.Node{} root.Right = &tree.Node{5, nil, nil} root.Right.Left = new(tree.Node) root.Left.Right = tree.CreateNode(2) root.Right.Left.SetValue(4) fmt.Print("In-order traversal: ") root.Traverse() fmt.Print("My own post-order traversal: ") myRoot := myTreeNode{&root} myRoot.postOrder() fmt.Println() nodeCount := 0 root.TraverseFunc(func(node *tree.Node) { nodeCount++ }) fmt.Println("Node count:", nodeCount) c := root.TraverseWithChannel() maxNodeValue := 0 for node := range c { if node.Value > maxNodeValue { maxNodeValue = node.Value } } fmt.Println("Max node value:", maxNodeValue) } ================================================ FILE: lang/tree/treeentry_embedded/entry.go ================================================ package main import ( "fmt" "imooc.com/ccmouse/learngo/lang/tree" ) type myTreeNode struct { *tree.Node // Embedding } func (myNode *myTreeNode) postOrder() { if myNode == nil || myNode.Node == nil { return } left := myTreeNode{myNode.Left} right := myTreeNode{myNode.Right} left.postOrder() right.postOrder() myNode.Print() } func main() { root := myTreeNode{&tree.Node{Value: 3}} root.Left = &tree.Node{} root.Right = &tree.Node{5, nil, nil} root.Right.Left = new(tree.Node) root.Left.Right = tree.CreateNode(2) root.Right.Left.SetValue(4) fmt.Print("In-order traversal: ") root.Traverse() fmt.Print("My own post-order traversal: ") root.postOrder() fmt.Println() } ================================================ FILE: mockserver/config/config.go ================================================ package config var ( // ServerAddress configures the server prefix in url generations. // 一般来说,我们网页中对其它网页的链接只需使用相对路径即可。 // 但目标网站的所有链接都是用了绝对路径,为了模拟,我们也需要生成绝对路径。 // 所以增加ServerAddress配置,所有的链接都使用形式: // http:///mock/album.zhenai.com/<相对路径> // 若将服务器部署在云,我们需要把这里替换成外网ip/域名:8080 ServerAddress = "localhost:8080" // ListenAddress configures where the server listens at. ListenAddress = ":8080" ) ================================================ FILE: mockserver/generator/city/city.go ================================================ // Package city implements city generator. package city import ( "bytes" "encoding/gob" "fmt" "hash/fnv" "html/template" "io" "log" "math/rand" "net/http" "github.com/gin-gonic/gin" "imooc.com/ccmouse/learngo/mockserver/config" "imooc.com/ccmouse/learngo/mockserver/generator/profile" ) // Generator represents the city generator. type Generator struct { Tmpl *template.Template ProfileGen *profile.Generator } const ( itemCount = 20 pageCount = 5 ) type params struct { City string `uri:"city" binding:"required"` Page int `uri:"page"` } // HandleRequest is the gin request handler for city generation. func (g *Generator) HandleRequest(c *gin.Context) { var p params err := c.BindUri(&p) if err != nil { log.Printf("BindUri(): %v.", err) return } if p.Page == 0 { p.Page = 1 } g.handleRequest(c, p) } func (g *Generator) handleRequest(c *gin.Context, p params) { err := g.generate(p, c.Writer) if err != nil { log.Printf("Cannot generate page for city %q and page %d: %v.", p.City, p.Page, err) c.Status(http.StatusInternalServerError) return } c.Status(http.StatusOK) } func (g *Generator) generate(p params, w io.Writer) error { h, err := hashCode(p) if err != nil { return fmt.Errorf("cannot calculate hash code: %v", err) } r := rand.New(rand.NewSource(h)) items := make([]ContentItem, itemCount) for i := range items { id := r.Int63() p := g.ProfileGen.GenerateProfile(id) items[i] = ContentItem{ ID: id, Profile: p, URL: fmt.Sprintf("http://%s/mock/album.zhenai.com/u/%d", config.ServerAddress, id), } } var pages []PageItem for i := 0; i < pageCount; i++ { targetPage := p.Page - 1 + i if targetPage < 1 { continue } url := "" if targetPage != p.Page { url = fmt.Sprintf("http://%s/mock/www.zhenai.com/zhenghun/%s/%d", config.ServerAddress, p.City, targetPage) } pages = append(pages, PageItem{ URL: url, Page: targetPage, }) } return g.Tmpl.Execute(w, Content{ Items: items, Pages: pages, }) } func hashCode(v interface{}) (int64, error) { h := fnv.New64() var b bytes.Buffer err := gob.NewEncoder(&b).Encode(v) if err != nil { return 0, fmt.Errorf("cannot encode gob for param: %v", err) } _, err = h.Write(b.Bytes()) if err != nil { return 0, fmt.Errorf("cannot write to hash: %v", err) } return int64(h.Sum64()), nil } // ContentItem defines content item bound into html template. type ContentItem struct { ID int64 Profile *profile.PhotoProfile URL string } // PageItem defines page item bound into html template. type PageItem struct { URL string Page int } // Content defines the master content bound into html template. type Content struct { Items []ContentItem Pages []PageItem } ================================================ FILE: mockserver/generator/city/city_test.go ================================================ package city import ( "bytes" "html/template" "testing" "imooc.com/ccmouse/learngo/crawler/zhenai/parser" "imooc.com/ccmouse/learngo/mockserver/config" "imooc.com/ccmouse/learngo/mockserver/generator/profile" ) func TestGenerate(t *testing.T) { config.ServerAddress = "localhost:8080" pg := &profile.Generator{Tmpl: template.Must(template.ParseFiles("../profile/profile_tmpl.html"))} g := Generator{ Tmpl: template.Must(template.ParseFiles("city_tmpl.html")), ProfileGen: pg, } var b bytes.Buffer err := g.generate(params{ City: "fuxin", Page: 34, }, &b) if err != nil { t.Fatalf("Cannot generate content: %v.", err) } r := parser.ParseCity(b.Bytes(), "") wantItems, wantRequests := 0, 24 if len(r.Items) != wantItems { t.Errorf("generate() want %d items, got %d: %v", wantItems, len(r.Items), r.Items) } if len(r.Requests) != wantRequests { t.Errorf("generate() want %d requests, got %d: %v", wantRequests, len(r.Requests), r.Requests) } verify := []struct { i int wantURL string wantParser string wantArg interface{} }{ { i: 0, wantURL: "http://localhost:8080/mock/album.zhenai.com/u/484971159322053275", wantParser: "ParseProfile", wantArg: "与你度余生迁就", }, { i: 23, wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/fuxin/37", wantParser: "ParseCity", }, } for _, v := range verify { gotURL := r.Requests[v.i].Url gotParser, gotArg := r.Requests[v.i].Parser.Serialize() if v.wantURL != gotURL || v.wantParser != gotParser || v.wantArg != gotArg { t.Errorf("generate() want %d-th request (%q, %q, %v), got (%q, %q, %v)", v.i, v.wantURL, v.wantParser, v.wantArg, gotURL, gotParser, gotArg) } } } ================================================ FILE: mockserver/generator/city/city_tmpl.html ================================================ 城市用户列表
{{range .Items}}

{{.Profile.Name}}

{{with .Profile}}
{{.Age}}岁
{{.Gender}}
{{.Income}}
{{.House}}
{{.Car}}
{{end}}
{{end}}
================================================ FILE: mockserver/generator/citylist/citylist.go ================================================ // Package citylist implements citylist generator. package citylist import ( "html/template" "io" "log" "net/http" "github.com/gin-gonic/gin" "imooc.com/ccmouse/learngo/mockserver/config" ) // Generator represents the citylist generator. type Generator struct { Tmpl *template.Template } // HandleRequest is the gin request handler for citylist generation. func (g *Generator) HandleRequest(c *gin.Context) { err := g.generate(c.Writer) if err != nil { log.Printf("Cannot generate page for citylist: %v.", err) c.Status(http.StatusInternalServerError) return } c.Status(http.StatusOK) } func (g *Generator) generate(w io.Writer) error { return g.Tmpl.Execute(w, struct { ServerAddress string }{ ServerAddress: config.ServerAddress, }) } ================================================ FILE: mockserver/generator/citylist/citylist_test.go ================================================ package citylist import ( "bytes" "html/template" "testing" "imooc.com/ccmouse/learngo/crawler/zhenai/parser" "imooc.com/ccmouse/learngo/mockserver/config" ) func TestGenerate(t *testing.T) { config.ServerAddress = "localhost:8080" g := Generator{ Tmpl: template.Must(template.ParseFiles("citylist_tmpl.html")), } var b bytes.Buffer err := g.generate(&b) if err != nil { t.Fatalf("Cannot generate content: %v.", err) } r := parser.ParseCityList(b.Bytes(), "") wantRequests := 470 if len(r.Requests) != wantRequests { t.Errorf("generate() want %d requests, got %d: %v", wantRequests, len(r.Requests), r.Requests) } verify := []struct { i int wantURL string wantParser string wantArg interface{} }{ { i: 0, wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/aba", wantParser: "ParseCity", }, { i: 23, wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/baotou", wantParser: "ParseCity", }, { i: 469, wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/zunyi", wantParser: "ParseCity", }, } for _, v := range verify { gotURL := r.Requests[v.i].Url gotParser, gotArg := r.Requests[v.i].Parser.Serialize() if v.wantURL != gotURL || v.wantParser != gotParser || v.wantArg != gotArg { t.Errorf("generate() want %d-th request (%q, %q, %v), got (%q, %q, %v)", v.i, v.wantURL, v.wantParser, v.wantArg, gotURL, gotParser, gotArg) } } } ================================================ FILE: mockserver/generator/citylist/citylist_tmpl.html ================================================ Go语言分布式爬虫测试网站
A
阿坝 阿克苏 阿拉善盟 阿勒泰 阿里 安徽 安康 安庆 鞍山 安顺 安阳 澳门
B
白城 百色 白山 白银 巴南 蚌埠 宝坻 保定 宝鸡 宝山 保山 包头 巴彦淖尔 巴音郭楞 巴中 北碚 北辰 北海 北京 本溪 毕节 滨海新 滨州 璧山 博尔塔拉 亳州
C
沧州 长春 常德 昌都 昌吉 长宁 昌平 长沙 长寿 长治 常州 朝阳 潮州 承德 成都 城口 郴州 赤峰 池州 崇明 重庆 崇左 楚雄 滁州
D
大渡口 大理 大连 丹东 大庆 大同 大兴 大兴安岭 达州 大足 德宏 德阳 德州 垫江 定西 迪庆 东城 东莞 东丽 东营
E
鄂尔多斯 恩施 鄂州
F
防城港 房山 丰都 奉节 丰台 奉贤 佛山 福建 涪陵 抚顺 阜新 阜阳 福州 抚州
G
甘南 甘肃 赣州 甘孜 高雄 广安 广东 广西 广元 广州 贵港 桂林 贵阳 贵州 果洛 固原
H
哈尔滨 海北 海淀 海东地 海口 海南 海南藏族自治州 海西 哈密 邯郸 杭州 汉中 河北 鹤壁 河池 合川 河东 合肥 鹤岗 黑河 黑龙江 河南 衡水 衡阳 和平 和田 河西 河源 菏泽 贺州 红河 虹口 红桥 淮安 淮北 怀化 淮南 怀柔 花莲 黄冈 黄南 黄浦 黄山 黄石 湖北 呼和浩特 惠州 葫芦岛 呼伦贝尔 湖南 湖州
J
嘉定 佳木斯 吉安 江北 江津 江门 江苏 江西 焦作 嘉兴 嘉义 嘉峪关 揭阳 吉林 基隆 济南 金昌 晋城 静安 景德镇 静海 荆门 荆州 金华 济宁 津南 金山 晋中 锦州 九江 九龙 九龙坡 酒泉 鸡西 蓟县
K
开封 开县 喀什 克拉玛依 克孜勒苏柯尔克孜 昆明
L
来宾 莱芜 廊坊 兰州 拉萨 乐山 梁平 凉山 连云港 聊城 辽宁 辽阳 辽源 丽江 临沧 临汾 临夏 临沂 林芝 丽水 六安 六盘水 柳州 陇南 龙岩 娄底 漯河 洛阳 泸州 吕梁
M
马鞍山 茂名 眉山 梅州 门头沟 绵阳 苗栗 闵行 密云 牡丹江
N
南岸 南昌 南充 南川 南京 南开 南宁 南平 南通 南投 南阳 那曲 内江 内蒙古 宁波 宁德 宁河 宁夏 怒江
P
盘锦 攀枝花 澎湖 彭水 平顶山 屏东 平谷 平凉 萍乡 浦东新 普洱 莆田 普陀 濮阳
Q
黔东 黔江 黔南 黔西 綦江 青岛 青海 青浦 庆阳 清远 秦皇岛 钦州 齐齐哈尔 七台河 泉州 曲靖 衢州
R
日喀则 日照 荣昌
S
三门峡 三明 三亚 山东 上海 商洛 商丘 上饶 山南 汕头 汕尾 山西 陕西 韶关 绍兴 邵阳 沙坪坝 沈阳 深圳 石家庄 石景山 十堰 石柱 石嘴山 双鸭山 顺义 朔州 四川 四平 松江 松原 绥化 遂宁 随州 宿迁 苏州 宿州
T
塔城 泰安 台北 台东 台南 台湾 太原 台中 泰州 台州 唐山 桃园 天津 天水 铁岭 铜川 通化 铜梁 通辽 铜陵 潼南 铜仁 通州 吐鲁番
W
万州 潍坊 威海 渭南 文山 温州 乌海 武汉 芜湖 乌兰察布 武隆 乌鲁木齐 武清 巫山 武威 无锡 巫溪 吴忠 梧州
X
厦门 西安 香港 湘潭 湘西 襄阳 咸宁 咸阳 孝感 西城 锡林郭勒盟 新北 兴安 邢台 西宁 新疆 新界 新乡 信阳 新余 忻州 新竹 西青 西双版纳 秀山 西藏 宣城 许昌 徐汇 徐州
Y
雅安 延安 延边 盐城 阳江 杨浦 阳泉 扬州 延庆 烟台 宜宾 宜昌 宜春 伊春 宜兰 伊犁 银川 营口 鹰潭 益阳 永川 永州 酉阳 渝北 岳阳 榆林 玉林 运城 云浮 云林 云南 云阳 玉树 玉溪 渝中
Z
枣庄 闸北 彰化 张家界 张家口 张掖 漳州 湛江 肇庆 昭通 浙江 郑州 镇江 中山 中卫 忠县 周口 舟山 珠海 驻马店 株洲 淄博 自贡 资阳 遵义
================================================ FILE: mockserver/generator/profile/profile.go ================================================ // Package profile implements profile generator. package profile import ( "fmt" "html/template" "io" "log" "math/rand" "net/http" "github.com/gin-gonic/gin" "imooc.com/ccmouse/learngo/crawler/model" "imooc.com/ccmouse/learngo/mockserver/config" ) // Recommendation defines the interface for recommendation subsystem. type Recommendation interface { NextGuess(id int64) []int64 } // Generator represents the profile generator. type Generator struct { Tmpl *template.Template Recommendation Recommendation } // HandleRequest is the gin request handler for profile generation. func (g *Generator) HandleRequest(c *gin.Context) { params := struct { ID int64 `uri:"id" binding:"required"` }{} err := c.BindUri(¶ms) if err != nil { log.Printf("BindUri(): %v.", err) return } err = g.generate(params.ID, c.Writer) if err != nil { log.Printf("Cannot generate profile for user %d: %v.", params.ID, err) c.Status(http.StatusInternalServerError) return } c.Status(http.StatusOK) } // GuessListItem defines guess list item bound into html template. type GuessListItem struct { URL string Name string PhotoURL string } // PhotoProfile defiens profile with photo bound into html template. type PhotoProfile struct { // Embed a *model.Profile here to save typing in template. *model.Profile PhotoURL string } // Content defines the master content bound into html template. type Content struct { *PhotoProfile GuessList []GuessListItem } func (g *Generator) generate(id int64, w io.Writer) error { p := g.GenerateProfile(id) var guessItems []GuessListItem if g.Recommendation != nil { guesses := g.Recommendation.NextGuess(id) guessItems = make([]GuessListItem, len(guesses)) for i, v := range guesses { gp := g.GenerateProfile(v) guessItems[i].URL = fmt.Sprintf("http://%s/mock/album.zhenai.com/u/%d", config.ServerAddress, v) guessItems[i].Name = gp.Name guessItems[i].PhotoURL = gp.PhotoURL } } return g.Tmpl.Execute(w, Content{ PhotoProfile: p, GuessList: guessItems, }) } // GenerateProfile generates a photo profile given a user id. func (g *Generator) GenerateProfile(id int64) *PhotoProfile { r := rand.New(rand.NewSource(id)) n1 := elementFromSlice(r, []string{ "断念", "街痞", "浪瘾", "猖狂", "归戾", "洒脱", "故我", "简白", "酒虑", "野性入骨", "学霸的芯", "一笑奈何", "隐匿微笑", "限时拥抱", "何必怀念", "厌与深情", "寂寞成影", "桀骜不驯", "浪痞孤王", "高冷绅士", "心事痕迹", "逍遥浪子", "隐身守候", "执手不离", "独久厌闹", "逆天飞翔", "醉生忧愁", "全球焦點", "孤者何惧", "君临天下", "无戏配角", "一身傲气", "冷暖自知", "海阔天空", "今朝醉者", "不良骚年", "薄情寡义", "称霸全服", "久别无恙°", "與我無關°", "逍遥べ无痕", "混過也愛過", "与你度余生", "一见不钟情", "孤独比酒暖°", "原来无话可说", "陪你浪迹天涯", }) n2 := elementFromSlice(r, []string{ "萌宝", "丁当", "遇到", "病娇", "莓哒", "深碍", "面码", "娇喘", "爱你", "迁就", "丸子", "初心", "baby", "猫儿.", "傻蛋.", "呆萌i", "如你*", "肉嘟嘟", "小气鬼", "胆小鬼", "欣欣雨", "酷酷猫", "柠小萌", "小丸子", "小可爱", "小仙女", "考莎啦", "晴妹儿.", "记得笑i", "稚性女", "小短腿i", "万能萌妹", "蓝莓布朗", "天瓓少女", "草莓裙摆", }) p := &model.Profile{ Name: n1 + n2, Gender: elementFromSlice(r, []string{"男", "女"}), Age: r.Intn(100), Height: r.Intn(200), Weight: r.Intn(300), Income: elementFromSlice(r, []string{ "1-2000元", "2001-3000元", "3001-5000元", "5001-8000元", "8001-10000元", "10001-20000元", "财务自由", }), Marriage: elementFromSlice(r, []string{"未婚", "离异"}), Education: elementFromSlice(r, []string{ "小学", "初中", "高中", "大学", "硕士", "博士及以上", }), Occupation: elementFromSlice(r, []string{ "人事/行政", "程序员", "产品经理", "测试工程师", "财务", "总经理", "金融", "销售", "其它", }), Hokou: elementFromSlice(r, []string{ "北京市", "上海市", "广州市", "深圳市", "成都市", "杭州市", "武汉市", "重庆市", "南京市", "天津市", "苏州市", "西安市", "长沙市", "沈阳市", "青岛市", "郑州市", "大连市", "东莞市", "宁波市", "其它", }), Xinzuo: elementFromSlice(r, []string{ "白羊座", "金牛座", "双子座", "巨蟹座", "狮子座", "处女座", "天秤座", "天蝎座", "射手座", "魔羯座", "水瓶座", "双鱼座", }), House: elementFromSlice(r, []string{ "有房", "租房", "无房", }), Car: elementFromSlice(r, []string{ "无车", "有车", "有豪车", }), } return &PhotoProfile{ Profile: p, PhotoURL: fmt.Sprintf("https://picsum.photos/seed/%d/300/300", r.Intn(100000)), } } func elementFromSlice(r *rand.Rand, s []string) string { return s[r.Intn(len(s))] } ================================================ FILE: mockserver/generator/profile/profile_test.go ================================================ package profile import ( "bytes" "html/template" "math/rand" "testing" "github.com/google/go-cmp/cmp" "imooc.com/ccmouse/learngo/crawler/engine" "imooc.com/ccmouse/learngo/crawler/model" "imooc.com/ccmouse/learngo/crawler/zhenai/parser" "imooc.com/ccmouse/learngo/mockserver/config" "imooc.com/ccmouse/learngo/mockserver/recommendation" ) func TestGenerate(t *testing.T) { config.ServerAddress = "localhost:8080" g := Generator{ Tmpl: template.Must(template.ParseFiles("profile_tmpl.html")), Recommendation: recommendation.Client{}, } rand.Seed(34534) var b bytes.Buffer err := g.generate(12345, &b) if err != nil { t.Fatalf("Cannot generate content: %v.", err) } want := engine.Item{ Url: "http://localhost:8080/mock/album.zhenai.com/u/12345", Type: "zhenai", Id: "12345", Payload: model.Profile{ Name: "逍遥べ无痕病娇", Gender: "男", Age: 36, Height: 41, Weight: 275, Income: "8001-10000元", Marriage: "未婚", Education: "初中", Occupation: "人事/行政", Hokou: "成都市", Xinzuo: "魔羯座", House: "有房", Car: "有豪车", }, } r := parser.NewProfileParser("逍遥べ无痕病娇").Parse(b.Bytes(), "http://localhost:8080/mock/album.zhenai.com/u/12345") if len(r.Items) != 1 { t.Errorf("want exactly 1 element, got %d items: %v", len(r.Items), r.Items) } else { if diff := cmp.Diff(want, r.Items[0]); diff != "" { t.Errorf("generated data is incorrect: diff: -want +got\n%s", diff) } } type reqData struct { URL string Parser string Arg interface{} } wantReq := []reqData{ { URL: "http://localhost:8080/mock/album.zhenai.com/u/12335", Parser: "ParseProfile", Arg: "酒虑肉嘟嘟", }, { URL: "http://localhost:8080/mock/album.zhenai.com/u/12340", Parser: "ParseProfile", Arg: "称霸全服胆小鬼", }, } var gotReq []reqData for _, req := range r.Requests { parser, arg := req.Parser.Serialize() gotReq = append(gotReq, reqData{ URL: req.Url, Parser: parser, Arg: arg, }) } if diff := cmp.Diff(wantReq, gotReq); diff != "" { t.Errorf("generated request is incorrect: diff: -want +got\n%s", diff) } } ================================================ FILE: mockserver/generator/profile/profile_tmpl.html ================================================ {{.Name}}的资料

{{.Name}}

{{.Age}}岁,{{.Height}}cm,{{.Income}}

发邮件 打招呼 问问题

个人信息简介

年龄:{{.Age}}岁 身高:{{.Height}}CM 月收入:{{.Income}}
婚况:{{.Marriage}} 学历:{{.Education}}
职业: {{.Occupation}} 籍贯:{{.Hokou}}

个人信息详情

想要听听TA对自己的想法?来邀请TA补充自我描述

详细资料

性别:{{.Gender}} 生肖:--
身高:{{.Height}}CM 星座:{{.Xinzuo}}
体重:{{.Weight}}KG 血型:--
体型:-- 职业:{{.Occupation}}
公司:--
信仰:--

生活状况

住房条件:{{.House}}
是否购车:{{.Car}}

兴趣爱好

喜欢的活动:-- 喜欢的食物:--
喜欢的体育运动:-- 喜欢的地方:--
喜欢的音乐:-- 喜欢的宠物:--
喜欢的影视节目:--
================================================ FILE: mockserver/main.go ================================================ package main import ( "html/template" "log" "math/rand" "net/http" "time" "github.com/gin-gonic/gin" "imooc.com/ccmouse/learngo/mockserver/config" "imooc.com/ccmouse/learngo/mockserver/generator/city" "imooc.com/ccmouse/learngo/mockserver/generator/citylist" "imooc.com/ccmouse/learngo/mockserver/generator/profile" "imooc.com/ccmouse/learngo/mockserver/recommendation" ) const templateSuggestion = "Please make sure working directory is the root of the repository, where we have go.mod/go.sum. Suggested command line: go run mockserver/main.go" func main() { profileTemplate, err := template.ParseFiles("mockserver/generator/profile/profile_tmpl.html") if err != nil { log.Fatalf("Cannot create profile template: %v. %s", err, templateSuggestion) } profileGen := &profile.Generator{ Tmpl: profileTemplate, Recommendation: recommendation.Client{}, } cityTemplate, err := template.ParseFiles("mockserver/generator/city/city_tmpl.html") if err != nil { log.Fatalf("Cannot create city template: %v. %s", err, templateSuggestion) } cityGen := &city.Generator{ Tmpl: cityTemplate, ProfileGen: profileGen, } cityListTemplate, err := template.ParseFiles("mockserver/generator/citylist/citylist_tmpl.html") if err != nil { log.Fatalf("Cannot create citylist template: %v. %s", err, templateSuggestion) } cityListGen := &citylist.Generator{ Tmpl: cityListTemplate, } rand.Seed(time.Now().Unix()) r := gin.Default() r.GET("/", func(c *gin.Context) { c.Redirect(http.StatusTemporaryRedirect, "/static/index.html") }) r.Static("/static", "mockserver/static") r.GET("mock/www.zhenai.com/zhenghun", cityListGen.HandleRequest) r.GET("mock/www.zhenai.com/zhenghun/:city/:page", cityGen.HandleRequest) r.GET("mock/www.zhenai.com/zhenghun/:city", cityGen.HandleRequest) r.GET("mock/album.zhenai.com/u/:id", profileGen.HandleRequest) log.Fatal(r.Run(config.ListenAddress)) } ================================================ FILE: mockserver/recommendation/rcmd.go ================================================ // Package recommendation provides a simple implementation mimicing a recommendation system. package recommendation import "math/rand" // If given id is 20, we may generate 10, 15, 25, ... // Intentionally desgined to increase duplication within recommendation cycles, // to mimic real-world behvior. const ( step = 5 offset = -10 maxGuess = 7 ) // Client defines the recommendation client. type Client struct{} // NextGuess returns the next batch of guesses for the given id. func (Client) NextGuess(id int64) []int64 { var res []int64 // We use a global random function because: // 1. We want recommendation result to be different each time. // 2. Only global random function is thread-safe. guessCnt := rand.Intn(maxGuess) for i, r := 0, id+offset; i < guessCnt; { if r > 0 && r != id { res = append(res, r) } i++ r += step } return res } ================================================ FILE: mockserver/recommendation/rcmd_test.go ================================================ package recommendation import ( "math" "math/rand" "testing" "github.com/google/go-cmp/cmp" ) func TestNextGuess(t *testing.T) { tests := []struct { name string id int64 want []int64 }{ { name: "normal", id: 12345, want: []int64{12335, 12340, 12350, 12355}, }, { name: "very small", id: 8, want: []int64{3, 13, 18}, }, { name: "very large", id: math.MaxInt64 - 7, want: []int64{9223372036854775790, 9223372036854775795, 9223372036854775805}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { rand.Seed(23465) cl := Client{} got := cl.NextGuess(tt.id) if diff := cmp.Diff(tt.want, got); diff != "" { t.Errorf("NextGuess(): diff: -want + got\n%s", diff) } }) } } ================================================ FILE: mockserver/static/css/blog.css ================================================ /*! Pure v1.0.1 Copyright 2013 Yahoo! Licensed under the BSD License. https://github.com/pure-css/pure/blob/master/LICENSE.md */ /*! normalize.css v^3.0 | MIT License | git.io/normalize Copyright (c) Nicolas Gallagher and Jonathan Neal */ /*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */ html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{font-size:2em;margin:.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;box-sizing:content-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}legend{border:0;padding:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}.hidden,[hidden]{display:none!important}.pure-img{max-width:100%;height:auto;display:block}.pure-g{letter-spacing:-.31em;text-rendering:optimizespeed;font-family:FreeSans,Arimo,"Droid Sans",Helvetica,Arial,sans-serif;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-align-content:flex-start;-ms-flex-line-pack:start;align-content:flex-start}@media all and (-ms-high-contrast:none),(-ms-high-contrast:active){table .pure-g{display:block}}.opera-only :-o-prefocus,.pure-g{word-spacing:-.43em}.pure-u{display:inline-block;zoom:1;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-g [class*=pure-u]{font-family:sans-serif}.pure-u-1,.pure-u-1-1,.pure-u-1-12,.pure-u-1-2,.pure-u-1-24,.pure-u-1-3,.pure-u-1-4,.pure-u-1-5,.pure-u-1-6,.pure-u-1-8,.pure-u-10-24,.pure-u-11-12,.pure-u-11-24,.pure-u-12-24,.pure-u-13-24,.pure-u-14-24,.pure-u-15-24,.pure-u-16-24,.pure-u-17-24,.pure-u-18-24,.pure-u-19-24,.pure-u-2-24,.pure-u-2-3,.pure-u-2-5,.pure-u-20-24,.pure-u-21-24,.pure-u-22-24,.pure-u-23-24,.pure-u-24-24,.pure-u-3-24,.pure-u-3-4,.pure-u-3-5,.pure-u-3-8,.pure-u-4-24,.pure-u-4-5,.pure-u-5-12,.pure-u-5-24,.pure-u-5-5,.pure-u-5-6,.pure-u-5-8,.pure-u-6-24,.pure-u-7-12,.pure-u-7-24,.pure-u-7-8,.pure-u-8-24,.pure-u-9-24{display:inline-block;zoom:1;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-u-1-24{width:4.1667%}.pure-u-1-12,.pure-u-2-24{width:8.3333%}.pure-u-1-8,.pure-u-3-24{width:12.5%}.pure-u-1-6,.pure-u-4-24{width:16.6667%}.pure-u-1-5{width:20%}.pure-u-5-24{width:20.8333%}.pure-u-1-4,.pure-u-6-24{width:25%}.pure-u-7-24{width:29.1667%}.pure-u-1-3,.pure-u-8-24{width:33.3333%}.pure-u-3-8,.pure-u-9-24{width:37.5%}.pure-u-2-5{width:40%}.pure-u-10-24,.pure-u-5-12{width:41.6667%}.pure-u-11-24{width:45.8333%}.pure-u-1-2,.pure-u-12-24{width:50%}.pure-u-13-24{width:54.1667%}.pure-u-14-24,.pure-u-7-12{width:58.3333%}.pure-u-3-5{width:60%}.pure-u-15-24,.pure-u-5-8{width:62.5%}.pure-u-16-24,.pure-u-2-3{width:66.6667%}.pure-u-17-24{width:70.8333%}.pure-u-18-24,.pure-u-3-4{width:75%}.pure-u-19-24{width:79.1667%}.pure-u-4-5{width:80%}.pure-u-20-24,.pure-u-5-6{width:83.3333%}.pure-u-21-24,.pure-u-7-8{width:87.5%}.pure-u-11-12,.pure-u-22-24{width:91.6667%}.pure-u-23-24{width:95.8333%}.pure-u-1,.pure-u-1-1,.pure-u-24-24,.pure-u-5-5{width:100%}.pure-button{display:inline-block;zoom:1;line-height:normal;white-space:nowrap;vertical-align:middle;text-align:center;cursor:pointer;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-button::-moz-focus-inner{padding:0;border:0}.pure-button-group{letter-spacing:-.31em;text-rendering:optimizespeed}.opera-only :-o-prefocus,.pure-button-group{word-spacing:-.43em}.pure-button-group .pure-button{letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-button{font-family:inherit;font-size:100%;padding:.5em 1em;color:#444;color:rgba(0,0,0,.8);border:1px solid #999;border:none transparent;background-color:#e6e6e6;text-decoration:none;border-radius:2px}.pure-button-hover,.pure-button:focus,.pure-button:hover{background-image:-webkit-gradient(linear,left top,left bottom,from(transparent),color-stop(40%,rgba(0,0,0,.05)),to(rgba(0,0,0,.1)));background-image:-webkit-linear-gradient(transparent,rgba(0,0,0,.05) 40%,rgba(0,0,0,.1));background-image:linear-gradient(transparent,rgba(0,0,0,.05) 40%,rgba(0,0,0,.1))}.pure-button:focus{outline:0}.pure-button-active,.pure-button:active{-webkit-box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;border-color:#000}.pure-button-disabled,.pure-button-disabled:active,.pure-button-disabled:focus,.pure-button-disabled:hover,.pure-button[disabled]{border:none;background-image:none;opacity:.4;cursor:not-allowed;-webkit-box-shadow:none;box-shadow:none;pointer-events:none}.pure-button-hidden{display:none}.pure-button-primary,.pure-button-selected,a.pure-button-primary,a.pure-button-selected{background-color:#0078e7;color:#fff}.pure-button-group .pure-button{margin:0;border-radius:0;border-right:1px solid #111;border-right:1px solid rgba(0,0,0,.2)}.pure-button-group .pure-button:first-child{border-top-left-radius:2px;border-bottom-left-radius:2px}.pure-button-group .pure-button:last-child{border-top-right-radius:2px;border-bottom-right-radius:2px;border-right:none}.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form select,.pure-form textarea{padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;vertical-align:middle;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input:not([type]){padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input[type=color]{padding:.2em .5em}.pure-form input[type=color]:focus,.pure-form input[type=date]:focus,.pure-form input[type=datetime-local]:focus,.pure-form input[type=datetime]:focus,.pure-form input[type=email]:focus,.pure-form input[type=month]:focus,.pure-form input[type=number]:focus,.pure-form input[type=password]:focus,.pure-form input[type=search]:focus,.pure-form input[type=tel]:focus,.pure-form input[type=text]:focus,.pure-form input[type=time]:focus,.pure-form input[type=url]:focus,.pure-form input[type=week]:focus,.pure-form select:focus,.pure-form textarea:focus{outline:0;border-color:#129fea}.pure-form input:not([type]):focus{outline:0;border-color:#129fea}.pure-form input[type=checkbox]:focus,.pure-form input[type=file]:focus,.pure-form input[type=radio]:focus{outline:thin solid #129fea;outline:1px auto #129fea}.pure-form .pure-checkbox,.pure-form .pure-radio{margin:.5em 0;display:block}.pure-form input[type=color][disabled],.pure-form input[type=date][disabled],.pure-form input[type=datetime-local][disabled],.pure-form input[type=datetime][disabled],.pure-form input[type=email][disabled],.pure-form input[type=month][disabled],.pure-form input[type=number][disabled],.pure-form input[type=password][disabled],.pure-form input[type=search][disabled],.pure-form input[type=tel][disabled],.pure-form input[type=text][disabled],.pure-form input[type=time][disabled],.pure-form input[type=url][disabled],.pure-form input[type=week][disabled],.pure-form select[disabled],.pure-form textarea[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input:not([type])[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input[readonly],.pure-form select[readonly],.pure-form textarea[readonly]{background-color:#eee;color:#777;border-color:#ccc}.pure-form input:focus:invalid,.pure-form select:focus:invalid,.pure-form textarea:focus:invalid{color:#b94a48;border-color:#e9322d}.pure-form input[type=checkbox]:focus:invalid:focus,.pure-form input[type=file]:focus:invalid:focus,.pure-form input[type=radio]:focus:invalid:focus{outline-color:#e9322d}.pure-form select{height:2.25em;border:1px solid #ccc;background-color:#fff}.pure-form select[multiple]{height:auto}.pure-form label{margin:.5em 0 .2em}.pure-form fieldset{margin:0;padding:.35em 0 .75em;border:0}.pure-form legend{display:block;width:100%;padding:.3em 0;margin-bottom:.3em;color:#333;border-bottom:1px solid #e5e5e5}.pure-form-stacked input[type=color],.pure-form-stacked input[type=date],.pure-form-stacked input[type=datetime-local],.pure-form-stacked input[type=datetime],.pure-form-stacked input[type=email],.pure-form-stacked input[type=file],.pure-form-stacked input[type=month],.pure-form-stacked input[type=number],.pure-form-stacked input[type=password],.pure-form-stacked input[type=search],.pure-form-stacked input[type=tel],.pure-form-stacked input[type=text],.pure-form-stacked input[type=time],.pure-form-stacked input[type=url],.pure-form-stacked input[type=week],.pure-form-stacked label,.pure-form-stacked select,.pure-form-stacked textarea{display:block;margin:.25em 0}.pure-form-stacked input:not([type]){display:block;margin:.25em 0}.pure-form-aligned .pure-help-inline,.pure-form-aligned input,.pure-form-aligned select,.pure-form-aligned textarea,.pure-form-message-inline{display:inline-block;vertical-align:middle}.pure-form-aligned textarea{vertical-align:top}.pure-form-aligned .pure-control-group{margin-bottom:.5em}.pure-form-aligned .pure-control-group label{text-align:right;display:inline-block;vertical-align:middle;width:10em;margin:0 1em 0 0}.pure-form-aligned .pure-controls{margin:1.5em 0 0 11em}.pure-form .pure-input-rounded,.pure-form input.pure-input-rounded{border-radius:2em;padding:.5em 1em}.pure-form .pure-group fieldset{margin-bottom:10px}.pure-form .pure-group input,.pure-form .pure-group textarea{display:block;padding:10px;margin:0 0 -1px;border-radius:0;position:relative;top:-1px}.pure-form .pure-group input:focus,.pure-form .pure-group textarea:focus{z-index:3}.pure-form .pure-group input:first-child,.pure-form .pure-group textarea:first-child{top:1px;border-radius:4px 4px 0 0;margin:0}.pure-form .pure-group input:first-child:last-child,.pure-form .pure-group textarea:first-child:last-child{top:1px;border-radius:4px;margin:0}.pure-form .pure-group input:last-child,.pure-form .pure-group textarea:last-child{top:-2px;border-radius:0 0 4px 4px;margin:0}.pure-form .pure-group button{margin:.35em 0}.pure-form .pure-input-1{width:100%}.pure-form .pure-input-3-4{width:75%}.pure-form .pure-input-2-3{width:66%}.pure-form .pure-input-1-2{width:50%}.pure-form .pure-input-1-3{width:33%}.pure-form .pure-input-1-4{width:25%}.pure-form .pure-help-inline,.pure-form-message-inline{display:inline-block;padding-left:.3em;color:#666;vertical-align:middle;font-size:.875em}.pure-form-message{display:block;color:#666;font-size:.875em}@media only screen and (max-width :480px){.pure-form button[type=submit]{margin:.7em 0 0}.pure-form input:not([type]),.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form label{margin-bottom:.3em;display:block}.pure-group input:not([type]),.pure-group input[type=color],.pure-group input[type=date],.pure-group input[type=datetime-local],.pure-group input[type=datetime],.pure-group input[type=email],.pure-group input[type=month],.pure-group input[type=number],.pure-group input[type=password],.pure-group input[type=search],.pure-group input[type=tel],.pure-group input[type=text],.pure-group input[type=time],.pure-group input[type=url],.pure-group input[type=week]{margin-bottom:0}.pure-form-aligned .pure-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.pure-form-aligned .pure-controls{margin:1.5em 0 0 0}.pure-form .pure-help-inline,.pure-form-message,.pure-form-message-inline{display:block;font-size:.75em;padding:.2em 0 .8em}}.pure-menu{-webkit-box-sizing:border-box;box-sizing:border-box}.pure-menu-fixed{position:fixed;left:0;top:0;z-index:3}.pure-menu-item,.pure-menu-list{position:relative}.pure-menu-list{list-style:none;margin:0;padding:0}.pure-menu-item{padding:0;margin:0;height:100%}.pure-menu-heading,.pure-menu-link{display:block;text-decoration:none;white-space:nowrap}.pure-menu-horizontal{width:100%;white-space:nowrap}.pure-menu-horizontal .pure-menu-list{display:inline-block}.pure-menu-horizontal .pure-menu-heading,.pure-menu-horizontal .pure-menu-item,.pure-menu-horizontal .pure-menu-separator{display:inline-block;zoom:1;vertical-align:middle}.pure-menu-item .pure-menu-item{display:block}.pure-menu-children{display:none;position:absolute;left:100%;top:0;margin:0;padding:0;z-index:3}.pure-menu-horizontal .pure-menu-children{left:0;top:auto;width:inherit}.pure-menu-active>.pure-menu-children,.pure-menu-allow-hover:hover>.pure-menu-children{display:block;position:absolute}.pure-menu-has-children>.pure-menu-link:after{padding-left:.5em;content:"\25B8";font-size:small}.pure-menu-horizontal .pure-menu-has-children>.pure-menu-link:after{content:"\25BE"}.pure-menu-scrollable{overflow-y:scroll;overflow-x:hidden}.pure-menu-scrollable .pure-menu-list{display:block}.pure-menu-horizontal.pure-menu-scrollable .pure-menu-list{display:inline-block}.pure-menu-horizontal.pure-menu-scrollable{white-space:nowrap;overflow-y:hidden;overflow-x:auto;-webkit-overflow-scrolling:touch;padding:.5em 0}.pure-menu-horizontal .pure-menu-children .pure-menu-separator,.pure-menu-separator{background-color:#ccc;height:1px;margin:.3em 0}.pure-menu-horizontal .pure-menu-separator{width:1px;height:1.3em;margin:0 .3em}.pure-menu-horizontal .pure-menu-children .pure-menu-separator{display:block;width:auto}.pure-menu-heading{text-transform:uppercase;color:#565d64}.pure-menu-link{color:#777}.pure-menu-children{background-color:#fff}.pure-menu-disabled,.pure-menu-heading,.pure-menu-link{padding:.5em 1em}.pure-menu-disabled{opacity:.5}.pure-menu-disabled .pure-menu-link:hover{background-color:transparent}.pure-menu-active>.pure-menu-link,.pure-menu-link:focus,.pure-menu-link:hover{background-color:#eee}.pure-menu-selected>.pure-menu-link,.pure-menu-selected>.pure-menu-link:visited{color:#000}.pure-table{border-collapse:collapse;border-spacing:0;empty-cells:show}.pure-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.pure-table td,.pure-table th{border-width:0 0 0 1px;font-size:inherit;margin:0;overflow:visible;padding:.5em 1em}.pure-table thead{background-color:#e0e0e0;color:#000;text-align:left;vertical-align:bottom}.pure-table td{background-color:transparent}.pure-table-odd td{background-color:#f2f2f2}.pure-table-striped tr:nth-child(2n-1) td{background-color:#f2f2f2}.pure-table-bordered td{border-bottom:1px solid #cbcbcb}.pure-table-bordered tbody>tr:last-child>td{border-bottom-width:0}.pure-table-horizontal td,.pure-table-horizontal th{border-width:0 0 1px 0;border-bottom:1px solid #cbcbcb}.pure-table-horizontal tbody>tr:last-child>td{border-bottom-width:0} * { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; } a { text-decoration: none; color: rgb(61, 146, 201); } a:hover, a:focus { text-decoration: underline; } h3 { font-weight: 100; } /* LAYOUT CSS */ .pure-img-responsive { max-width: 100%; height: auto; } #layout { padding: 0; } .header { text-align: center; top: auto; margin: 3em auto; } .sidebar { background: rgb(61, 79, 93); color: #fff; } .brand-title, .brand-tagline { margin: 0; } .brand-title { text-transform: uppercase; } .brand-tagline { font-weight: 300; color: rgb(176, 202, 219); } .nav-list { margin: 0; padding: 0; list-style: none; } .nav-item { display: inline-block; *display: inline; zoom: 1; } .nav-item a { background: transparent; border: 2px solid rgb(176, 202, 219); color: #fff; margin-top: 1em; letter-spacing: 0.05em; text-transform: uppercase; font-size: 85%; } .nav-item a:hover, .nav-item a:focus { border: 2px solid rgb(61, 146, 201); text-decoration: none; } .content-subhead { text-transform: uppercase; color: #aaa; padding: 0.4em 0; font-size: 80%; font-weight: 500; letter-spacing: 0.1em; } .content { padding: 2em 1em 0; } .post { padding-bottom: 2em; } .post-title { font-size: 2em; color: #222; margin-bottom: 0.2em; } .post-avatar { border-radius: 50px; float: right; margin-left: 1em; } .post-description { font-family: Georgia, "Cambria", serif; color: #444; line-height: 1.8em; } .post-meta { color: #999; font-size: 90%; margin: 0; } .post-category { margin: 0 0.1em; padding: 0.3em 1em; color: #fff; background: #999; font-size: 80%; } .post-category-design { background: #5aba59; } .post-category-pure { background: #4d85d1; } .post-category-yui { background: #8156a7; } .post-category-js { background: #df2d4f; } .post-images { margin: 1em 0; } .post-image-meta { margin-top: -3.5em; margin-left: 1em; color: #fff; text-shadow: 0 1px 1px #333; } .footer { padding: 1em 0; } .footer a { color: #ccc; font-size: 80%; } .footer .pure-menu a:hover, .footer .pure-menu a:focus { background: none; } @media (min-width: 48em) { .content { padding: 2em 3em 0; margin-left: 5%; } .header { margin: 80% 2em 0; text-align: right; } .sidebar { position: fixed; top: 0; bottom: 0; } } .city-photo { width: 80%; height: 80%; } .pager { margin-right: 10px; } body { color: #777; } .pure-img-responsive { max-width: 100%; height: auto; } /* Add transition to containers so they can push in and out. */ #layout, #menu, .menu-link { -webkit-transition: all 0.2s ease-out; -moz-transition: all 0.2s ease-out; -ms-transition: all 0.2s ease-out; -o-transition: all 0.2s ease-out; transition: all 0.2s ease-out; } /* This is the parent `
` that contains the menu and the content area. */ #layout { position: relative; left: 0; padding-left: 0; } #layout.active #menu { left: 150px; width: 150px; } #layout.active .menu-link { left: 150px; } /* The content `
` is where all your content goes. */ .content { padding: 0 2em; margin-bottom: 50px; line-height: 1.6em; } .header { margin: 0; color: #333; text-align: center; padding: 2.5em 2em 0; border-bottom: 1px solid #eee; } .header h1 { margin: 0.2em 0; font-size: 3em; font-weight: 300; } .header h2 { font-weight: 300; color: #ccc; padding: 0; margin-top: 0; } .content-subhead { margin: 50px 0 20px 0; font-weight: 300; color: #888; } /* The `#menu` `
` is the parent `
` that contains the `.pure-menu` that appears on the left side of the page. */ #menu { margin-left: -150px; /* "#menu" width */ width: 150px; position: fixed; top: 0; left: 0; bottom: 0; z-index: 1000; /* so the menu or its navicon stays above all content */ background: #191818; overflow-y: auto; -webkit-overflow-scrolling: touch; } /* All anchors inside the menu should be styled like this. */ #menu a { color: #999; border: none; padding: 0.6em 0 0.6em 0.6em; } /* Remove all background/borders, since we are applying them to #menu. */ #menu .pure-menu, #menu .pure-menu ul { border: none; background: transparent; } /* Add that light border to separate items into groups. */ #menu .pure-menu ul, #menu .pure-menu .menu-item-divided { border-top: 1px solid #333; } /* Change color of the anchor links on hover/focus. */ #menu .pure-menu li a:hover, #menu .pure-menu li a:focus { background: #333; } /* This styles the selected menu item `
  • `. */ #menu .pure-menu-selected, #menu .pure-menu-heading { background: #1f8dd6; } /* This styles a link within a selected menu item `
  • `. */ #menu .pure-menu-selected a { color: #fff; } /* This styles the menu heading. */ #menu .pure-menu-heading { font-size: 110%; color: #fff; margin: 0; } /* -- Dynamic Button For Responsive Menu -------------------------------------*/ /* The button to open/close the Menu is custom-made and not part of Pure. Here's how it works: */ /* `.menu-link` represents the responsive menu toggle that shows/hides on small screens. */ .menu-link { position: fixed; display: block; /* show this only on small screens */ top: 0; left: 0; /* "#menu width" */ background: #000; background: rgba(0,0,0,0.7); font-size: 10px; /* change this value to increase/decrease button size */ z-index: 10; width: 2em; height: auto; padding: 2.1em 1.6em; } .menu-link:hover, .menu-link:focus { background: #000; } .menu-link span { position: relative; display: block; } .menu-link span, .menu-link span:before, .menu-link span:after { background-color: #fff; width: 100%; height: 0.2em; } .menu-link span:before, .menu-link span:after { position: absolute; margin-top: -0.6em; content: " "; } .menu-link span:after { margin-top: 0.6em; } /* -- Responsive Styles (Media Queries) ------------------------------------- */ /* Hides the menu at `48em`, but modify this based on your app's needs. */ @media (min-width: 48em) { .header, .content { padding-left: 2em; padding-right: 2em; } #layout { padding-left: 150px; /* left col width "#menu" */ left: 0; } #menu { left: 150px; } .menu-link { position: fixed; left: 150px; display: none; } #layout.active .menu-link { left: 150px; } } @media (max-width: 48em) { /* Only apply this when the window is small. Otherwise, the following case results in extra padding on the left: * Make the window small. * Tap the menu to trigger the active state. * Make the window large again. */ #layout.active { position: relative; left: 150px; } } .profile-photo { width: 100%; height: 100%; } .guess-photo { width: 80%; height: 80%; } * { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; } /* * -- BASE STYLES -- * Most of these are inherited from Base, but I want to change a few. */ body { line-height: 1.7em; color: #7f8c8d; font-size: 13px; } h1, h2, h3, h4, h5, h6, label { color: #34495e; } .pure-img-responsive { max-width: 100%; height: auto; } /* * -- LAYOUT STYLES -- * These are some useful classes which I will need */ .l-box { padding: 1em; } .l-box-lrg { padding: 2em; border-bottom: 1px solid rgba(0,0,0,0.1); } .is-center { text-align: center; } /* * -- PURE FORM STYLES -- * Style the form inputs and labels */ .pure-form label { margin: 1em 0 0; font-weight: bold; font-size: 100%; } .pure-form input[type] { border: 2px solid #ddd; box-shadow: none; font-size: 100%; width: 100%; margin-bottom: 1em; } /* * -- PURE BUTTON STYLES -- * I want my pure-button elements to look a little different */ .pure-button { background-color: #1f8dd6; color: white; padding: 0.5em 2em; border-radius: 5px; } a.pure-button-primary { background: white; color: #1f8dd6; border-radius: 5px; font-size: 120%; } /* * -- MENU STYLES -- * I want to customize how my .pure-menu looks at the top of the page */ .home-menu { padding: 0.5em; text-align: center; box-shadow: 0 1px 1px rgba(0,0,0, 0.10); } .home-menu { background: #2d3e50; } .pure-menu.pure-menu-fixed { /* Fixed menus normally have a border at the bottom. */ border-bottom: none; /* I need a higher z-index here because of the scroll-over effect. */ z-index: 4; } .home-menu .pure-menu-heading { color: white; font-weight: 400; font-size: 120%; } .home-menu .pure-menu-selected a { color: white; } .home-menu a { color: #6FBEF3; } .home-menu li a:hover, .home-menu li a:focus { background: none; border: none; color: #AECFE5; } /* * -- SPLASH STYLES -- * This is the blue top section that appears on the page. */ .splash-container { background: #1f8dd6; z-index: 1; overflow: hidden; /* The following styles are required for the "scroll-over" effect */ width: 100%; height: 88%; top: 0; left: 0; position: fixed !important; } .splash { /* absolute center .splash within .splash-container */ width: 80%; height: 50%; margin: auto; position: absolute; top: 100px; left: 0; bottom: 0; right: 0; text-align: center; } /* This is the main heading that appears on the blue section */ .splash-head { font-size: 20px; font-weight: bold; color: white; border: 3px solid white; padding: 1em 1.6em; font-weight: 100; border-radius: 30px; line-height: 1em; } /* This is the subheading that appears on the blue section */ .splash-subhead { color: white; letter-spacing: 0.05em; opacity: 0.8; } /* * -- CONTENT STYLES -- * This represents the content area (everything below the blue section) */ .content-wrapper { /* These styles are required for the "scroll-over" effect */ position: absolute; top: 87%; width: 100%; min-height: 12%; z-index: 2; background: white; } /* We want to give the content area some more padding */ .content { padding: 1em 1em 3em; } /* This is the class used for the main content headers (

    ) */ .content-head { font-weight: 400; text-transform: uppercase; letter-spacing: 0.1em; margin: 2em 0 1em; } /* This is a modifier class used when the content-head is inside a ribbon */ .content-head-ribbon { color: white; } /* This is the class used for the content sub-headers (

    ) */ .content-subhead { color: #1f8dd6; } .content-subhead i { margin-right: 7px; } /* This is the class used for the dark-background areas. */ .ribbon { background: #2d3e50; color: #aaa; } /* * -- TABLET (AND UP) MEDIA QUERIES -- * On tablets and other medium-sized devices, we want to customize some * of the mobile styles. */ @media (min-width: 48em) { /* We increase the body font size */ body { font-size: 16px; } /* We can align the menu header to the left, but float the menu items to the right. */ .home-menu { text-align: left; } .home-menu ul { float: right; } /* We increase the height of the splash-container */ /* .splash-container { height: 500px; }*/ /* We decrease the width of the .splash, since we have more width to work with */ .splash { width: 50%; height: 50%; } .splash-head { font-size: 250%; } /* We remove the border-separator assigned to .l-box-lrg */ .l-box-lrg { border: none; } } .crawler-prompt{ color: white; } ================================================ FILE: mockserver/static/index.html ================================================ Google资深工程师深度讲解Go语言-课程实战项目爬虫测试站

    Google资深工程师
    深度讲解Go语言

    课程实战项目爬虫测试站

    进入

    爬虫设置

    ================================================ FILE: mockserver/static/instructions.html ================================================

    模拟相亲网站使用方法

    本模拟相亲网站可针对性响应原本发到第三方网站的请求。
    原本发送到类似http://www.zhenai.com/...的请求在本服务器/mock/www.zhanai.com/...被响应。
    响应的数据可被本课程原针对zhenai.com的爬虫爬取。

  • 例:http://www.zhenai.com/zhenghun/aomen --> http://localhost:8080/mock/www.zhenai.com/zhenghun/aomen
  • 例:http://album.zhenai.com/u/9166010045394758737--> http://localhost:8080/mock/album.zhenai.com/u/9166010045394758737
  • 爬虫入口请设置为:

    当然不要忘了我们在main.go 中配置正确的Parser。

    在视频中我们会从 http://www.zhenai.com/zhenghun 这个地址入手去分析对方网站的结构,采用本模拟网站时,我们只需相应的访问 即可。

    不过视频中我们对于抓取数据的URL的验证过于严格。原本我们会采用正则表达式匹配http://www.zhenai.com/...等,现在需要放宽限制,只需匹配www.zhenai.com等特征串即可,不是必须要http://
    即把视频正则表达式中类似 http://album.zhenai.com/u/[0-9]+ 改为 .*album\.zhenai\.com/u/[0-9]+
    我将在后面的章节中详细讲述有关正则表达式,在讲到时作相应的修改便可。
    具体修改请参考 https://git.imooc.com/coding-180/coding-180/commit/1c9e644e901c6c84de99ad20c14e73c45abc06ec