Repository: icpd/subscribe2clash
Branch: main
Commit: 41a0dfdfc9b4
Files: 117
Total size: 784.3 KB
Directory structure:
gitextract_x32qht1_/
├── .gitattributes
├── .gitignore
├── README.md
├── crawler/
│ ├── config/
│ │ └── config.go
│ ├── engine/
│ │ ├── concurrent.go
│ │ ├── simple.go
│ │ ├── types.go
│ │ └── worker.go
│ ├── fetcher/
│ │ └── fetcher.go
│ ├── frontend/
│ │ ├── controller/
│ │ │ └── searchresult.go
│ │ ├── model/
│ │ │ └── page.go
│ │ ├── starter.go
│ │ └── view/
│ │ ├── css/
│ │ │ └── style.css
│ │ ├── index.html
│ │ ├── js/
│ │ │ └── index.js
│ │ ├── searchresult.go
│ │ ├── searchresult_test.go
│ │ └── template.html
│ ├── main.go
│ ├── model/
│ │ ├── car.go
│ │ └── profile.go
│ ├── persist/
│ │ ├── itemsaver.go
│ │ └── itemsaver_test.go
│ ├── scheduler/
│ │ ├── queued.go
│ │ └── simple.go
│ ├── xcar/
│ │ └── parser/
│ │ ├── cardetail.go
│ │ ├── cardetail_test.go
│ │ ├── cardetail_test_data.html
│ │ ├── carlist.go
│ │ ├── carlist_test.go
│ │ ├── carlist_test_data.html
│ │ ├── carmodel.go
│ │ ├── carmodel_test.go
│ │ └── carmodel_test_data.html
│ └── zhenai/
│ └── parser/
│ ├── city.go
│ ├── citylist.go
│ ├── citylist_test.go
│ ├── citylist_test_data.html
│ ├── profile.go
│ ├── profile_test.go
│ └── profile_test_data.html
├── crawler_distributed/
│ ├── config/
│ │ └── config.go
│ ├── main.go
│ ├── persist/
│ │ ├── client/
│ │ │ └── itemsaver.go
│ │ ├── rpc.go
│ │ └── server/
│ │ ├── client_test.go
│ │ └── itemsaver.go
│ ├── rpcsupport/
│ │ └── rpc.go
│ └── worker/
│ ├── client/
│ │ └── worker.go
│ ├── rpc.go
│ ├── server/
│ │ ├── client_test.go
│ │ └── worker.go
│ └── types.go
├── go.mod
├── go.sum
├── lang/
│ ├── basic/
│ │ ├── atomic/
│ │ │ └── atomic.go
│ │ ├── basic/
│ │ │ ├── basic.go
│ │ │ └── triangle_test.go
│ │ ├── branch/
│ │ │ ├── abc.txt
│ │ │ └── branch.go
│ │ ├── func/
│ │ │ └── func.go
│ │ ├── loop/
│ │ │ └── loop.go
│ │ └── regex/
│ │ └── regex.go
│ ├── channel/
│ │ ├── channel.go
│ │ ├── done/
│ │ │ └── done.go
│ │ ├── pattern/
│ │ │ └── main.go
│ │ └── select/
│ │ └── select.go
│ ├── container/
│ │ ├── arrays/
│ │ │ └── arrays.go
│ │ ├── maps/
│ │ │ └── maps.go
│ │ ├── nonrepeatingsubstr/
│ │ │ ├── nonrepeating.go
│ │ │ └── nonrepeating_test.go
│ │ ├── slices/
│ │ │ ├── sliceops.go
│ │ │ └── slices.go
│ │ └── strings/
│ │ └── strings.go
│ ├── errhandling/
│ │ ├── defer/
│ │ │ └── defer.go
│ │ ├── filelistingserver/
│ │ │ ├── errwrapper_test.go
│ │ │ ├── filelisting/
│ │ │ │ └── handler.go
│ │ │ └── web.go
│ │ └── recover/
│ │ └── recover.go
│ ├── functional/
│ │ ├── adder/
│ │ │ └── adder.go
│ │ ├── fib/
│ │ │ └── fib.go
│ │ └── main.go
│ ├── goroutine/
│ │ └── goroutine.go
│ ├── http/
│ │ ├── client.go
│ │ └── gindemo/
│ │ └── ginserver.go
│ ├── json/
│ │ └── main.go
│ ├── maze/
│ │ ├── maze.go
│ │ └── maze.in
│ ├── queue/
│ │ ├── queue.go
│ │ ├── queue_test.go
│ │ └── queueentry/
│ │ └── main.go
│ ├── retriever/
│ │ ├── main.go
│ │ ├── mock/
│ │ │ └── mockretriever.go
│ │ └── real/
│ │ └── retriever.go
│ ├── rpc/
│ │ ├── client/
│ │ │ └── main.go
│ │ ├── rpc.go
│ │ └── server/
│ │ └── main.go
│ └── tree/
│ ├── node.go
│ ├── traversal.go
│ ├── treeentry/
│ │ └── entry.go
│ └── treeentry_embedded/
│ └── entry.go
└── mockserver/
├── config/
│ └── config.go
├── generator/
│ ├── city/
│ │ ├── city.go
│ │ ├── city_test.go
│ │ └── city_tmpl.html
│ ├── citylist/
│ │ ├── citylist.go
│ │ ├── citylist_test.go
│ │ └── citylist_tmpl.html
│ └── profile/
│ ├── profile.go
│ ├── profile_test.go
│ └── profile_tmpl.html
├── main.go
├── recommendation/
│ ├── rcmd.go
│ └── rcmd_test.go
└── static/
├── css/
│ └── blog.css
├── index.html
└── instructions.html
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
*.html linguist-language=go
*.css linguist-language=go
*.js linguist-language=go
================================================
FILE: .gitignore
================================================
================================================
FILE: README.md
================================================
Interesting things start from viewing forks.
================================================
FILE: crawler/config/config.go
================================================
package config
const (
// Parser names
ParseCity = "ParseCity"
ParseCityList = "ParseCityList"
ParseProfile = "ParseProfile"
ParseCarDetail = "ParseCarDetail"
ParseCarList = "ParseCarList"
ParseCarModel = "ParseCarModel"
NilParser = "NilParser"
// ElasticSearch
ElasticIndex = "car_profile"
// Rate limiting
Qps = 2
)
================================================
FILE: crawler/engine/concurrent.go
================================================
package engine
type ConcurrentEngine struct {
Scheduler Scheduler
WorkerCount int
ItemChan chan Item
RequestProcessor Processor
}
type Processor func(Request) (ParseResult, error)
type Scheduler interface {
ReadyNotifier
Submit(Request)
WorkerChan() chan Request
Run()
}
type ReadyNotifier interface {
WorkerReady(chan Request)
}
func (e *ConcurrentEngine) Run(seeds ...Request) {
out := make(chan ParseResult)
e.Scheduler.Run()
for i := 0; i < e.WorkerCount; i++ {
e.createWorker(e.Scheduler.WorkerChan(),
out, e.Scheduler)
}
for _, r := range seeds {
if isDuplicate(r.Url) {
continue
}
e.Scheduler.Submit(r)
}
for {
result := <-out
for _, item := range result.Items {
go func(i Item) {
e.ItemChan <- i
}(item)
}
for _, request := range result.Requests {
if isDuplicate(request.Url) {
continue
}
e.Scheduler.Submit(request)
}
}
}
func (e *ConcurrentEngine) createWorker(
in chan Request,
out chan ParseResult, ready ReadyNotifier) {
go func() {
for {
ready.WorkerReady(in)
request := <-in
result, err := e.RequestProcessor(
request)
if err != nil {
continue
}
out <- result
}
}()
}
var visitedUrls = make(map[string]bool)
func isDuplicate(url string) bool {
if visitedUrls[url] {
return true
}
visitedUrls[url] = true
return false
}
================================================
FILE: crawler/engine/simple.go
================================================
package engine
import (
"log"
)
type SimpleEngine struct{}
func (e SimpleEngine) Run(seeds ...Request) {
var requests []Request
for _, r := range seeds {
requests = append(requests, r)
}
for len(requests) > 0 {
r := requests[0]
requests = requests[1:]
parseResult, err := Worker(r)
if err != nil {
continue
}
requests = append(requests,
parseResult.Requests...)
for _, item := range parseResult.Items {
log.Printf("Got item: %v", item)
}
}
}
================================================
FILE: crawler/engine/types.go
================================================
package engine
import "imooc.com/ccmouse/learngo/crawler/config"
type ParserFunc func(
contents []byte, url string) ParseResult
type Parser interface {
Parse(contents []byte, url string) ParseResult
Serialize() (name string, args interface{})
}
type Request struct {
Url string
Parser Parser
}
type ParseResult struct {
Requests []Request
Items []Item
}
type Item struct {
Url string
Type string
Id string
Payload interface{}
}
type NilParser struct{}
func (NilParser) Parse(
_ []byte, _ string) ParseResult {
return ParseResult{}
}
func (NilParser) Serialize() (
name string, args interface{}) {
return config.NilParser, nil
}
type FuncParser struct {
parser ParserFunc
name string
}
func (f *FuncParser) Parse(
contents []byte, url string) ParseResult {
return f.parser(contents, url)
}
func (f *FuncParser) Serialize() (
name string, args interface{}) {
return f.name, nil
}
func NewFuncParser(
p ParserFunc, name string) *FuncParser {
return &FuncParser{
parser: p,
name: name,
}
}
================================================
FILE: crawler/engine/worker.go
================================================
package engine
import (
"log"
"imooc.com/ccmouse/learngo/crawler/fetcher"
)
func Worker(r Request) (ParseResult, error) {
body, err := fetcher.Fetch(r.Url)
if err != nil {
log.Printf("Fetcher: error "+
"fetching url %s: %v",
r.Url, err)
return ParseResult{}, err
}
return r.Parser.Parse(body, r.Url), nil
}
================================================
FILE: crawler/fetcher/fetcher.go
================================================
package fetcher
import (
"bufio"
"fmt"
"io/ioutil"
"net/http"
"log"
"time"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
"imooc.com/ccmouse/learngo/crawler/config"
)
var (
rateLimiter = time.Tick(
time.Second / config.Qps)
verboseLogging = false
)
func SetVerboseLogging() {
verboseLogging = true
}
func Fetch(url string) ([]byte, error) {
<-rateLimiter
if verboseLogging {
log.Printf("Fetching url %s", url)
}
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil,
fmt.Errorf("wrong status code: %d",
resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determineEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader,
e.NewDecoder())
return ioutil.ReadAll(utf8Reader)
}
func determineEncoding(
r *bufio.Reader) encoding.Encoding {
bytes, err := r.Peek(1024)
if err != nil {
log.Printf("Fetcher error: %v", err)
return unicode.UTF8
}
e, _, _ := charset.DetermineEncoding(
bytes, "")
return e
}
================================================
FILE: crawler/frontend/controller/searchresult.go
================================================
package controller
import (
"context"
"net/http"
"reflect"
"regexp"
"strconv"
"strings"
"github.com/olivere/elastic/v7"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/frontend/model"
"imooc.com/ccmouse/learngo/crawler/frontend/view"
)
type SearchResultHandler struct {
view view.SearchResultView
client *elastic.Client
}
func CreateSearchResultHandler(
template string) SearchResultHandler {
client, err := elastic.NewClient(
elastic.SetSniff(false))
if err != nil {
panic(err)
}
return SearchResultHandler{
view: view.CreateSearchResultView(
template),
client: client,
}
}
func (h SearchResultHandler) ServeHTTP(
w http.ResponseWriter, req *http.Request) {
q := strings.TrimSpace(req.FormValue("q"))
from, err := strconv.Atoi(
req.FormValue("from"))
if err != nil {
from = 0
}
page, err := h.getSearchResult(q, from)
if err != nil {
http.Error(w, err.Error(),
http.StatusBadRequest)
return
}
err = h.view.Render(w, page)
if err != nil {
http.Error(w, err.Error(),
http.StatusBadRequest)
return
}
}
const pageSize = 10
func (h SearchResultHandler) getSearchResult(
q string, from int) (model.SearchResult, error) {
var result model.SearchResult
result.Query = q
resp, err := h.client.
Search(config.ElasticIndex).
Query(elastic.NewQueryStringQuery(
rewriteQueryString(q))).
From(from).
Do(context.Background())
if err != nil {
return result, err
}
result.Hits = resp.TotalHits()
result.Start = from
result.Items = resp.Each(
reflect.TypeOf(engine.Item{}))
if result.Start == 0 {
result.PrevFrom = -1
} else {
result.PrevFrom =
(result.Start - 1) /
pageSize * pageSize
}
result.NextFrom =
result.Start + len(result.Items)
return result, nil
}
// Rewrites query string. Replaces field names
// like "Age" to "Payload.Age"
func rewriteQueryString(q string) string {
re := regexp.MustCompile(`([A-Z][a-z]*):`)
return re.ReplaceAllString(q, "Payload.$1:")
}
================================================
FILE: crawler/frontend/model/page.go
================================================
package model
type SearchResult struct {
Hits int64
Start int
Query string
PrevFrom int
NextFrom int
Items []interface{}
}
================================================
FILE: crawler/frontend/starter.go
================================================
package main
import (
"net/http"
"imooc.com/ccmouse/learngo/crawler/frontend/controller"
)
func main() {
http.Handle("/", http.FileServer(
http.Dir("crawler/frontend/view")))
http.Handle("/search",
controller.CreateSearchResultHandler(
"crawler/frontend/view/template.html"))
err := http.ListenAndServe(":8888", nil)
if err != nil {
panic(err)
}
}
================================================
FILE: crawler/frontend/view/css/style.css
================================================
/* -- import Roboto Font ---------------------------- */
@import "https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic&subset=latin,cyrillic";
/* -- You can use this tables in Bootstrap (v3) projects. -- */
/* -- Box model ------------------------------- */
*,
*:after,
*:before {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
/* -- Demo style ------------------------------- */
html,
body {
position: relative;
min-height: 100%;
height: 100%;
}
html {
position: relative;
overflow-x: hidden;
margin: 16px;
padding: 0;
min-height: 100%;
font-size: 62.5%;
}
body {
font-family: 'RobotoDraft', 'Roboto', 'Helvetica Neue, Helvetica, Arial', sans-serif;
font-style: normal;
font-weight: 300;
font-size: 1.4rem;
line-height: 2rem;
letter-spacing: 0.01rem;
color: #212121;
background-color: #f5f5f5;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
text-rendering: optimizeLegibility;
}
#demo {
margin: 20px auto;
max-width: 1200px;
}
#demo h1 {
font-size: 2.4rem;
line-height: 3.2rem;
letter-spacing: 0;
font-weight: 300;
color: #212121;
text-transform: inherit;
margin-bottom: 1rem;
text-align: center;
}
#demo h2 {
font-size: 1.5rem;
line-height: 2.8rem;
letter-spacing: 0.01rem;
font-weight: 400;
color: #212121;
text-align: center;
}
.shadow-z-1 {
-webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
-moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
}
/* -- Material Design Table style -------------- */
.table {
width: 100%;
max-width: 100%;
margin-bottom: 2rem;
background-color: #fff;
}
.table > thead > tr,
.table > tbody > tr,
.table > tfoot > tr {
-webkit-transition: all 0.3s ease;
-o-transition: all 0.3s ease;
transition: all 0.3s ease;
}
.table > thead > tr > th,
.table > tbody > tr > th,
.table > tfoot > tr > th,
.table > thead > tr > td,
.table > tbody > tr > td,
.table > tfoot > tr > td {
text-align: left;
padding: 1.6rem;
vertical-align: top;
border-top: 0;
-webkit-transition: all 0.3s ease;
-o-transition: all 0.3s ease;
transition: all 0.3s ease;
}
.table > thead > tr > th {
font-weight: 400;
color: #757575;
vertical-align: bottom;
border-bottom: 1px solid rgba(0, 0, 0, 0.12);
}
.table > caption + thead > tr:first-child > th,
.table > colgroup + thead > tr:first-child > th,
.table > thead:first-child > tr:first-child > th,
.table > caption + thead > tr:first-child > td,
.table > colgroup + thead > tr:first-child > td,
.table > thead:first-child > tr:first-child > td {
border-top: 0;
}
.table > tbody + tbody {
border-top: 1px solid rgba(0, 0, 0, 0.12);
}
.table .table {
background-color: #fff;
}
.table .no-border {
border: 0;
}
.table-condensed > thead > tr > th,
.table-condensed > tbody > tr > th,
.table-condensed > tfoot > tr > th,
.table-condensed > thead > tr > td,
.table-condensed > tbody > tr > td,
.table-condensed > tfoot > tr > td {
padding: 0.8rem;
}
.table-bordered {
border: 0;
}
.table-bordered > thead > tr > th,
.table-bordered > tbody > tr > th,
.table-bordered > tfoot > tr > th,
.table-bordered > thead > tr > td,
.table-bordered > tbody > tr > td,
.table-bordered > tfoot > tr > td {
border: 0;
border-bottom: 1px solid #e0e0e0;
}
.table-bordered > thead > tr > th,
.table-bordered > thead > tr > td {
border-bottom-width: 2px;
}
.table-striped > tbody > tr:nth-child(odd) > td,
.table-striped > tbody > tr:nth-child(odd) > th {
background-color: #f5f5f5;
}
.table-hover > tbody > tr:hover > td,
.table-hover > tbody > tr:hover > th {
background-color: rgba(0, 0, 0, 0.12);
}
@media screen and (max-width: 768px) {
.table-responsive-vertical > .table {
margin-bottom: 0;
background-color: transparent;
}
.table-responsive-vertical > .table > thead,
.table-responsive-vertical > .table > tfoot {
display: none;
}
.table-responsive-vertical > .table > tbody {
display: block;
}
.table-responsive-vertical > .table > tbody > tr {
display: block;
border: 1px solid #e0e0e0;
border-radius: 2px;
margin-bottom: 1.6rem;
}
.table-responsive-vertical > .table > tbody > tr > td {
background-color: #fff;
display: block;
vertical-align: middle;
text-align: right;
}
.table-responsive-vertical > .table > tbody > tr > td[data-title]:before {
content: attr(data-title);
float: left;
font-size: inherit;
font-weight: 400;
color: #757575;
}
.table-responsive-vertical.shadow-z-1 {
-webkit-box-shadow: none;
-moz-box-shadow: none;
box-shadow: none;
}
.table-responsive-vertical.shadow-z-1 > .table > tbody > tr {
border: none;
-webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
-moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24);
}
.table-responsive-vertical > .table-bordered {
border: 0;
}
.table-responsive-vertical > .table-bordered > tbody > tr > td {
border: 0;
border-bottom: 1px solid #e0e0e0;
}
.table-responsive-vertical > .table-bordered > tbody > tr > td:last-child {
border-bottom: 0;
}
.table-responsive-vertical > .table-striped > tbody > tr > td,
.table-responsive-vertical > .table-striped > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical > .table-striped > tbody > tr > td:nth-child(odd) {
background-color: #f5f5f5;
}
.table-responsive-vertical > .table-hover > tbody > tr:hover > td,
.table-responsive-vertical > .table-hover > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical > .table-hover > tbody > tr > td:hover {
background-color: rgba(0, 0, 0, 0.12);
}
}
.table-striped.table-mc-red > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-red > tbody > tr:nth-child(odd) > th {
background-color: #fde0dc;
}
.table-hover.table-mc-red > tbody > tr:hover > td,
.table-hover.table-mc-red > tbody > tr:hover > th {
background-color: #f9bdbb;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-red > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td:nth-child(odd) {
background-color: #fde0dc;
}
.table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-red > tbody > tr > td:hover {
background-color: #f9bdbb;
}
}
.table-striped.table-mc-pink > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-pink > tbody > tr:nth-child(odd) > th {
background-color: #fce4ec;
}
.table-hover.table-mc-pink > tbody > tr:hover > td,
.table-hover.table-mc-pink > tbody > tr:hover > th {
background-color: #f8bbd0;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-pink > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td:nth-child(odd) {
background-color: #fce4ec;
}
.table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-pink > tbody > tr > td:hover {
background-color: #f8bbd0;
}
}
.table-striped.table-mc-purple > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-purple > tbody > tr:nth-child(odd) > th {
background-color: #f3e5f5;
}
.table-hover.table-mc-purple > tbody > tr:hover > td,
.table-hover.table-mc-purple > tbody > tr:hover > th {
background-color: #e1bee7;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-purple > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td:nth-child(odd) {
background-color: #f3e5f5;
}
.table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-purple > tbody > tr > td:hover {
background-color: #e1bee7;
}
}
.table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > th {
background-color: #ede7f6;
}
.table-hover.table-mc-deep-purple > tbody > tr:hover > td,
.table-hover.table-mc-deep-purple > tbody > tr:hover > th {
background-color: #d1c4e9;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td:nth-child(odd) {
background-color: #ede7f6;
}
.table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr > td:hover {
background-color: #d1c4e9;
}
}
.table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > th {
background-color: #e8eaf6;
}
.table-hover.table-mc-indigo > tbody > tr:hover > td,
.table-hover.table-mc-indigo > tbody > tr:hover > th {
background-color: #c5cae9;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td:nth-child(odd) {
background-color: #e8eaf6;
}
.table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr > td:hover {
background-color: #c5cae9;
}
}
.table-striped.table-mc-blue > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-blue > tbody > tr:nth-child(odd) > th {
background-color: #e7e9fd;
}
.table-hover.table-mc-blue > tbody > tr:hover > td,
.table-hover.table-mc-blue > tbody > tr:hover > th {
background-color: #d0d9ff;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-blue > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td:nth-child(odd) {
background-color: #e7e9fd;
}
.table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-blue > tbody > tr > td:hover {
background-color: #d0d9ff;
}
}
.table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > th {
background-color: #e1f5fe;
}
.table-hover.table-mc-light-blue > tbody > tr:hover > td,
.table-hover.table-mc-light-blue > tbody > tr:hover > th {
background-color: #b3e5fc;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td:nth-child(odd) {
background-color: #e1f5fe;
}
.table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr > td:hover {
background-color: #b3e5fc;
}
}
.table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > th {
background-color: #e0f7fa;
}
.table-hover.table-mc-cyan > tbody > tr:hover > td,
.table-hover.table-mc-cyan > tbody > tr:hover > th {
background-color: #b2ebf2;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td:nth-child(odd) {
background-color: #e0f7fa;
}
.table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr > td:hover {
background-color: #b2ebf2;
}
}
.table-striped.table-mc-teal > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-teal > tbody > tr:nth-child(odd) > th {
background-color: #e0f2f1;
}
.table-hover.table-mc-teal > tbody > tr:hover > td,
.table-hover.table-mc-teal > tbody > tr:hover > th {
background-color: #b2dfdb;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-teal > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td:nth-child(odd) {
background-color: #e0f2f1;
}
.table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-teal > tbody > tr > td:hover {
background-color: #b2dfdb;
}
}
.table-striped.table-mc-green > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-green > tbody > tr:nth-child(odd) > th {
background-color: #d0f8ce;
}
.table-hover.table-mc-green > tbody > tr:hover > td,
.table-hover.table-mc-green > tbody > tr:hover > th {
background-color: #a3e9a4;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-green > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td:nth-child(odd) {
background-color: #d0f8ce;
}
.table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-green > tbody > tr > td:hover {
background-color: #a3e9a4;
}
}
.table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > th {
background-color: #f1f8e9;
}
.table-hover.table-mc-light-green > tbody > tr:hover > td,
.table-hover.table-mc-light-green > tbody > tr:hover > th {
background-color: #dcedc8;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td:nth-child(odd) {
background-color: #f1f8e9;
}
.table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr > td:hover {
background-color: #dcedc8;
}
}
.table-striped.table-mc-lime > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-lime > tbody > tr:nth-child(odd) > th {
background-color: #f9fbe7;
}
.table-hover.table-mc-lime > tbody > tr:hover > td,
.table-hover.table-mc-lime > tbody > tr:hover > th {
background-color: #f0f4c3;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-lime > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td:nth-child(odd) {
background-color: #f9fbe7;
}
.table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-lime > tbody > tr > td:hover {
background-color: #f0f4c3;
}
}
.table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > th {
background-color: #fffde7;
}
.table-hover.table-mc-yellow > tbody > tr:hover > td,
.table-hover.table-mc-yellow > tbody > tr:hover > th {
background-color: #fff9c4;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td:nth-child(odd) {
background-color: #fffde7;
}
.table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr > td:hover {
background-color: #fff9c4;
}
}
.table-striped.table-mc-amber > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-amber > tbody > tr:nth-child(odd) > th {
background-color: #fff8e1;
}
.table-hover.table-mc-amber > tbody > tr:hover > td,
.table-hover.table-mc-amber > tbody > tr:hover > th {
background-color: #ffecb3;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-amber > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td:nth-child(odd) {
background-color: #fff8e1;
}
.table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-amber > tbody > tr > td:hover {
background-color: #ffecb3;
}
}
.table-striped.table-mc-orange > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-orange > tbody > tr:nth-child(odd) > th {
background-color: #fff3e0;
}
.table-hover.table-mc-orange > tbody > tr:hover > td,
.table-hover.table-mc-orange > tbody > tr:hover > th {
background-color: #ffe0b2;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-orange > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td:nth-child(odd) {
background-color: #fff3e0;
}
.table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-orange > tbody > tr > td:hover {
background-color: #ffe0b2;
}
}
.table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > td,
.table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > th {
background-color: #fbe9e7;
}
.table-hover.table-mc-deep-orange > tbody > tr:hover > td,
.table-hover.table-mc-deep-orange > tbody > tr:hover > th {
background-color: #ffccbc;
}
@media screen and (max-width: 767px) {
.table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td,
.table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) {
background-color: #fff;
}
.table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td:nth-child(odd) {
background-color: #fbe9e7;
}
.table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover > td,
.table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover {
background-color: #fff;
}
.table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr > td:hover {
background-color: #ffccbc;
}
}
================================================
FILE: crawler/frontend/view/index.html
================================================
================================================
FILE: crawler/frontend/view/js/index.js
================================================
/**
* Created by Kupletsky Sergey on 05.11.14.
*
* Material Design Responsive Table
* Tested on Win8.1 with browsers: Chrome 37, Firefox 32, Opera 25, IE 11, Safari 5.1.7
* You can use this table in Bootstrap (v3) projects. Material Design Responsive Table CSS-style will override basic bootstrap style.
* JS used only for table constructor: you don't need it in your project
*/
$(document).ready(function() {
var table = $('#table');
// Table bordered
$('#table-bordered').change(function() {
var value = $( this ).val();
table.removeClass('table-bordered').addClass(value);
});
// Table striped
$('#table-striped').change(function() {
var value = $( this ).val();
table.removeClass('table-striped').addClass(value);
});
// Table hover
$('#table-hover').change(function() {
var value = $( this ).val();
table.removeClass('table-hover').addClass(value);
});
// Table color
$('#table-color').change(function() {
var value = $(this).val();
table.removeClass(/^table-mc-/).addClass(value);
});
});
// jQuery’s hasClass and removeClass on steroids
// by Nikita Vasilyev
// https://github.com/NV/jquery-regexp-classes
(function(removeClass) {
jQuery.fn.removeClass = function( value ) {
if ( value && typeof value.test === "function" ) {
for ( var i = 0, l = this.length; i < l; i++ ) {
var elem = this[i];
if ( elem.nodeType === 1 && elem.className ) {
var classNames = elem.className.split( /\s+/ );
for ( var n = classNames.length; n--; ) {
if ( value.test(classNames[n]) ) {
classNames.splice(n, 1);
}
}
elem.className = jQuery.trim( classNames.join(" ") );
}
}
} else {
removeClass.call(this, value);
}
return this;
}
})(jQuery.fn.removeClass);
================================================
FILE: crawler/frontend/view/searchresult.go
================================================
package view
import (
"html/template"
"io"
"imooc.com/ccmouse/learngo/crawler/frontend/model"
)
type SearchResultView struct {
template *template.Template
}
func CreateSearchResultView(
filename string) SearchResultView {
return SearchResultView{
template: template.Must(
template.ParseFiles(filename)),
}
}
func (s SearchResultView) Render(
w io.Writer, data model.SearchResult) error {
return s.template.Execute(w, data)
}
================================================
FILE: crawler/frontend/view/searchresult_test.go
================================================
package view
import (
"os"
"testing"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/frontend/model"
common "imooc.com/ccmouse/learngo/crawler/model"
)
func TestSearchResultView_Render(t *testing.T) {
view := CreateSearchResultView(
"template.html")
out, err := os.Create("template.test.html")
if err != nil {
panic(err)
}
defer out.Close()
page := model.SearchResult{}
page.Hits = 123
item := engine.Item{
Url: "http://album.zhenai.com/u/108906739",
Type: "zhenai",
Id: "108906739",
Payload: common.Profile{
Age: 34,
Height: 162,
Weight: 57,
Income: "3001-5000元",
Gender: "女",
Name: "安静的雪",
Xinzuo: "牡羊座",
Occupation: "人事/行政",
Marriage: "离异",
House: "已购房",
Hokou: "山东菏泽",
Education: "大学本科",
Car: "未购车",
},
}
for i := 0; i < 10; i++ {
page.Items = append(page.Items, item)
}
err = view.Render(out, page)
if err != nil {
t.Error(err)
}
// TODO: verify contents in template.test.html
}
================================================
FILE: crawler/frontend/view/template.html
================================================
共为你找到相关结果约为{{.Hits}}个。显示从{{.Start}}起共{{len .Items}}个。
{{range .Items}}
{{if eq .Type "xcar"}}
{{.Payload.Name}}
{{with .Payload}}
{{.Name}}
{{.Price}}万元
{{.Size}}
{{.Fuel}}L/100km
{{.Transmission}}
{{.Engine}}
{{.Displacement}}L
{{.MaxSpeed}}km/h
{{.Acceleration}}s,0-100km/h
{{end}}
{{end}}
{{if eq .Type "zhenai"}}
{{.Payload.Name}}
{{with .Payload}}
{{.Gender}}
{{.Age}}
{{.Height}}cm
{{.Weight}}kg
{{.Income}}
{{.Education}}
{{.Occupation}}
{{.Hokou}}
{{.Xinzuo}}
{{.House}}
{{.Car}}
{{end}}
{{end}}
{{else}}
没有找到相关用户
{{end}}
{{if ge .PrevFrom 0}}
上一页
{{end}}
下一页
================================================
FILE: crawler/main.go
================================================
package main
import (
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/persist"
"imooc.com/ccmouse/learngo/crawler/scheduler"
"imooc.com/ccmouse/learngo/crawler/zhenai/parser"
)
func main() {
itemChan, err := persist.ItemSaver(
config.ElasticIndex)
if err != nil {
panic(err)
}
e := engine.ConcurrentEngine{
Scheduler: &scheduler.QueuedScheduler{},
WorkerCount: 100,
ItemChan: itemChan,
RequestProcessor: engine.Worker,
}
e.Run(engine.Request{
Url: "http://localhost:8080/mock/www.zhenai.com/zhenghun",
Parser: engine.NewFuncParser(
parser.ParseCityList,
config.ParseCityList),
})
}
================================================
FILE: crawler/model/car.go
================================================
package model
type Car struct {
Name string
Price float64
ImageURL string
Size string
Fuel float64
Transmission string
Engine string
Displacement float64 // 排量
MaxSpeed float64
Acceleration float64
}
================================================
FILE: crawler/model/profile.go
================================================
package model
import "encoding/json"
type Profile struct {
Name string
Gender string
Age int
Height int
Weight int
Income string
Marriage string
Education string
Occupation string
Hokou string
Xinzuo string
House string
Car string
}
func FromJsonObj(o interface{}) (Profile, error) {
var profile Profile
s, err := json.Marshal(o)
if err != nil {
return profile, err
}
err = json.Unmarshal(s, &profile)
return profile, err
}
================================================
FILE: crawler/persist/itemsaver.go
================================================
package persist
import (
"context"
"errors"
"log"
"github.com/olivere/elastic/v7"
"imooc.com/ccmouse/learngo/crawler/engine"
)
func ItemSaver(
index string) (chan engine.Item, error) {
client, err := elastic.NewClient(
// Must turn off sniff in docker
elastic.SetSniff(false))
if err != nil {
return nil, err
}
out := make(chan engine.Item)
go func() {
itemCount := 0
for {
item := <-out
log.Printf("Item Saver: got item "+
"#%d: %v", itemCount, item)
itemCount++
err := Save(client, index, item)
if err != nil {
log.Printf("Item Saver: error "+
"saving item %v: %v",
item, err)
}
}
}()
return out, nil
}
func Save(
client *elastic.Client, index string,
item engine.Item) error {
if item.Type == "" {
return errors.New("must supply Type")
}
indexService := client.Index().
Index(index).
Type(item.Type).
BodyJson(item)
if item.Id != "" {
indexService.Id(item.Id)
}
_, err := indexService.
Do(context.Background())
return err
}
================================================
FILE: crawler/persist/itemsaver_test.go
================================================
package persist
import (
"context"
"encoding/json"
"testing"
"github.com/olivere/elastic/v7"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
)
func TestSave(t *testing.T) {
expected := engine.Item{
Url: "http://album.zhenai.com/u/108906739",
Type: "zhenai",
Id: "108906739",
Payload: model.Profile{
Age: 34,
Height: 162,
Weight: 57,
Income: "3001-5000元",
Gender: "女",
Name: "安静的雪",
Xinzuo: "牡羊座",
Occupation: "人事/行政",
Marriage: "离异",
House: "已购房",
Hokou: "山东菏泽",
Education: "大学本科",
Car: "未购车",
},
}
// TODO: Try to start up elastic search
// here using docker go client.
client, err := elastic.NewClient(
elastic.SetSniff(false))
if err != nil {
panic(err)
}
const index = "dating_test"
// Save expected item
err = Save(client, index, expected)
if err != nil {
panic(err)
}
// Fetch saved item
resp, err := client.Get().
Index(index).
Type(expected.Type).
Id(expected.Id).
Do(context.Background())
if err != nil {
panic(err)
}
t.Logf("%s", resp.Source)
var actual engine.Item
json.Unmarshal(resp.Source, &actual)
actualProfile, _ := model.FromJsonObj(
actual.Payload)
actual.Payload = actualProfile
// Verify result
if actual != expected {
t.Errorf("got %v; expected %v",
actual, expected)
}
}
================================================
FILE: crawler/scheduler/queued.go
================================================
package scheduler
import "imooc.com/ccmouse/learngo/crawler/engine"
type QueuedScheduler struct {
requestChan chan engine.Request
workerChan chan chan engine.Request
}
func (s *QueuedScheduler) WorkerChan() chan engine.Request {
return make(chan engine.Request)
}
func (s *QueuedScheduler) Submit(r engine.Request) {
s.requestChan <- r
}
func (s *QueuedScheduler) WorkerReady(
w chan engine.Request) {
s.workerChan <- w
}
func (s *QueuedScheduler) Run() {
s.workerChan = make(chan chan engine.Request)
s.requestChan = make(chan engine.Request)
go func() {
var requestQ []engine.Request
var workerQ []chan engine.Request
for {
var activeRequest engine.Request
var activeWorker chan engine.Request
if len(requestQ) > 0 &&
len(workerQ) > 0 {
activeWorker = workerQ[0]
activeRequest = requestQ[0]
}
select {
case r := <-s.requestChan:
requestQ = append(requestQ, r)
case w := <-s.workerChan:
workerQ = append(workerQ, w)
case activeWorker <- activeRequest:
workerQ = workerQ[1:]
requestQ = requestQ[1:]
}
}
}()
}
================================================
FILE: crawler/scheduler/simple.go
================================================
package scheduler
import "imooc.com/ccmouse/learngo/crawler/engine"
type SimpleScheduler struct {
workerChan chan engine.Request
}
func (s *SimpleScheduler) WorkerChan() chan engine.Request {
return s.workerChan
}
func (s *SimpleScheduler) WorkerReady(chan engine.Request) {
}
func (s *SimpleScheduler) Run() {
s.workerChan = make(chan engine.Request)
}
func (s *SimpleScheduler) Submit(
r engine.Request) {
go func() { s.workerChan <- r }()
}
================================================
FILE: crawler/xcar/parser/cardetail.go
================================================
package parser
import (
"fmt"
"regexp"
"strconv"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
)
var priceReTmpl = `(\d+\.\d+) `
var nameRe = regexp.MustCompile(`【(.*)报价_图片_参数】.* `)
var carImageRe = regexp.MustCompile(` (\d+[^\d]\d+[^\d]\d+mm)`)
var fuelRe = regexp.MustCompile(`(\d+\.\d+)L/100km`)
var transmissionRe = regexp.MustCompile(`(.+)`)
var engineRe = regexp.MustCompile(`发\s*动\s*机.*\s*.*<.*>(\d+kW[^<]*)<`)
var displacementRe = regexp.MustCompile(`]*>(\d+)`)
var accelRe = regexp.MustCompile(`]*>([\d\.]+)`)
var urlRe = regexp.MustCompile(`http://newcar.xcar.com.cn/(m\d+)/`)
func ParseCarDetail(contents []byte, url string) engine.ParseResult {
id := extractString([]byte(url), urlRe)
car := model.Car{
Name: extractString(contents, nameRe),
ImageURL: "http:" + extractString(contents, carImageRe),
Size: extractString(contents, sizeRe),
Fuel: extractFloat(contents, fuelRe),
Transmission: extractString(contents, transmissionRe),
Engine: extractString(contents, engineRe),
Displacement: extractFloat(contents, displacementRe),
MaxSpeed: extractFloat(contents, maxSpeedRe),
Acceleration: extractFloat(contents, accelRe),
}
priceRe, err := regexp.Compile(
fmt.Sprintf(priceReTmpl, regexp.QuoteMeta(id)))
if err == nil {
car.Price = extractFloat(contents, priceRe)
}
result := engine.ParseResult{
Items: []engine.Item{
{
Id: id,
Url: url,
Type: "xcar",
Payload: car,
},
},
}
carModelResult := ParseCarModel(contents, url)
result.Requests = carModelResult.Requests
return result
}
func extractString(
contents []byte, re *regexp.Regexp) string {
match := re.FindSubmatch(contents)
if len(match) >= 2 {
return string(match[1])
} else {
return ""
}
}
func extractFloat(contents []byte, re *regexp.Regexp) float64 {
f, err := strconv.ParseFloat(extractString(contents, re), 64)
if err != nil {
return 0
}
return f
}
================================================
FILE: crawler/xcar/parser/cardetail_test.go
================================================
package parser
import (
"io/ioutil"
"testing"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
)
func TestParseCarDetail(t *testing.T) {
contents, err := ioutil.ReadFile(
"cardetail_test_data.html")
if err != nil {
panic(err)
}
expectedItem := engine.Item{
Url: "http://newcar.xcar.com.cn/m35001/",
Type: "xcar",
Id: "m35001",
Payload: model.Car{
Name: "奥迪TT双门2017款45 TFSI",
Price: 47.18,
ImageURL: "http://img1.xcarimg.com/b63/s8386/m_20170616000036181753843373443.jpg-280x210.jpg",
Size: "4191×1832×1353mm",
Fuel: 16.7,
Transmission: "6挡双离合",
Engine: "169kW(2.0L涡轮增压)",
Displacement: 2,
MaxSpeed: 250,
Acceleration: 5.9,
},
}
result := ParseCarDetail(contents, "http://newcar.xcar.com.cn/m35001/")
if len(result.Items) != 1 {
t.Errorf("Must only return one item, but %d returned",
len(result.Items))
}
actualItem := result.Items[0]
if actualItem != expectedItem {
t.Errorf("expected item: %+v, but was %+v",
expectedItem, actualItem)
}
const resultSize = 8
expectedUrls := []string{
"http://newcar.xcar.com.cn/m45776/",
"http://newcar.xcar.com.cn/m45776/",
"http://newcar.xcar.com.cn/m32946/",
}
if len(result.Requests) != resultSize {
t.Errorf("result should have %d "+
"requests; but had %d",
resultSize, len(result.Requests))
}
for i, url := range expectedUrls {
if result.Requests[i].Url != url {
t.Errorf("expected url #%d: %s; but "+
"was %s",
i, url, result.Requests[i].Url)
}
}
}
================================================
FILE: crawler/xcar/parser/cardetail_test_data.html
================================================
【奥迪TT双门2017款45 TFSI报价_图片_参数】_奥迪(进口)奥迪TT双门怎么样_爱卡汽车
车身尺寸: 4191×1832×1353mm
工信部油耗: 16.7L/100km
变 速 箱: 6挡双离合
发 动 机: 169kW(2.0L涡轮增压)
驱动方式: 前置前驱
排 量: 2.0L
基本参数 解释: -无 ● 有 ○ 可选 ▲ 待查
级别
跑车
发动机
169kW(2.0L涡轮增压)
动力类型
汽油机
变速箱
6挡双离合
长×宽×高(mm)
4191×1832×1353
车身结构
2门 4座 跑车
上市年份
2017
最高车速(km/h)
250
0-100加速时间(s)
5.9
工信部油耗(L/100km)
6.7
车身参数
车长(mm)
4191
车宽(mm)
1832
车高(mm)
1353
轴距(mm)
2502
整备质量(kg)
1350
最小离地间隙(mm)
-
前轮距(mm)
1572
后轮距(mm)
1552
车门数
2
座位数
4
油箱容积(L)
50
行李厢最小容积(L)
305
查看全部配置
爱卡汽车提醒您:过低的报价在实际交易中可能存在附加条件
全款购车
裸车价格:471800元
+ 购置税:40325 元
+ 交强险:950 元
+ 商业险:约13386 元
+ 车船税: 480 元
+ 上牌费: 500 元
参考成交价:约527,400元
注:此结果仅供参考,实际费用以当地缴费为准
1 大众 (关注度:130449)
2 丰田 (关注度:116359)
3 奥迪 (关注度:104486)
4 宝马 (关注度:89802)
5 现代 (关注度:88085)
6 本田 (关注度:87921)
7 起亚 (关注度:84619)
8 日产 (关注度:80030)
9 雪佛兰 (关注度:71721)
10 奔驰 (关注度:69187)
1 奔腾B50 (关注度:231966)
2 卡罗拉 (关注度:46392)
3 凌渡 (关注度:36010)
4 思域 (关注度:33959)
5 标致308 (关注度:18191)
6 科鲁兹三厢 (关注度:17781)
7 蔚领 (关注度:17715)
8 轩逸 (关注度:17666)
9 帝豪GL (关注度:16443)
10 英朗 (关注度:15303)
================================================
FILE: crawler/xcar/parser/carlist.go
================================================
package parser
import (
"regexp"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
)
const host = "http://newcar.xcar.com.cn"
var carModelRe = regexp.MustCompile(``)
var carListRe = regexp.MustCompile(`
【图】买什么车好_最新上市汽车车型推荐-爱卡汽车网
价格
0万 5万 10万 15万 20万 25万 30万 35万 50万 70万 100万
1326 个符合条件的车系,
10587 款符合条件的车型
================================================
FILE: crawler/xcar/parser/carmodel.go
================================================
package parser
import (
"regexp"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
)
var carDetailRe = regexp.MustCompile(`
【奥迪TT双门】新奥迪_奥迪TT双门报价|图片_2019奥迪TT双门怎么样_爱卡汽车
坐进前排舒服吗?
◆ ◆
适中
空间刚刚好
适中 -
一般单排座车型腿部空间都可调节到合适位置,因此适中代表了头部空间基本能够满足使用。
◆ ◆
局促
坐姿受局限
局促 -
单排座车型局促情况很少见,头部空间很压抑,并且腿部不能调节到合适位置。
◆ ◆
适中
空间刚刚好
适中 -
一般单排座车型腿部空间都可调节到合适位置,因此适中代表了头部空间基本能够满足使用。
◆ ◆
宽裕
空间很宽敞
宽裕 -
充足的头部空间和前后调节幅度较大的座椅,带给您宽裕的驾驶空间。
坐进第二排舒服吗?
◆ ◆
局促
坐姿受局限
局促 -
单排座车型局促情况很少见,头部空间很压抑,并且腿部不能调节到合适位置。
◆ ◆
局促
坐姿受局限
局促 -
单排座车型局促情况很少见,头部空间很压抑,并且腿部不能调节到合适位置。
◆ ◆
适中
空间刚刚好
适中 -
一般单排座车型腿部空间都可调节到合适位置,因此适中代表了头部空间基本能够满足使用。
◆ ◆
宽裕
空间很宽敞
宽裕 -
充足的头部空间和前后调节幅度较大的座椅,带给您宽裕的驾驶空间。
================================================
FILE: crawler/zhenai/parser/city.go
================================================
package parser
import (
"regexp"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
)
var (
profileRe = regexp.MustCompile(
` ]*>([^<]+) `)
cityUrlRe = regexp.MustCompile(
`href="(.*www\.zhenai\.com/zhenghun/[^"]+)"`)
)
func ParseCity(
contents []byte, _ string) engine.ParseResult {
matches := profileRe.FindAllSubmatch(
contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
result.Requests = append(
result.Requests, engine.Request{
Url: string(m[1]),
Parser: NewProfileParser(
string(m[2])),
})
}
matches = cityUrlRe.FindAllSubmatch(
contents, -1)
for _, m := range matches {
result.Requests = append(result.Requests,
engine.Request{
Url: string(m[1]),
Parser: engine.NewFuncParser(
ParseCity, config.ParseCity),
})
}
return result
}
================================================
FILE: crawler/zhenai/parser/citylist.go
================================================
package parser
import (
"regexp"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
)
const cityListRe = `]*>([^<]+) `
func ParseCityList(
contents []byte, _ string) engine.ParseResult {
re := regexp.MustCompile(cityListRe)
matches := re.FindAllSubmatch(contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
result.Requests = append(
result.Requests, engine.Request{
Url: string(m[1]),
Parser: engine.NewFuncParser(
ParseCity, config.ParseCity),
})
}
return result
}
================================================
FILE: crawler/zhenai/parser/citylist_test.go
================================================
package parser
import (
"testing"
"io/ioutil"
)
func TestParseCityList(t *testing.T) {
contents, err := ioutil.ReadFile(
"citylist_test_data.html")
if err != nil {
panic(err)
}
result := ParseCityList(contents, "")
const resultSize = 470
expectedUrls := []string{
"http://www.zhenai.com/zhenghun/aba",
"http://www.zhenai.com/zhenghun/akesu",
"http://www.zhenai.com/zhenghun/alashanmeng",
}
if len(result.Requests) != resultSize {
t.Errorf("result should have %d "+
"requests; but had %d",
resultSize, len(result.Requests))
}
for i, url := range expectedUrls {
if result.Requests[i].Url != url {
t.Errorf("expected url #%d: %s; but "+
"was %s",
i, url, result.Requests[i].Url)
}
}
}
================================================
FILE: crawler/zhenai/parser/citylist_test_data.html
================================================
珍爱征婚网_征婚交友信息_同城征婚网站
A
阿坝
阿克苏
阿拉善盟
阿勒泰
阿里
安徽
安康
安庆
鞍山
安顺
安阳
澳门
B
白城
百色
白山
白银
巴南
蚌埠
宝坻
保定
宝鸡
宝山
保山
包头
巴彦淖尔
巴音郭楞
巴中
北碚
北辰
北海
北京
本溪
毕节
滨海新
滨州
璧山
博尔塔拉
亳州
C
沧州
长春
常德
昌都
昌吉
长宁
昌平
长沙
长寿
长治
常州
朝阳
潮州
承德
成都
城口
郴州
赤峰
池州
崇明
重庆
崇左
楚雄
滁州
D
大渡口
大理
大连
丹东
大庆
大同
大兴
大兴安岭
达州
大足
德宏
德阳
德州
垫江
定西
迪庆
东城
东莞
东丽
东营
E
鄂尔多斯
恩施
鄂州
F
防城港
房山
丰都
奉节
丰台
奉贤
佛山
福建
涪陵
抚顺
阜新
阜阳
福州
抚州
G
甘南
甘肃
赣州
甘孜
高雄
广安
广东
广西
广元
广州
贵港
桂林
贵阳
贵州
果洛
固原
H
哈尔滨
海北
海淀
海东地
海口
海南
海南藏族自治州
海西
哈密
邯郸
杭州
汉中
河北
鹤壁
河池
合川
河东
合肥
鹤岗
黑河
黑龙江
河南
衡水
衡阳
和平
和田
河西
河源
菏泽
贺州
红河
虹口
红桥
淮安
淮北
怀化
淮南
怀柔
花莲
黄冈
黄南
黄浦
黄山
黄石
湖北
呼和浩特
惠州
葫芦岛
呼伦贝尔
湖南
湖州
J
嘉定
佳木斯
吉安
江北
江津
江门
江苏
江西
焦作
嘉兴
嘉义
嘉峪关
揭阳
吉林
基隆
济南
金昌
晋城
静安
景德镇
静海
荆门
荆州
金华
济宁
津南
金山
晋中
锦州
九江
九龙
九龙坡
酒泉
鸡西
蓟县
K
开封
开县
喀什
克拉玛依
克孜勒苏柯尔克孜
昆明
L
来宾
莱芜
廊坊
兰州
拉萨
乐山
梁平
凉山
连云港
聊城
辽宁
辽阳
辽源
丽江
临沧
临汾
临夏
临沂
林芝
丽水
六安
六盘水
柳州
陇南
龙岩
娄底
漯河
洛阳
泸州
吕梁
M
马鞍山
茂名
眉山
梅州
门头沟
绵阳
苗栗
闵行
密云
牡丹江
N
南岸
南昌
南充
南川
南京
南开
南宁
南平
南通
南投
南阳
那曲
内江
内蒙古
宁波
宁德
宁河
宁夏
怒江
P
盘锦
攀枝花
澎湖
彭水
平顶山
屏东
平谷
平凉
萍乡
浦东新
普洱
莆田
普陀
濮阳
Q
黔东
黔江
黔南
黔西
綦江
青岛
青海
青浦
庆阳
清远
秦皇岛
钦州
齐齐哈尔
七台河
泉州
曲靖
衢州
R
日喀则
日照
荣昌
S
三门峡
三明
三亚
山东
上海
商洛
商丘
上饶
山南
汕头
汕尾
山西
陕西
韶关
绍兴
邵阳
沙坪坝
沈阳
深圳
石家庄
石景山
十堰
石柱
石嘴山
双鸭山
顺义
朔州
四川
四平
松江
松原
绥化
遂宁
随州
宿迁
苏州
宿州
T
塔城
泰安
台北
台东
台南
台湾
太原
台中
泰州
台州
唐山
桃园
天津
天水
铁岭
铜川
通化
铜梁
通辽
铜陵
潼南
铜仁
通州
吐鲁番
W
万州
潍坊
威海
渭南
文山
温州
乌海
武汉
芜湖
乌兰察布
武隆
乌鲁木齐
武清
巫山
武威
无锡
巫溪
吴忠
梧州
X
厦门
西安
香港
湘潭
湘西
襄阳
咸宁
咸阳
孝感
西城
锡林郭勒盟
新北
兴安
邢台
西宁
新疆
新界
新乡
信阳
新余
忻州
新竹
西青
西双版纳
秀山
西藏
宣城
许昌
徐汇
徐州
Y
雅安
延安
延边
盐城
阳江
杨浦
阳泉
扬州
延庆
烟台
宜宾
宜昌
宜春
伊春
宜兰
伊犁
银川
营口
鹰潭
益阳
永川
永州
酉阳
渝北
岳阳
榆林
玉林
运城
云浮
云林
云南
云阳
玉树
玉溪
渝中
Z
枣庄
闸北
彰化
张家界
张家口
张掖
漳州
湛江
肇庆
昭通
浙江
郑州
镇江
中山
中卫
忠县
周口
舟山
珠海
驻马店
株洲
淄博
自贡
资阳
遵义
>
================================================
FILE: crawler/zhenai/parser/profile.go
================================================
package parser
import (
"regexp"
"strconv"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
)
var ageRe = regexp.MustCompile(
`年龄: (\d+)岁 `)
var heightRe = regexp.MustCompile(
`身高: (\d+)CM `)
var incomeRe = regexp.MustCompile(
`月收入: ([^<]+) `)
var weightRe = regexp.MustCompile(
`体重: (\d+)KG `)
var genderRe = regexp.MustCompile(
`性别: ([^<]+) `)
var xinzuoRe = regexp.MustCompile(
`星座: ([^<]+) `)
var marriageRe = regexp.MustCompile(
`婚况: ([^<]+) `)
var educationRe = regexp.MustCompile(
`学历: ([^<]+) `)
var occupationRe = regexp.MustCompile(
`职业: ([^<]+) `)
var hokouRe = regexp.MustCompile(
`籍贯: ([^<]+) `)
var houseRe = regexp.MustCompile(
`住房条件: ([^<]+) `)
var carRe = regexp.MustCompile(
`是否购车: ([^<]+) `)
var guessRe = regexp.MustCompile(
`]*href="(.*album\.zhenai\.com/u/[\d]+)">([^<]+) `)
var idUrlRe = regexp.MustCompile(
`.*album\.zhenai\.com/u/([\d]+)`)
func parseProfile(
contents []byte, url string,
name string) engine.ParseResult {
profile := model.Profile{}
profile.Name = name
age, err := strconv.Atoi(
extractString(contents, ageRe))
if err == nil {
profile.Age = age
}
height, err := strconv.Atoi(
extractString(contents, heightRe))
if err == nil {
profile.Height = height
}
weight, err := strconv.Atoi(
extractString(contents, weightRe))
if err == nil {
profile.Weight = weight
}
profile.Income = extractString(
contents, incomeRe)
profile.Gender = extractString(
contents, genderRe)
profile.Car = extractString(
contents, carRe)
profile.Education = extractString(
contents, educationRe)
profile.Hokou = extractString(
contents, hokouRe)
profile.House = extractString(
contents, houseRe)
profile.Marriage = extractString(
contents, marriageRe)
profile.Occupation = extractString(
contents, occupationRe)
profile.Xinzuo = extractString(
contents, xinzuoRe)
result := engine.ParseResult{
Items: []engine.Item{
{
Url: url,
Type: "zhenai",
Id: extractString(
[]byte(url), idUrlRe),
Payload: profile,
},
},
}
matches := guessRe.FindAllSubmatch(
contents, -1)
for _, m := range matches {
result.Requests = append(result.Requests,
engine.Request{
Url: string(m[1]),
Parser: NewProfileParser(
string(m[2])),
})
}
return result
}
func extractString(
contents []byte, re *regexp.Regexp) string {
match := re.FindSubmatch(contents)
if len(match) >= 2 {
return string(match[1])
} else {
return ""
}
}
type ProfileParser struct {
userName string
}
func (p *ProfileParser) Parse(
contents []byte,
url string) engine.ParseResult {
return parseProfile(contents, url, p.userName)
}
func (p *ProfileParser) Serialize() (
name string, args interface{}) {
return config.ParseProfile, p.userName
}
func NewProfileParser(
name string) *ProfileParser {
return &ProfileParser{
userName: name,
}
}
================================================
FILE: crawler/zhenai/parser/profile_test.go
================================================
package parser
import (
"io/ioutil"
"testing"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
)
func TestParseProfile(t *testing.T) {
contents, err := ioutil.ReadFile(
"profile_test_data.html")
if err != nil {
panic(err)
}
result := parseProfile(contents,
"http://album.zhenai.com/u/108906739",
"安静的雪")
if len(result.Items) != 1 {
t.Errorf("Items should contain 1 "+
"element; but was %v", result.Items)
}
actual := result.Items[0]
expected := engine.Item{
Url: "http://album.zhenai.com/u/108906739",
Type: "zhenai",
Id: "108906739",
Payload: model.Profile{
Age: 34,
Height: 162,
Weight: 57,
Income: "3001-5000元",
Gender: "女",
Name: "安静的雪",
Xinzuo: "牡羊座",
Occupation: "人事/行政",
Marriage: "离异",
House: "已购房",
Hokou: "山东菏泽",
Education: "大学本科",
Car: "未购车",
},
}
if actual != expected {
t.Errorf("expected %v; but was %v",
expected, actual)
}
}
================================================
FILE: crawler/zhenai/parser/profile_test_data.html
================================================
安静的雪资料照片_新疆阿勒泰征婚交友_珍爱网
年龄: 34岁
身高: 162CM
月收入: 3001-5000元
婚况: 离异
学历: 大学本科
工作地: 新疆阿勒泰
职业: 人事/行政
有无孩子: 有,我们住在一起
籍贯: 山东菏泽
不完美又何妨?万物皆有裂隙,那是光进来的地方。随缘
性别: 女
生肖: --
身高: 162CM
星座: 牡羊座
体重: 57KG
血型: --
体型: --
职业: 人事/行政
民族: 汉族
公司: --
信仰: --
住房条件: 已购房
想何时结婚: --
是否购车: 未购车
婚后与父母住吗: --
是否吸烟: 不吸烟
与对方父母同住: --
是否喝酒: 不喝酒
较大的消费: --
厨艺: --
喜欢怎样的约会: --
家务: --
喜欢的活动: --
喜欢的食物: --
喜欢的体育运动: --
喜欢的地方: --
喜欢的音乐: --
喜欢的宠物: --
喜欢的影视节目: --
性别: 男
体型: 不限
年龄: 35 - 43岁
职业: 不限
身高: 173 - 185厘米
是否抽烟: 不吸烟
学历: 不限
是否喝酒: 不限
月收入: 5000元以上
有没有孩子: 不限
婚况: 不限
是否想要孩子: 不限
工作地区: 新疆阿勒泰
是否有照片: 不限
蝴蝶在飞舞
31岁 162cm
我是东阳人,喜欢另一半也是东阳或义乌本地人。只要是我喜欢的哪里都好。
燕子
35岁 170cm
有一个女孩在一起生活,顾家,无不良嗜好的男士为伴。耍者玩的勿扰,只想有一家。诚心诚意交往的可以,有一个可以靠靠的肩膀,做一回小女人。携手建立完整温暖的家。
================================================
FILE: crawler_distributed/config/config.go
================================================
package config
const (
ItemSaverRpc = "ItemSaverService.Save"
CrawlServiceRpc = "CrawlService.Process"
)
================================================
FILE: crawler_distributed/main.go
================================================
package main
import (
"errors"
"net/rpc"
"log"
"flag"
"strings"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/scheduler"
"imooc.com/ccmouse/learngo/crawler/zhenai/parser"
itemsaver "imooc.com/ccmouse/learngo/crawler_distributed/persist/client"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
worker "imooc.com/ccmouse/learngo/crawler_distributed/worker/client"
)
var (
itemSaverHost = flag.String(
"itemsaver_host", "", "itemsaver host")
workerHosts = flag.String(
"worker_hosts", "",
"worker hosts (comma separated)")
)
func main() {
flag.Parse()
itemChan, err := itemsaver.ItemSaver(
*itemSaverHost)
if err != nil {
panic(err)
}
pool, err := createClientPool(
strings.Split(*workerHosts, ","))
if err != nil {
panic(err)
}
processor := worker.CreateProcessor(pool)
e := engine.ConcurrentEngine{
Scheduler: &scheduler.QueuedScheduler{},
WorkerCount: 100,
ItemChan: itemChan,
RequestProcessor: processor,
}
e.Run(engine.Request{
Url: "http://localhost:8080/mock/www.zhenai.com/zhenghun",
Parser: engine.NewFuncParser(
parser.ParseCityList,
config.ParseCityList),
})
}
func createClientPool(
hosts []string) (chan *rpc.Client, error) {
var clients []*rpc.Client
for _, h := range hosts {
client, err := rpcsupport.NewClient(h)
if err == nil {
clients = append(clients, client)
log.Printf("Connected to %s", h)
} else {
log.Printf(
"Error connecting to %s: %v",
h, err)
}
}
if len(clients) == 0 {
return nil, errors.New(
"no connections available")
}
out := make(chan *rpc.Client)
go func() {
for {
for _, client := range clients {
out <- client
}
}
}()
return out, nil
}
================================================
FILE: crawler_distributed/persist/client/itemsaver.go
================================================
package client
import (
"log"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler_distributed/config"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
)
func ItemSaver(
host string) (chan engine.Item, error) {
client, err := rpcsupport.NewClient(host)
if err != nil {
return nil, err
}
out := make(chan engine.Item)
go func() {
itemCount := 0
for {
item := <-out
log.Printf("Item Saver: got item "+
"#%d: %v", itemCount, item)
itemCount++
// Call RPC to save item
result := ""
err := client.Call(
config.ItemSaverRpc,
item, &result)
if err != nil {
log.Printf("Item Saver: error "+
"saving item %v: %v",
item, err)
}
}
}()
return out, nil
}
================================================
FILE: crawler_distributed/persist/rpc.go
================================================
package persist
import (
"log"
"github.com/olivere/elastic/v7"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/persist"
)
type ItemSaverService struct {
Client *elastic.Client
Index string
}
func (s *ItemSaverService) Save(
item engine.Item, result *string) error {
err := persist.Save(s.Client, s.Index, item)
log.Printf("Item %v saved.", item)
if err == nil {
*result = "ok"
} else {
log.Printf("Error saving item %v: %v",
item, err)
}
return err
}
================================================
FILE: crawler_distributed/persist/server/client_test.go
================================================
package main
import (
"testing"
"time"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
"imooc.com/ccmouse/learngo/crawler_distributed/config"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
)
func TestItemSaver(t *testing.T) {
const host = ":1234"
// start ItemSaverServer
go serveRpc(host, "test1")
time.Sleep(time.Second)
// start ItemSaverClient
client, err := rpcsupport.NewClient(host)
if err != nil {
panic(err)
}
// Call save
item := engine.Item{
Url: "http://album.zhenai.com/u/108906739",
Type: "zhenai",
Id: "108906739",
Payload: model.Profile{
Age: 34,
Height: 162,
Weight: 57,
Income: "3001-5000元",
Gender: "女",
Name: "安静的雪",
Xinzuo: "牡羊座",
Occupation: "人事/行政",
Marriage: "离异",
House: "已购房",
Hokou: "山东菏泽",
Education: "大学本科",
Car: "未购车",
},
}
result := ""
err = client.Call(config.ItemSaverRpc,
item, &result)
if err != nil || result != "ok" {
t.Errorf("result: %s; err: %s",
result, err)
}
}
================================================
FILE: crawler_distributed/persist/server/itemsaver.go
================================================
package main
import (
"flag"
"fmt"
"log"
"github.com/olivere/elastic/v7"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler_distributed/persist"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
)
var port = flag.Int("port", 0,
"the port for me to listen on")
func main() {
flag.Parse()
if *port == 0 {
fmt.Println("must specify a port")
return
}
log.Fatal(serveRpc(
fmt.Sprintf(":%d", *port),
config.ElasticIndex))
}
func serveRpc(host, index string) error {
client, err := elastic.NewClient(
elastic.SetSniff(false))
if err != nil {
return err
}
return rpcsupport.ServeRpc(host,
&persist.ItemSaverService{
Client: client,
Index: index,
})
}
================================================
FILE: crawler_distributed/rpcsupport/rpc.go
================================================
package rpcsupport
import (
"log"
"net"
"net/rpc"
"net/rpc/jsonrpc"
)
func ServeRpc(
host string, service interface{}) error {
rpc.Register(service)
listener, err := net.Listen("tcp", host)
if err != nil {
return err
}
log.Printf("Listening on %s", host)
for {
conn, err := listener.Accept()
if err != nil {
log.Printf("accept error: %v", err)
continue
}
go jsonrpc.ServeConn(conn)
}
}
func NewClient(host string) (*rpc.Client, error) {
conn, err := net.Dial("tcp", host)
if err != nil {
return nil, err
}
return jsonrpc.NewClient(conn), nil
}
================================================
FILE: crawler_distributed/worker/client/worker.go
================================================
package client
import (
"net/rpc"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler_distributed/config"
"imooc.com/ccmouse/learngo/crawler_distributed/worker"
)
func CreateProcessor(
clientChan chan *rpc.Client) engine.Processor {
return func(
req engine.Request) (
engine.ParseResult, error) {
sReq := worker.SerializeRequest(req)
var sResult worker.ParseResult
c := <-clientChan
err := c.Call(config.CrawlServiceRpc,
sReq, &sResult)
if err != nil {
return engine.ParseResult{}, err
}
return worker.DeserializeResult(sResult),
nil
}
}
================================================
FILE: crawler_distributed/worker/rpc.go
================================================
package worker
import "imooc.com/ccmouse/learngo/crawler/engine"
type CrawlService struct{}
func (CrawlService) Process(
req Request, result *ParseResult) error {
engineReq, err := DeserializeRequest(req)
if err != nil {
return err
}
engineResult, err := engine.Worker(engineReq)
if err != nil {
return err
}
*result = SerializeResult(engineResult)
return nil
}
================================================
FILE: crawler_distributed/worker/server/client_test.go
================================================
package main
import (
"fmt"
"testing"
"time"
"imooc.com/ccmouse/learngo/crawler/config"
rpcnames "imooc.com/ccmouse/learngo/crawler_distributed/config"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
"imooc.com/ccmouse/learngo/crawler_distributed/worker"
)
func TestCrawlService(t *testing.T) {
const host = ":9000"
go rpcsupport.ServeRpc(
host, worker.CrawlService{})
time.Sleep(time.Second)
client, err := rpcsupport.NewClient(host)
if err != nil {
panic(err)
}
// TODO: Use a fake fetcher to handle the url.
// So we don't get data from zhenai.com
req := worker.Request{
Url: "http://album.zhenai.com/u/108906739",
Parser: worker.SerializedParser{
Name: config.ParseProfile,
Args: "安静的雪",
},
}
var result worker.ParseResult
err = client.Call(
rpcnames.CrawlServiceRpc, req, &result)
if err != nil {
t.Error(err)
} else {
fmt.Println(result)
}
// TODO: Verify results
}
================================================
FILE: crawler_distributed/worker/server/worker.go
================================================
package main
import (
"fmt"
"log"
"flag"
"imooc.com/ccmouse/learngo/crawler/fetcher"
"imooc.com/ccmouse/learngo/crawler_distributed/rpcsupport"
"imooc.com/ccmouse/learngo/crawler_distributed/worker"
)
var port = flag.Int("port", 0,
"the port for me to listen on")
func main() {
flag.Parse()
fetcher.SetVerboseLogging()
if *port == 0 {
fmt.Println("must specify a port")
return
}
log.Fatal(rpcsupport.ServeRpc(
fmt.Sprintf(":%d", *port),
worker.CrawlService{}))
}
================================================
FILE: crawler_distributed/worker/types.go
================================================
package worker
import (
"errors"
"fmt"
"log"
"imooc.com/ccmouse/learngo/crawler/config"
"imooc.com/ccmouse/learngo/crawler/engine"
xcar "imooc.com/ccmouse/learngo/crawler/xcar/parser"
zhenai "imooc.com/ccmouse/learngo/crawler/zhenai/parser"
)
type SerializedParser struct {
Name string
Args interface{}
}
type Request struct {
Url string
Parser SerializedParser
}
type ParseResult struct {
Items []engine.Item
Requests []Request
}
func SerializeRequest(r engine.Request) Request {
name, args := r.Parser.Serialize()
return Request{
Url: r.Url,
Parser: SerializedParser{
Name: name,
Args: args,
},
}
}
func SerializeResult(
r engine.ParseResult) ParseResult {
result := ParseResult{
Items: r.Items,
}
for _, req := range r.Requests {
result.Requests = append(result.Requests,
SerializeRequest(req))
}
return result
}
func DeserializeRequest(
r Request) (engine.Request, error) {
parser, err := deserializeParser(r.Parser)
if err != nil {
return engine.Request{}, err
}
return engine.Request{
Url: r.Url,
Parser: parser,
}, nil
}
func DeserializeResult(
r ParseResult) engine.ParseResult {
result := engine.ParseResult{
Items: r.Items,
}
for _, req := range r.Requests {
engineReq, err := DeserializeRequest(req)
if err != nil {
log.Printf("error deserializing "+
"request: %v", err)
continue
}
result.Requests = append(result.Requests,
engineReq)
}
return result
}
func deserializeParser(
p SerializedParser) (engine.Parser, error) {
switch p.Name {
case config.ParseCityList:
return engine.NewFuncParser(
zhenai.ParseCityList,
config.ParseCityList), nil
case config.ParseCity:
return engine.NewFuncParser(
zhenai.ParseCity,
config.ParseCity), nil
case config.ParseProfile:
if userName, ok := p.Args.(string); ok {
return zhenai.NewProfileParser(
userName), nil
} else {
return nil, fmt.Errorf("invalid "+
"arg: %v", p.Args)
}
case config.ParseCarDetail:
return engine.NewFuncParser(
xcar.ParseCarDetail,
config.ParseCarDetail), nil
case config.ParseCarModel:
return engine.NewFuncParser(
xcar.ParseCarModel,
config.ParseCarModel), nil
case config.ParseCarList:
return engine.NewFuncParser(
xcar.ParseCarList,
config.ParseCarList), nil
case config.NilParser:
return engine.NilParser{}, nil
default:
return nil, errors.New(
"unknown parser name")
}
}
================================================
FILE: go.mod
================================================
module imooc.com/ccmouse/learngo
go 1.13
require (
github.com/gin-gonic/gin v1.9.1
github.com/google/go-cmp v0.6.0
github.com/olivere/elastic/v7 v7.0.8
go.uber.org/zap v1.12.0
golang.org/x/net v0.38.0
golang.org/x/text v0.23.0
)
================================================
FILE: go.sum
================================================
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.19.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/olivere/elastic v6.2.23+incompatible h1:oRGUA/8fKcnkDcqLuwGb5YCzgbgEBo+Y9gamsWqZ0qU=
github.com/olivere/elastic v6.2.23+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/olivere/elastic/v7 v7.0.8 h1:tp9BHGFilpoH7O7fQOwWiXJQFJkl9PZJvUwO74OvfKc=
github.com/olivere/elastic/v7 v7.0.8/go.mod h1:UcXCjbh5xfX9uMB1VCcIYgGJBItbd4uRBdYRsBnnXHo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw=
go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
================================================
FILE: lang/basic/atomic/atomic.go
================================================
package main
import (
"fmt"
"sync"
"time"
)
type atomicInt struct {
value int
lock sync.Mutex
}
func (a *atomicInt) increment() {
fmt.Println("safe increment")
func() {
a.lock.Lock()
defer a.lock.Unlock()
a.value++
}()
}
func (a *atomicInt) get() int {
a.lock.Lock()
defer a.lock.Unlock()
return a.value
}
func main() {
var a atomicInt
a.increment()
go func() {
a.increment()
}()
time.Sleep(time.Millisecond)
fmt.Println(a.get())
}
================================================
FILE: lang/basic/basic/basic.go
================================================
package main
import (
"fmt"
"math"
"math/cmplx"
)
var (
aa = 3
ss = "kkk"
bb = true
)
func variableZeroValue() {
var a int
var s string
fmt.Printf("%d %q\n", a, s)
}
func variableInitialValue() {
var a, b int = 3, 4
var s string = "abc"
fmt.Println(a, b, s)
}
func variableTypeDeduction() {
var a, b, c, s = 3, 4, true, "def"
fmt.Println(a, b, c, s)
}
func variableShorter() {
a, b, c, s := 3, 4, true, "def"
b = 5
fmt.Println(a, b, c, s)
}
func euler() {
fmt.Printf("%.3f\n",
cmplx.Exp(1i*math.Pi)+1)
}
func triangle() {
var a, b int = 3, 4
fmt.Println(calcTriangle(a, b))
}
func calcTriangle(a, b int) int {
var c int
c = int(math.Sqrt(float64(a*a + b*b)))
return c
}
func consts() {
const (
filename = "abc.txt"
a, b = 3, 4
)
var c int
c = int(math.Sqrt(a*a + b*b))
fmt.Println(filename, c)
}
func enums() {
const (
cpp = iota
_
python
golang
javascript
)
const (
b = 1 << (10 * iota)
kb
mb
gb
tb
pb
)
fmt.Println(cpp, javascript, python, golang)
fmt.Println(b, kb, mb, gb, tb, pb)
}
func main() {
fmt.Println("Hello world")
variableZeroValue()
variableInitialValue()
variableTypeDeduction()
variableShorter()
fmt.Println(aa, ss, bb)
euler()
triangle()
consts()
enums()
}
================================================
FILE: lang/basic/basic/triangle_test.go
================================================
package main
import "testing"
func TestTriangle(t *testing.T) {
tests := []struct{ a, b, c int }{
{3, 4, 5},
{5, 12, 13},
{8, 15, 17},
{12, 35, 37},
{30000, 40000, 50000},
}
for _, tt := range tests {
if actual := calcTriangle(tt.a, tt.b); actual != tt.c {
t.Errorf("calcTriangle(%d, %d); "+
"got %d; expected %d",
tt.a, tt.b, actual, tt.c)
}
}
}
================================================
FILE: lang/basic/branch/abc.txt
================================================
abcde
12345
hello
again
================================================
FILE: lang/basic/branch/branch.go
================================================
package main
import (
"fmt"
"io/ioutil"
)
func grade(score int) string {
g := ""
switch {
case score < 0 || score > 100:
panic(fmt.Sprintf(
"Wrong score: %d", score))
case score < 60:
g = "F"
case score < 80:
g = "C"
case score < 90:
g = "B"
case score <= 100:
g = "A"
}
return g
}
func main() {
// If "abc.txt" is not found,
// please check what current directory is,
// and change filename accordingly.
const filename = "abc.txt"
if contents, err := ioutil.ReadFile(filename); err != nil {
fmt.Println(err)
} else {
fmt.Printf("%s\n", contents)
}
fmt.Println(
grade(0),
grade(59),
grade(60),
grade(82),
grade(99),
grade(100),
// Uncomment to see it panics.
// grade(-3),
)
}
================================================
FILE: lang/basic/func/func.go
================================================
package main
import (
"fmt"
"math"
"reflect"
"runtime"
)
func eval(a, b int, op string) (int, error) {
switch op {
case "+":
return a + b, nil
case "-":
return a - b, nil
case "*":
return a * b, nil
case "/":
q, _ := div(a, b)
return q, nil
default:
return 0, fmt.Errorf(
"unsupported operation: %s", op)
}
}
func div(a, b int) (q, r int) {
return a / b, a % b
}
func apply(op func(int, int) int, a, b int) int {
p := reflect.ValueOf(op).Pointer()
opName := runtime.FuncForPC(p).Name()
fmt.Printf("Calling function %s with args "+
"(%d, %d)\n", opName, a, b)
return op(a, b)
}
func sum(numbers ...int) int {
s := 0
for i := range numbers {
s += numbers[i]
}
return s
}
func swap(a, b int) (int, int) {
return b, a
}
func main() {
fmt.Println("Error handling")
if result, err := eval(3, 4, "x"); err != nil {
fmt.Println("Error:", err)
} else {
fmt.Println(result)
}
q, r := div(13, 3)
fmt.Printf("13 div 3 is %d mod %d\n", q, r)
fmt.Println("pow(3, 4) is:", apply(
func(a int, b int) int {
return int(math.Pow(
float64(a), float64(b)))
}, 3, 4))
fmt.Println("1+2+...+5 =", sum(1, 2, 3, 4, 5))
a, b := 3, 4
a, b = swap(a, b)
fmt.Println("a, b after swap is:", a, b)
}
================================================
FILE: lang/basic/loop/loop.go
================================================
package main
import (
"bufio"
"fmt"
"io"
"os"
"strconv"
"strings"
)
func convertToBin(n int) string {
result := ""
for ; n > 0; n /= 2 {
lsb := n % 2
result = strconv.Itoa(lsb) + result
}
return result
}
func printFile(filename string) {
file, err := os.Open(filename)
if err != nil {
panic(err)
}
printFileContents(file)
}
func printFileContents(reader io.Reader) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
fmt.Println(scanner.Text())
}
}
func forever() {
for {
fmt.Println("abc")
}
}
func main() {
fmt.Println("convertToBin results:")
fmt.Println(
convertToBin(5), // 101
convertToBin(13), // 1101
convertToBin(72387885),
convertToBin(0),
)
fmt.Println("abc.txt contents:")
printFile("lang/basic/branch/abc.txt")
fmt.Println("printing a string:")
s := `abc"d"
kkkk
123
p`
printFileContents(strings.NewReader(s))
// Uncomment to see it runs forever
// forever()
}
================================================
FILE: lang/basic/regex/regex.go
================================================
package main
import (
"fmt"
"regexp"
)
const text = `
my email is ccmouse@gmail.com@abc.com
email1 is abc@def.org
email2 is kkk@qq.com
email3 is ddd@abc.com.cn
`
func main() {
re := regexp.MustCompile(
`([a-zA-Z0-9]+)@([a-zA-Z0-9]+)(\.[a-zA-Z0-9.]+)`)
match := re.FindAllStringSubmatch(text, -1)
for _, m := range match {
fmt.Println(m)
}
}
================================================
FILE: lang/channel/channel.go
================================================
package main
import (
"fmt"
"time"
)
func worker(id int, c chan int) {
for n := range c {
fmt.Printf("Worker %d received %c\n",
id, n)
}
}
func createWorker(id int) chan<- int {
c := make(chan int)
go worker(id, c)
return c
}
func chanDemo() {
var channels [10]chan<- int
for i := 0; i < 10; i++ {
channels[i] = createWorker(i)
}
for i := 0; i < 10; i++ {
channels[i] <- 'a' + i
}
for i := 0; i < 10; i++ {
channels[i] <- 'A' + i
}
time.Sleep(time.Millisecond)
}
func bufferedChannel() {
c := make(chan int, 3)
go worker(0, c)
c <- 'a'
c <- 'b'
c <- 'c'
c <- 'd'
time.Sleep(time.Millisecond)
}
func channelClose() {
c := make(chan int)
go worker(0, c)
c <- 'a'
c <- 'b'
c <- 'c'
c <- 'd'
close(c)
time.Sleep(time.Millisecond)
}
func main() {
fmt.Println("Channel as first-class citizen")
chanDemo()
fmt.Println("Buffered channel")
bufferedChannel()
fmt.Println("Channel close and range")
channelClose()
}
================================================
FILE: lang/channel/done/done.go
================================================
package main
import (
"fmt"
"sync"
)
func doWork(id int,
w worker) {
for n := range w.in {
fmt.Printf("Worker %d received %c\n",
id, n)
w.done()
}
}
type worker struct {
in chan int
done func()
}
func createWorker(
id int, wg *sync.WaitGroup) worker {
w := worker{
in: make(chan int),
done: func() {
wg.Done()
},
}
go doWork(id, w)
return w
}
func chanDemo() {
var wg sync.WaitGroup
var workers [10]worker
for i := 0; i < 10; i++ {
workers[i] = createWorker(i, &wg)
}
wg.Add(20)
for i, worker := range workers {
worker.in <- 'a' + i
}
for i, worker := range workers {
worker.in <- 'A' + i
}
wg.Wait()
}
func main() {
chanDemo()
}
================================================
FILE: lang/channel/pattern/main.go
================================================
package main
import (
"fmt"
"math/rand"
"time"
)
func msgGen(name string) chan string {
c := make(chan string)
go func() {
i := 0
for {
time.Sleep(time.Duration(rand.Intn(2000)) * time.Millisecond)
c <- fmt.Sprintf("service %s: message %d", name, i)
i++
}
}()
return c
}
func fanIn(chs ...chan string) chan string {
c := make(chan string)
for _, ch := range chs {
go func(in chan string) {
for {
c <- <-in
}
}(ch)
}
return c
}
func fanInBySelect(c1, c2 chan string) chan string {
c := make(chan string)
go func() {
for {
select {
case m := <-c1:
c <- m
case m := <-c2:
c <- m
}
}
}()
return c
}
func main() {
m1 := msgGen("service1")
m2 := msgGen("service2")
m3 := msgGen("service3")
m := fanIn(m1, m2, m3)
for {
fmt.Println(<-m)
}
}
================================================
FILE: lang/channel/select/select.go
================================================
package main
import (
"fmt"
"math/rand"
"time"
)
func generator() chan int {
out := make(chan int)
go func() {
i := 0
for {
time.Sleep(
time.Duration(rand.Intn(1500)) *
time.Millisecond)
out <- i
i++
}
}()
return out
}
func worker(id int, c chan int) {
for n := range c {
time.Sleep(time.Second)
fmt.Printf("Worker %d received %d\n",
id, n)
}
}
func createWorker(id int) chan<- int {
c := make(chan int)
go worker(id, c)
return c
}
func main() {
var c1, c2 = generator(), generator()
var worker = createWorker(0)
var values []int
tm := time.After(10 * time.Second)
tick := time.Tick(time.Second)
for {
var activeWorker chan<- int
var activeValue int
if len(values) > 0 {
activeWorker = worker
activeValue = values[0]
}
select {
case n := <-c1:
values = append(values, n)
case n := <-c2:
values = append(values, n)
case activeWorker <- activeValue:
values = values[1:]
case <-time.After(800 * time.Millisecond):
fmt.Println("timeout")
case <-tick:
fmt.Println(
"queue len =", len(values))
case <-tm:
fmt.Println("bye")
return
}
}
}
================================================
FILE: lang/container/arrays/arrays.go
================================================
package main
import "fmt"
func printArray(arr [5]int) {
arr[0] = 100
for i, v := range arr {
fmt.Println(i, v)
}
}
func main() {
var arr1 [5]int
arr2 := [3]int{1, 3, 5}
arr3 := [...]int{2, 4, 6, 8, 10}
var grid [4][5]int
fmt.Println("array definitions:")
fmt.Println(arr1, arr2, arr3)
fmt.Println(grid)
fmt.Println("printArray(arr1)")
printArray(arr1)
fmt.Println("printArray(arr3)")
printArray(arr3)
fmt.Println("arr1 and arr3")
fmt.Println(arr1, arr3)
}
================================================
FILE: lang/container/maps/maps.go
================================================
package main
import "fmt"
func main() {
m := map[string]string{
"name": "ccmouse",
"course": "golang",
"site": "imooc",
"quality": "notbad",
}
m2 := make(map[string]int) // m2 == empty map
var m3 map[string]int // m3 == nil
fmt.Println("m, m2, m3:")
fmt.Println(m, m2, m3)
fmt.Println("Traversing map m")
for k, v := range m {
fmt.Println(k, v)
}
fmt.Println("Getting values")
courseName := m["course"]
fmt.Println(`m["course"] =`, courseName)
if causeName, ok := m["cause"]; ok {
fmt.Println(causeName)
} else {
fmt.Println("key 'cause' does not exist")
}
fmt.Println("Deleting values")
name, ok := m["name"]
fmt.Printf("m[%q] before delete: %q, %v\n",
"name", name, ok)
delete(m, "name")
name, ok = m["name"]
fmt.Printf("m[%q] after delete: %q, %v\n",
"name", name, ok)
}
================================================
FILE: lang/container/nonrepeatingsubstr/nonrepeating.go
================================================
package main
import (
"fmt"
)
func lengthOfNonRepeatingSubStr(s string) int {
lastOccurred := make(map[rune]int)
start := 0
maxLength := 0
for i, ch := range []rune(s) {
if lastI, ok := lastOccurred[ch]; ok && lastI >= start {
start = lastI + 1
}
if i-start+1 > maxLength {
maxLength = i - start + 1
}
lastOccurred[ch] = i
}
return maxLength
}
func main() {
fmt.Println(
lengthOfNonRepeatingSubStr("abcabcbb"))
fmt.Println(
lengthOfNonRepeatingSubStr("bbbbb"))
fmt.Println(
lengthOfNonRepeatingSubStr("pwwkew"))
fmt.Println(
lengthOfNonRepeatingSubStr(""))
fmt.Println(
lengthOfNonRepeatingSubStr("b"))
fmt.Println(
lengthOfNonRepeatingSubStr("abcdef"))
fmt.Println(
lengthOfNonRepeatingSubStr("这里是慕课网"))
fmt.Println(
lengthOfNonRepeatingSubStr("一二三二一"))
fmt.Println(
lengthOfNonRepeatingSubStr(
"黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花"))
}
================================================
FILE: lang/container/nonrepeatingsubstr/nonrepeating_test.go
================================================
package main
import "testing"
func TestSubstr(t *testing.T) {
tests := []struct {
s string
ans int
}{
// Normal cases
{"abcabcbb", 3},
{"pwwkew", 3},
// Edge cases
{"", 0},
{"b", 1},
{"bbbbbbbbb", 1},
{"abcabcabcd", 4},
// Chinese support
{"这里是慕课网", 6},
{"一二三二一", 3},
{"黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花", 8},
}
for _, tt := range tests {
actual := lengthOfNonRepeatingSubStr(tt.s)
if actual != tt.ans {
t.Errorf("got %d for input %s; "+
"expected %d",
actual, tt.s, tt.ans)
}
}
}
func BenchmarkSubstr(b *testing.B) {
s := "黑化肥挥发发灰会花飞灰化肥挥发发黑会飞花"
for i := 0; i < 13; i++ {
s = s + s
}
b.Logf("len(s) = %d", len(s))
ans := 8
b.ResetTimer()
for i := 0; i < b.N; i++ {
actual := lengthOfNonRepeatingSubStr(s)
if actual != ans {
b.Errorf("got %d for input %s; "+
"expected %d",
actual, s, ans)
}
}
}
================================================
FILE: lang/container/slices/sliceops.go
================================================
package main
import "fmt"
func printSlice(s []int) {
fmt.Printf("%v, len=%d, cap=%d\n",
s, len(s), cap(s))
}
func sliceOps() {
fmt.Println("Creating slice")
var s []int // Zero value for slice is nil
for i := 0; i < 100; i++ {
printSlice(s)
s = append(s, 2*i+1)
}
fmt.Println(s)
s1 := []int{2, 4, 6, 8}
printSlice(s1)
s2 := make([]int, 16)
s3 := make([]int, 10, 32)
printSlice(s2)
printSlice(s3)
fmt.Println("Copying slice")
copy(s2, s1)
printSlice(s2)
fmt.Println("Deleting elements from slice")
s2 = append(s2[:3], s2[4:]...)
printSlice(s2)
fmt.Println("Popping from front")
front := s2[0]
s2 = s2[1:]
fmt.Println(front)
printSlice(s2)
fmt.Println("Popping from back")
tail := s2[len(s2)-1]
s2 = s2[:len(s2)-1]
fmt.Println(tail)
printSlice(s2)
}
================================================
FILE: lang/container/slices/slices.go
================================================
package main
import "fmt"
func updateSlice(s []int) {
s[0] = 100
}
func main() {
arr := [...]int{0, 1, 2, 3, 4, 5, 6, 7}
fmt.Println("arr[2:6] =", arr[2:6])
fmt.Println("arr[:6] =", arr[:6])
s1 := arr[2:]
fmt.Println("s1 =", s1)
s2 := arr[:]
fmt.Println("s2 =", s2)
fmt.Println("After updateSlice(s1)")
updateSlice(s1)
fmt.Println(s1)
fmt.Println(arr)
fmt.Println("After updateSlice(s2)")
updateSlice(s2)
fmt.Println(s2)
fmt.Println(arr)
fmt.Println("Reslice")
fmt.Println(s2)
s2 = s2[:5]
fmt.Println(s2)
s2 = s2[2:]
fmt.Println(s2)
fmt.Println("Extending slice")
arr[0], arr[2] = 0, 2
fmt.Println("arr =", arr)
s1 = arr[2:6]
s2 = s1[3:5] // [s1[3], s1[4]]
fmt.Printf("s1=%v, len(s1)=%d, cap(s1)=%d\n",
s1, len(s1), cap(s1))
fmt.Printf("s2=%v, len(s2)=%d, cap(s2)=%d\n",
s2, len(s2), cap(s2))
s3 := append(s2, 10)
s4 := append(s3, 11)
s5 := append(s4, 12)
fmt.Println("s3, s4, s5 =", s3, s4, s5)
// s4 and s5 no longer view arr.
fmt.Println("arr =", arr)
// Uncomment to run sliceOps demo.
// If we see undefined: sliceOps
// please try go run slices.go sliceops.go
fmt.Println("Uncomment to see sliceOps demo")
// sliceOps()
}
================================================
FILE: lang/container/strings/strings.go
================================================
package main
import (
"fmt"
"unicode/utf8"
)
func main() {
s := "Yes我爱慕课网!" // UTF-8
fmt.Println(s)
for _, b := range []byte(s) {
fmt.Printf("%X ", b)
}
fmt.Println()
for i, ch := range s { // ch is a rune
fmt.Printf("(%d %X) ", i, ch)
}
fmt.Println()
fmt.Println("Rune count:",
utf8.RuneCountInString(s))
bytes := []byte(s)
for len(bytes) > 0 {
ch, size := utf8.DecodeRune(bytes)
bytes = bytes[size:]
fmt.Printf("%c ", ch)
}
fmt.Println()
for i, ch := range []rune(s) {
fmt.Printf("(%d %c) ", i, ch)
}
fmt.Println()
}
================================================
FILE: lang/errhandling/defer/defer.go
================================================
package main
import (
"fmt"
"os"
"bufio"
"imooc.com/ccmouse/learngo/lang/functional/fib"
)
func tryDefer() {
for i := 0; i < 100; i++ {
defer fmt.Println(i)
if i == 30 {
// Uncomment panic to see
// how it works with defer
// panic("printed too many")
}
}
}
func writeFile(filename string) {
file, err := os.OpenFile(filename,
os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
if pathError, ok := err.(*os.PathError); !ok {
panic(err)
} else {
fmt.Printf("%s, %s, %s\n",
pathError.Op,
pathError.Path,
pathError.Err)
}
return
}
defer file.Close()
writer := bufio.NewWriter(file)
defer writer.Flush()
f := fib.Fibonacci()
for i := 0; i < 20; i++ {
fmt.Fprintln(writer, f())
}
}
func main() {
tryDefer()
writeFile("fib.txt")
}
================================================
FILE: lang/errhandling/filelistingserver/errwrapper_test.go
================================================
package main
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
)
func errPanic(_ http.ResponseWriter,
_ *http.Request) error {
panic(123)
}
type testingUserError string
func (e testingUserError) Error() string {
return e.Message()
}
func (e testingUserError) Message() string {
return string(e)
}
func errUserError(_ http.ResponseWriter,
_ *http.Request) error {
return testingUserError("user error")
}
func errNotFound(_ http.ResponseWriter,
_ *http.Request) error {
return os.ErrNotExist
}
func errNoPermission(_ http.ResponseWriter,
_ *http.Request) error {
return os.ErrPermission
}
func errUnknown(_ http.ResponseWriter,
_ *http.Request) error {
return errors.New("unknown error")
}
func noError(writer http.ResponseWriter,
_ *http.Request) error {
fmt.Fprintln(writer, "no error")
return nil
}
var tests = []struct {
h appHandler
code int
message string
}{
{errPanic, 500, "Internal Server Error"},
{errUserError, 400, "user error"},
{errNotFound, 404, "Not Found"},
{errNoPermission, 403, "Forbidden"},
{errUnknown, 500, "Internal Server Error"},
{noError, 200, "no error"},
}
func TestErrWrapper(t *testing.T) {
for _, tt := range tests {
f := errWrapper(tt.h)
response := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"http://www.imooc.com", nil)
f(response, request)
verifyResponse(response.Result(),
tt.code, tt.message, t)
}
}
func TestErrWrapperInServer(t *testing.T) {
for _, tt := range tests {
f := errWrapper(tt.h)
server := httptest.NewServer(
http.HandlerFunc(f))
resp, _ := http.Get(server.URL)
verifyResponse(
resp, tt.code, tt.message, t)
}
}
func verifyResponse(resp *http.Response,
expectedCode int, expectedMsg string,
t *testing.T) {
b, _ := ioutil.ReadAll(resp.Body)
body := strings.Trim(string(b), "\n")
if resp.StatusCode != expectedCode ||
body != expectedMsg {
t.Errorf("expect (%d, %s); "+
"got (%d, %s)",
expectedCode, expectedMsg,
resp.StatusCode, body)
}
}
================================================
FILE: lang/errhandling/filelistingserver/filelisting/handler.go
================================================
package filelisting
import (
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
)
const prefix = "/list/"
type userError string
func (e userError) Error() string {
return e.Message()
}
func (e userError) Message() string {
return string(e)
}
func HandleFileList(writer http.ResponseWriter,
request *http.Request) error {
fmt.Println()
if strings.Index(
request.URL.Path, prefix) != 0 {
return userError(
fmt.Sprintf("path %s must start "+
"with %s",
request.URL.Path, prefix))
}
path := request.URL.Path[len(prefix):]
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
all, err := ioutil.ReadAll(file)
if err != nil {
return err
}
writer.Write(all)
return nil
}
================================================
FILE: lang/errhandling/filelistingserver/web.go
================================================
package main
import (
"log"
"net/http"
_ "net/http/pprof"
"os"
"imooc.com/ccmouse/learngo/lang/errhandling/filelistingserver/filelisting"
)
type appHandler func(writer http.ResponseWriter,
request *http.Request) error
func errWrapper(
handler appHandler) func(
http.ResponseWriter, *http.Request) {
return func(writer http.ResponseWriter,
request *http.Request) {
// panic
defer func() {
if r := recover(); r != nil {
log.Printf("Panic: %v", r)
http.Error(writer,
http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
}
}()
err := handler(writer, request)
if err != nil {
log.Printf("Error occurred "+
"handling request: %s",
err.Error())
// user error
if userErr, ok := err.(userError); ok {
http.Error(writer,
userErr.Message(),
http.StatusBadRequest)
return
}
// system error
code := http.StatusOK
switch {
case os.IsNotExist(err):
code = http.StatusNotFound
case os.IsPermission(err):
code = http.StatusForbidden
default:
code = http.StatusInternalServerError
}
http.Error(writer,
http.StatusText(code), code)
}
}
}
type userError interface {
error
Message() string
}
func main() {
http.HandleFunc("/",
errWrapper(filelisting.HandleFileList))
err := http.ListenAndServe(":8888", nil)
if err != nil {
panic(err)
}
}
================================================
FILE: lang/errhandling/recover/recover.go
================================================
package main
import (
"fmt"
)
func tryRecover() {
defer func() {
r := recover()
if r == nil {
fmt.Println("Nothing to recover. " +
"Please try uncomment errors " +
"below.")
return
}
if err, ok := r.(error); ok {
fmt.Println("Error occurred:", err)
} else {
panic(fmt.Sprintf(
"I don't know what to do: %v", r))
}
}()
// Uncomment each block to see different panic
// scenarios.
// Normal error
//panic(errors.New("this is an error"))
// Division by zero
//b := 0
//a := 5 / b
//fmt.Println(a)
// Causes re-panic
//panic(123)
}
func main() {
tryRecover()
}
================================================
FILE: lang/functional/adder/adder.go
================================================
package main
import "fmt"
func adder() func(int) int {
sum := 0
return func(v int) int {
sum += v
return sum
}
}
type iAdder func(int) (int, iAdder)
func adder2(base int) iAdder {
return func(v int) (int, iAdder) {
return base + v, adder2(base + v)
}
}
func main() {
// a := adder() is trivial and also works.
a := adder2(0)
for i := 0; i < 10; i++ {
var s int
s, a = a(i)
fmt.Printf("0 + 1 + ... + %d = %d\n",
i, s)
}
}
================================================
FILE: lang/functional/fib/fib.go
================================================
package fib
// 1, 1, 2, 3, 5, 8, 13, ...
func Fibonacci() func() int {
a, b := 0, 1
return func() int {
a, b = b, a+b
return a
}
}
================================================
FILE: lang/functional/main.go
================================================
package main
import (
"bufio"
"fmt"
"io"
"strings"
"imooc.com/ccmouse/learngo/lang/functional/fib"
)
type intGen func() int
func (g intGen) Read(
p []byte) (n int, err error) {
next := g()
if next > 10000 {
return 0, io.EOF
}
s := fmt.Sprintf("%d\n", next)
// TODO: incorrect if p is too small!
return strings.NewReader(s).Read(p)
}
func printFileContents(reader io.Reader) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
fmt.Println(scanner.Text())
}
}
func main() {
var f intGen = fib.Fibonacci()
printFileContents(f)
}
================================================
FILE: lang/goroutine/goroutine.go
================================================
package main
import (
"fmt"
"time"
)
func main() {
for i := 0; i < 1000; i++ {
go func(i int) {
for {
fmt.Printf("Hello from "+
"goroutine %d\n", i)
}
}(i)
}
time.Sleep(time.Minute)
}
================================================
FILE: lang/http/client.go
================================================
package main
import (
"fmt"
"net/http"
"net/http/httputil"
)
func main() {
request, err := http.NewRequest(
http.MethodGet,
"http://www.imooc.com", nil)
request.Header.Add("User-Agent",
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1")
client := http.Client{
CheckRedirect: func(
req *http.Request,
via []*http.Request) error {
fmt.Println("Redirect:", req)
return nil
},
}
resp, err := client.Do(request)
if err != nil {
panic(err)
}
defer resp.Body.Close()
s, err := httputil.DumpResponse(resp, true)
if err != nil {
panic(err)
}
fmt.Printf("%s\n", s)
}
================================================
FILE: lang/http/gindemo/ginserver.go
================================================
package main
import (
"github.com/gin-gonic/gin"
"go.uber.org/zap"
"math/rand"
"time"
)
const keyRequestId = "requestId"
func main() {
r := gin.Default()
logger, err := zap.NewProduction()
if err != nil {
panic(err)
}
r.Use(func(c *gin.Context) {
s := time.Now()
c.Next()
logger.Info("incoming request",
zap.String("path", c.Request.URL.Path),
zap.Int("status", c.Writer.Status()),
zap.Duration("elapsed", time.Now().Sub(s)))
}, func(c *gin.Context) {
c.Set(keyRequestId, rand.Int())
c.Next()
})
r.GET("/ping", func(c *gin.Context) {
h := gin.H{
"message": "pong",
}
if rid, exists := c.Get(keyRequestId); exists {
h[keyRequestId] = rid
}
c.JSON(200, h)
})
r.GET("/hello", func(c *gin.Context) {
c.String(200, "hello")
})
r.Run()
}
================================================
FILE: lang/json/main.go
================================================
package main
import (
"encoding/json"
"fmt"
)
type OrderItem struct {
ID string `json:"id"`
Name string `json:"name"`
Price float64 `json:"price"`
}
type Order struct {
ID string `json:"id"`
Items []OrderItem `json:"items"`
TotalPrice float64 `json:"total_price"`
}
func main() {
parseNLP()
}
func marshal() {
o := Order{
ID: "1234",
TotalPrice: 20,
Items: []OrderItem{
{
ID: "item_1",
Name: "learn go",
Price: 15,
},
{
ID: "item_2",
Name: "interview",
Price: 10,
},
},
}
b, err := json.Marshal(o)
if err != nil {
panic(err)
}
fmt.Printf("%s\n", b)
}
func unmarshal() {
s := `{"id":"1234","items":[{"id":"item_1","name":"learn go","price":15},{"id":"item_2","name":"interview","price":10}],"total_price":20}`
var o Order
err := json.Unmarshal([]byte(s), &o)
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", o)
}
func parseNLP() {
res := `{
"data": [
{
"synonym":"",
"weight":"0.6",
"word": "真丝",
"tag":"材质"
},
{
"synonym":"",
"weight":"0.8",
"word": "韩都衣舍",
"tag":"品牌"
},
{
"synonym":"连身裙;联衣裙",
"weight":"1.0",
"word": "连衣裙",
"tag":"品类"
}
]
}`
m := struct {
Data []struct {
Synonym string `json:"synonym"`
Tag string `json:"tag"`
} `json:"data"`
}{}
err := json.Unmarshal([]byte(res), &m)
if err != nil {
panic(err)
}
fmt.Printf("%+v, %+v\n", m.Data[2].Synonym, m.Data[2].Tag)
}
================================================
FILE: lang/maze/maze.go
================================================
package main
import (
"fmt"
"os"
)
func readMaze(filename string) [][]int {
file, err := os.Open(filename)
if err != nil {
panic(err)
}
var row, col int
fmt.Fscanf(file, "%d %d", &row, &col)
maze := make([][]int, row)
for i := range maze {
maze[i] = make([]int, col)
for j := range maze[i] {
fmt.Fscanf(file, "%d", &maze[i][j])
}
}
return maze
}
type point struct {
i, j int
}
var dirs = [4]point{
{-1, 0}, {0, -1}, {1, 0}, {0, 1}}
func (p point) add(r point) point {
return point{p.i + r.i, p.j + r.j}
}
func (p point) at(grid [][]int) (int, bool) {
if p.i < 0 || p.i >= len(grid) {
return 0, false
}
if p.j < 0 || p.j >= len(grid[p.i]) {
return 0, false
}
return grid[p.i][p.j], true
}
func walk(maze [][]int,
start, end point) [][]int {
steps := make([][]int, len(maze))
for i := range steps {
steps[i] = make([]int, len(maze[i]))
}
Q := []point{start}
for len(Q) > 0 {
cur := Q[0]
Q = Q[1:]
if cur == end {
break
}
for _, dir := range dirs {
next := cur.add(dir)
val, ok := next.at(maze)
if !ok || val == 1 {
continue
}
val, ok = next.at(steps)
if !ok || val != 0 {
continue
}
if next == start {
continue
}
curSteps, _ := cur.at(steps)
steps[next.i][next.j] =
curSteps + 1
Q = append(Q, next)
}
}
return steps
}
func main() {
maze := readMaze("lang/maze/maze.in")
steps := walk(maze, point{0, 0},
point{len(maze) - 1, len(maze[0]) - 1})
for _, row := range steps {
for _, val := range row {
fmt.Printf("%3d", val)
}
fmt.Println()
}
// TODO: construct path from steps
}
================================================
FILE: lang/maze/maze.in
================================================
6 5
0 1 0 0 0
0 0 0 1 0
0 1 0 1 0
1 1 1 0 0
0 1 0 0 1
0 1 0 0 0
================================================
FILE: lang/queue/queue.go
================================================
package queue
// A FIFO queue.
type Queue []int
// Pushes the element into the queue.
//
// e.g. q.Push(123)
func (q *Queue) Push(v int) {
*q = append(*q, v)
}
// Pops element from head.
func (q *Queue) Pop() int {
head := (*q)[0]
*q = (*q)[1:]
return head
}
// Returns if the queue is empty or not.
func (q *Queue) IsEmpty() bool {
return len(*q) == 0
}
================================================
FILE: lang/queue/queue_test.go
================================================
package queue
import "fmt"
func ExampleQueue_Pop() {
q := Queue{1}
q.Push(2)
q.Push(3)
fmt.Println(q.Pop())
fmt.Println(q.Pop())
fmt.Println(q.IsEmpty())
fmt.Println(q.Pop())
fmt.Println(q.IsEmpty())
// Output:
// 1
// 2
// false
// 3
// true
}
================================================
FILE: lang/queue/queueentry/main.go
================================================
package main
import (
"fmt"
"imooc.com/ccmouse/learngo/lang/queue"
)
func main() {
q := queue.Queue{1}
q.Push(2)
q.Push(3)
fmt.Println(q.Pop())
fmt.Println(q.Pop())
fmt.Println(q.IsEmpty())
fmt.Println(q.Pop())
fmt.Println(q.IsEmpty())
}
================================================
FILE: lang/retriever/main.go
================================================
package main
import (
"fmt"
"time"
"imooc.com/ccmouse/learngo/lang/retriever/mock"
"imooc.com/ccmouse/learngo/lang/retriever/real"
)
type Retriever interface {
Get(url string) string
}
type Poster interface {
Post(url string,
form map[string]string) string
}
const url = "http://www.imooc.com"
func download(r Retriever) string {
return r.Get(url)
}
func post(poster Poster) {
poster.Post(url,
map[string]string{
"name": "ccmouse",
"course": "golang",
})
}
type RetrieverPoster interface {
Retriever
Poster
}
func session(s RetrieverPoster) string {
s.Post(url, map[string]string{
"contents": "another faked imooc.com",
})
return s.Get(url)
}
func main() {
var r Retriever
mockRetriever := mock.Retriever{
Contents: "this is a fake imooc.com"}
r = &mockRetriever
inspect(r)
r = &real.Retriever{
UserAgent: "Mozilla/5.0",
TimeOut: time.Minute,
}
inspect(r)
// Type assertion
if mockRetriever, ok := r.(*mock.Retriever); ok {
fmt.Println(mockRetriever.Contents)
} else {
fmt.Println("r is not a mock retriever")
}
fmt.Println(
"Try a session with mockRetriever")
fmt.Println(session(&mockRetriever))
}
func inspect(r Retriever) {
fmt.Println("Inspecting", r)
fmt.Printf(" > Type:%T Value:%v\n", r, r)
fmt.Print(" > Type switch: ")
switch v := r.(type) {
case *mock.Retriever:
fmt.Println("Contents:", v.Contents)
case *real.Retriever:
fmt.Println("UserAgent:", v.UserAgent)
}
fmt.Println()
}
================================================
FILE: lang/retriever/mock/mockretriever.go
================================================
package mock
import "fmt"
type Retriever struct {
Contents string
}
func (r *Retriever) String() string {
return fmt.Sprintf(
"Retriever: {Contents=%s}", r.Contents)
}
func (r *Retriever) Post(url string,
form map[string]string) string {
r.Contents = form["contents"]
return "ok"
}
func (r *Retriever) Get(url string) string {
return r.Contents
}
================================================
FILE: lang/retriever/real/retriever.go
================================================
package real
import (
"net/http"
"net/http/httputil"
"time"
)
type Retriever struct {
UserAgent string
TimeOut time.Duration
}
func (r *Retriever) Get(url string) string {
resp, err := http.Get(url)
if err != nil {
panic(err)
}
result, err := httputil.DumpResponse(
resp, true)
resp.Body.Close()
if err != nil {
panic(err)
}
return string(result)
}
================================================
FILE: lang/rpc/client/main.go
================================================
package main
import (
"fmt"
"net"
"net/rpc/jsonrpc"
"imooc.com/ccmouse/learngo/lang/rpc"
)
func main() {
conn, err := net.Dial("tcp", ":1234")
if err != nil {
panic(err)
}
client := jsonrpc.NewClient(conn)
var result float64
err = client.Call("DemoService.Div",
rpcdemo.Args{10, 3}, &result)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(result)
}
err = client.Call("DemoService.Div",
rpcdemo.Args{10, 0}, &result)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(result)
}
}
================================================
FILE: lang/rpc/rpc.go
================================================
package rpcdemo
import "errors"
type DemoService struct{}
type Args struct {
A, B int
}
func (DemoService) Div(
args Args, result *float64) error {
if args.B == 0 {
return errors.New("division by zero")
}
*result = float64(args.A) / float64(args.B)
return nil
}
================================================
FILE: lang/rpc/server/main.go
================================================
package main
import (
"log"
"net"
"net/rpc"
"net/rpc/jsonrpc"
"imooc.com/ccmouse/learngo/lang/rpc"
)
func main() {
rpc.Register(rpcdemo.DemoService{})
listener, err := net.Listen("tcp", ":1234")
if err != nil {
panic(err)
}
for {
conn, err := listener.Accept()
if err != nil {
log.Printf("accept error: %v", err)
continue
}
go jsonrpc.ServeConn(conn)
}
}
================================================
FILE: lang/tree/node.go
================================================
package tree
import "fmt"
type Node struct {
Value int
Left, Right *Node
}
func (node Node) Print() {
fmt.Print(node.Value, " ")
}
func (node *Node) SetValue(value int) {
if node == nil {
fmt.Println("Setting Value to nil " +
"node. Ignored.")
return
}
node.Value = value
}
func CreateNode(value int) *Node {
return &Node{Value: value}
}
================================================
FILE: lang/tree/traversal.go
================================================
package tree
import "fmt"
func (node *Node) Traverse() {
node.TraverseFunc(func(n *Node) {
n.Print()
})
fmt.Println()
}
func (node *Node) TraverseFunc(f func(*Node)) {
if node == nil {
return
}
node.Left.TraverseFunc(f)
f(node)
node.Right.TraverseFunc(f)
}
func (node *Node) TraverseWithChannel() chan *Node {
out := make(chan *Node)
go func() {
node.TraverseFunc(func(node *Node) {
out <- node
})
close(out)
}()
return out
}
================================================
FILE: lang/tree/treeentry/entry.go
================================================
package main
import (
"fmt"
"imooc.com/ccmouse/learngo/lang/tree"
)
type myTreeNode struct {
node *tree.Node
}
func (myNode *myTreeNode) postOrder() {
if myNode == nil || myNode.node == nil {
return
}
left := myTreeNode{myNode.node.Left}
right := myTreeNode{myNode.node.Right}
left.postOrder()
right.postOrder()
myNode.node.Print()
}
func main() {
var root tree.Node
root = tree.Node{Value: 3}
root.Left = &tree.Node{}
root.Right = &tree.Node{5, nil, nil}
root.Right.Left = new(tree.Node)
root.Left.Right = tree.CreateNode(2)
root.Right.Left.SetValue(4)
fmt.Print("In-order traversal: ")
root.Traverse()
fmt.Print("My own post-order traversal: ")
myRoot := myTreeNode{&root}
myRoot.postOrder()
fmt.Println()
nodeCount := 0
root.TraverseFunc(func(node *tree.Node) {
nodeCount++
})
fmt.Println("Node count:", nodeCount)
c := root.TraverseWithChannel()
maxNodeValue := 0
for node := range c {
if node.Value > maxNodeValue {
maxNodeValue = node.Value
}
}
fmt.Println("Max node value:", maxNodeValue)
}
================================================
FILE: lang/tree/treeentry_embedded/entry.go
================================================
package main
import (
"fmt"
"imooc.com/ccmouse/learngo/lang/tree"
)
type myTreeNode struct {
*tree.Node // Embedding
}
func (myNode *myTreeNode) postOrder() {
if myNode == nil || myNode.Node == nil {
return
}
left := myTreeNode{myNode.Left}
right := myTreeNode{myNode.Right}
left.postOrder()
right.postOrder()
myNode.Print()
}
func main() {
root := myTreeNode{&tree.Node{Value: 3}}
root.Left = &tree.Node{}
root.Right = &tree.Node{5, nil, nil}
root.Right.Left = new(tree.Node)
root.Left.Right = tree.CreateNode(2)
root.Right.Left.SetValue(4)
fmt.Print("In-order traversal: ")
root.Traverse()
fmt.Print("My own post-order traversal: ")
root.postOrder()
fmt.Println()
}
================================================
FILE: mockserver/config/config.go
================================================
package config
var (
// ServerAddress configures the server prefix in url generations.
// 一般来说,我们网页中对其它网页的链接只需使用相对路径即可。
// 但目标网站的所有链接都是用了绝对路径,为了模拟,我们也需要生成绝对路径。
// 所以增加ServerAddress配置,所有的链接都使用形式:
// http:///mock/album.zhenai.com/<相对路径>
// 若将服务器部署在云,我们需要把这里替换成外网ip/域名:8080
ServerAddress = "localhost:8080"
// ListenAddress configures where the server listens at.
ListenAddress = ":8080"
)
================================================
FILE: mockserver/generator/city/city.go
================================================
// Package city implements city generator.
package city
import (
"bytes"
"encoding/gob"
"fmt"
"hash/fnv"
"html/template"
"io"
"log"
"math/rand"
"net/http"
"github.com/gin-gonic/gin"
"imooc.com/ccmouse/learngo/mockserver/config"
"imooc.com/ccmouse/learngo/mockserver/generator/profile"
)
// Generator represents the city generator.
type Generator struct {
Tmpl *template.Template
ProfileGen *profile.Generator
}
const (
itemCount = 20
pageCount = 5
)
type params struct {
City string `uri:"city" binding:"required"`
Page int `uri:"page"`
}
// HandleRequest is the gin request handler for city generation.
func (g *Generator) HandleRequest(c *gin.Context) {
var p params
err := c.BindUri(&p)
if err != nil {
log.Printf("BindUri(): %v.", err)
return
}
if p.Page == 0 {
p.Page = 1
}
g.handleRequest(c, p)
}
func (g *Generator) handleRequest(c *gin.Context, p params) {
err := g.generate(p, c.Writer)
if err != nil {
log.Printf("Cannot generate page for city %q and page %d: %v.", p.City, p.Page, err)
c.Status(http.StatusInternalServerError)
return
}
c.Status(http.StatusOK)
}
func (g *Generator) generate(p params, w io.Writer) error {
h, err := hashCode(p)
if err != nil {
return fmt.Errorf("cannot calculate hash code: %v", err)
}
r := rand.New(rand.NewSource(h))
items := make([]ContentItem, itemCount)
for i := range items {
id := r.Int63()
p := g.ProfileGen.GenerateProfile(id)
items[i] = ContentItem{
ID: id,
Profile: p,
URL: fmt.Sprintf("http://%s/mock/album.zhenai.com/u/%d", config.ServerAddress, id),
}
}
var pages []PageItem
for i := 0; i < pageCount; i++ {
targetPage := p.Page - 1 + i
if targetPage < 1 {
continue
}
url := ""
if targetPage != p.Page {
url = fmt.Sprintf("http://%s/mock/www.zhenai.com/zhenghun/%s/%d", config.ServerAddress, p.City, targetPage)
}
pages = append(pages, PageItem{
URL: url,
Page: targetPage,
})
}
return g.Tmpl.Execute(w, Content{
Items: items,
Pages: pages,
})
}
func hashCode(v interface{}) (int64, error) {
h := fnv.New64()
var b bytes.Buffer
err := gob.NewEncoder(&b).Encode(v)
if err != nil {
return 0, fmt.Errorf("cannot encode gob for param: %v", err)
}
_, err = h.Write(b.Bytes())
if err != nil {
return 0, fmt.Errorf("cannot write to hash: %v", err)
}
return int64(h.Sum64()), nil
}
// ContentItem defines content item bound into html template.
type ContentItem struct {
ID int64
Profile *profile.PhotoProfile
URL string
}
// PageItem defines page item bound into html template.
type PageItem struct {
URL string
Page int
}
// Content defines the master content bound into html template.
type Content struct {
Items []ContentItem
Pages []PageItem
}
================================================
FILE: mockserver/generator/city/city_test.go
================================================
package city
import (
"bytes"
"html/template"
"testing"
"imooc.com/ccmouse/learngo/crawler/zhenai/parser"
"imooc.com/ccmouse/learngo/mockserver/config"
"imooc.com/ccmouse/learngo/mockserver/generator/profile"
)
func TestGenerate(t *testing.T) {
config.ServerAddress = "localhost:8080"
pg := &profile.Generator{Tmpl: template.Must(template.ParseFiles("../profile/profile_tmpl.html"))}
g := Generator{
Tmpl: template.Must(template.ParseFiles("city_tmpl.html")),
ProfileGen: pg,
}
var b bytes.Buffer
err := g.generate(params{
City: "fuxin",
Page: 34,
}, &b)
if err != nil {
t.Fatalf("Cannot generate content: %v.", err)
}
r := parser.ParseCity(b.Bytes(), "")
wantItems, wantRequests := 0, 24
if len(r.Items) != wantItems {
t.Errorf("generate() want %d items, got %d: %v", wantItems, len(r.Items), r.Items)
}
if len(r.Requests) != wantRequests {
t.Errorf("generate() want %d requests, got %d: %v", wantRequests, len(r.Requests), r.Requests)
}
verify := []struct {
i int
wantURL string
wantParser string
wantArg interface{}
}{
{
i: 0,
wantURL: "http://localhost:8080/mock/album.zhenai.com/u/484971159322053275",
wantParser: "ParseProfile",
wantArg: "与你度余生迁就",
},
{
i: 23,
wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/fuxin/37",
wantParser: "ParseCity",
},
}
for _, v := range verify {
gotURL := r.Requests[v.i].Url
gotParser, gotArg := r.Requests[v.i].Parser.Serialize()
if v.wantURL != gotURL || v.wantParser != gotParser || v.wantArg != gotArg {
t.Errorf("generate() want %d-th request (%q, %q, %v), got (%q, %q, %v)",
v.i, v.wantURL, v.wantParser, v.wantArg, gotURL, gotParser, gotArg)
}
}
}
================================================
FILE: mockserver/generator/city/city_tmpl.html
================================================
城市用户列表
{{range .Items}}
{{with .Profile}}
{{.Income}}
{{.House}}
{{.Car}}
{{end}}
{{end}}
================================================
FILE: mockserver/generator/citylist/citylist.go
================================================
// Package citylist implements citylist generator.
package citylist
import (
"html/template"
"io"
"log"
"net/http"
"github.com/gin-gonic/gin"
"imooc.com/ccmouse/learngo/mockserver/config"
)
// Generator represents the citylist generator.
type Generator struct {
Tmpl *template.Template
}
// HandleRequest is the gin request handler for citylist generation.
func (g *Generator) HandleRequest(c *gin.Context) {
err := g.generate(c.Writer)
if err != nil {
log.Printf("Cannot generate page for citylist: %v.", err)
c.Status(http.StatusInternalServerError)
return
}
c.Status(http.StatusOK)
}
func (g *Generator) generate(w io.Writer) error {
return g.Tmpl.Execute(w, struct {
ServerAddress string
}{
ServerAddress: config.ServerAddress,
})
}
================================================
FILE: mockserver/generator/citylist/citylist_test.go
================================================
package citylist
import (
"bytes"
"html/template"
"testing"
"imooc.com/ccmouse/learngo/crawler/zhenai/parser"
"imooc.com/ccmouse/learngo/mockserver/config"
)
func TestGenerate(t *testing.T) {
config.ServerAddress = "localhost:8080"
g := Generator{
Tmpl: template.Must(template.ParseFiles("citylist_tmpl.html")),
}
var b bytes.Buffer
err := g.generate(&b)
if err != nil {
t.Fatalf("Cannot generate content: %v.", err)
}
r := parser.ParseCityList(b.Bytes(), "")
wantRequests := 470
if len(r.Requests) != wantRequests {
t.Errorf("generate() want %d requests, got %d: %v", wantRequests, len(r.Requests), r.Requests)
}
verify := []struct {
i int
wantURL string
wantParser string
wantArg interface{}
}{
{
i: 0,
wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/aba",
wantParser: "ParseCity",
},
{
i: 23,
wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/baotou",
wantParser: "ParseCity",
},
{
i: 469,
wantURL: "http://localhost:8080/mock/www.zhenai.com/zhenghun/zunyi",
wantParser: "ParseCity",
},
}
for _, v := range verify {
gotURL := r.Requests[v.i].Url
gotParser, gotArg := r.Requests[v.i].Parser.Serialize()
if v.wantURL != gotURL || v.wantParser != gotParser || v.wantArg != gotArg {
t.Errorf("generate() want %d-th request (%q, %q, %v), got (%q, %q, %v)",
v.i, v.wantURL, v.wantParser, v.wantArg, gotURL, gotParser, gotArg)
}
}
}
================================================
FILE: mockserver/generator/citylist/citylist_tmpl.html
================================================
Go语言分布式爬虫测试网站
A
阿坝
阿克苏
阿拉善盟
阿勒泰
阿里
安徽
安康
安庆
鞍山
安顺
安阳
澳门
B
白城
百色
白山
白银
巴南
蚌埠
宝坻
保定
宝鸡
宝山
保山
包头
巴彦淖尔
巴音郭楞
巴中
北碚
北辰
北海
北京
本溪
毕节
滨海新
滨州
璧山
博尔塔拉
亳州
C
沧州
长春
常德
昌都
昌吉
长宁
昌平
长沙
长寿
长治
常州
朝阳
潮州
承德
成都
城口
郴州
赤峰
池州
崇明
重庆
崇左
楚雄
滁州
D
大渡口
大理
大连
丹东
大庆
大同
大兴
大兴安岭
达州
大足
德宏
德阳
德州
垫江
定西
迪庆
东城
东莞
东丽
东营
E
鄂尔多斯
恩施
鄂州
F
防城港
房山
丰都
奉节
丰台
奉贤
佛山
福建
涪陵
抚顺
阜新
阜阳
福州
抚州
G
甘南
甘肃
赣州
甘孜
高雄
广安
广东
广西
广元
广州
贵港
桂林
贵阳
贵州
果洛
固原
H
哈尔滨
海北
海淀
海东地
海口
海南
海南藏族自治州
海西
哈密
邯郸
杭州
汉中
河北
鹤壁
河池
合川
河东
合肥
鹤岗
黑河
黑龙江
河南
衡水
衡阳
和平
和田
河西
河源
菏泽
贺州
红河
虹口
红桥
淮安
淮北
怀化
淮南
怀柔
花莲
黄冈
黄南
黄浦
黄山
黄石
湖北
呼和浩特
惠州
葫芦岛
呼伦贝尔
湖南
湖州
J
嘉定
佳木斯
吉安
江北
江津
江门
江苏
江西
焦作
嘉兴
嘉义
嘉峪关
揭阳
吉林
基隆
济南
金昌
晋城
静安
景德镇
静海
荆门
荆州
金华
济宁
津南
金山
晋中
锦州
九江
九龙
九龙坡
酒泉
鸡西
蓟县
K
开封
开县
喀什
克拉玛依
克孜勒苏柯尔克孜
昆明
L
来宾
莱芜
廊坊
兰州
拉萨
乐山
梁平
凉山
连云港
聊城
辽宁
辽阳
辽源
丽江
临沧
临汾
临夏
临沂
林芝
丽水
六安
六盘水
柳州
陇南
龙岩
娄底
漯河
洛阳
泸州
吕梁
M
马鞍山
茂名
眉山
梅州
门头沟
绵阳
苗栗
闵行
密云
牡丹江
N
南岸
南昌
南充
南川
南京
南开
南宁
南平
南通
南投
南阳
那曲
内江
内蒙古
宁波
宁德
宁河
宁夏
怒江
P
盘锦
攀枝花
澎湖
彭水
平顶山
屏东
平谷
平凉
萍乡
浦东新
普洱
莆田
普陀
濮阳
Q
黔东
黔江
黔南
黔西
綦江
青岛
青海
青浦
庆阳
清远
秦皇岛
钦州
齐齐哈尔
七台河
泉州
曲靖
衢州
R
日喀则
日照
荣昌
S
三门峡
三明
三亚
山东
上海
商洛
商丘
上饶
山南
汕头
汕尾
山西
陕西
韶关
绍兴
邵阳
沙坪坝
沈阳
深圳
石家庄
石景山
十堰
石柱
石嘴山
双鸭山
顺义
朔州
四川
四平
松江
松原
绥化
遂宁
随州
宿迁
苏州
宿州
T
塔城
泰安
台北
台东
台南
台湾
太原
台中
泰州
台州
唐山
桃园
天津
天水
铁岭
铜川
通化
铜梁
通辽
铜陵
潼南
铜仁
通州
吐鲁番
W
万州
潍坊
威海
渭南
文山
温州
乌海
武汉
芜湖
乌兰察布
武隆
乌鲁木齐
武清
巫山
武威
无锡
巫溪
吴忠
梧州
X
厦门
西安
香港
湘潭
湘西
襄阳
咸宁
咸阳
孝感
西城
锡林郭勒盟
新北
兴安
邢台
西宁
新疆
新界
新乡
信阳
新余
忻州
新竹
西青
西双版纳
秀山
西藏
宣城
许昌
徐汇
徐州
Y
雅安
延安
延边
盐城
阳江
杨浦
阳泉
扬州
延庆
烟台
宜宾
宜昌
宜春
伊春
宜兰
伊犁
银川
营口
鹰潭
益阳
永川
永州
酉阳
渝北
岳阳
榆林
玉林
运城
云浮
云林
云南
云阳
玉树
玉溪
渝中
Z
枣庄
闸北
彰化
张家界
张家口
张掖
漳州
湛江
肇庆
昭通
浙江
郑州
镇江
中山
中卫
忠县
周口
舟山
珠海
驻马店
株洲
淄博
自贡
资阳
遵义
================================================
FILE: mockserver/generator/profile/profile.go
================================================
// Package profile implements profile generator.
package profile
import (
"fmt"
"html/template"
"io"
"log"
"math/rand"
"net/http"
"github.com/gin-gonic/gin"
"imooc.com/ccmouse/learngo/crawler/model"
"imooc.com/ccmouse/learngo/mockserver/config"
)
// Recommendation defines the interface for recommendation subsystem.
type Recommendation interface {
NextGuess(id int64) []int64
}
// Generator represents the profile generator.
type Generator struct {
Tmpl *template.Template
Recommendation Recommendation
}
// HandleRequest is the gin request handler for profile generation.
func (g *Generator) HandleRequest(c *gin.Context) {
params := struct {
ID int64 `uri:"id" binding:"required"`
}{}
err := c.BindUri(¶ms)
if err != nil {
log.Printf("BindUri(): %v.", err)
return
}
err = g.generate(params.ID, c.Writer)
if err != nil {
log.Printf("Cannot generate profile for user %d: %v.", params.ID, err)
c.Status(http.StatusInternalServerError)
return
}
c.Status(http.StatusOK)
}
// GuessListItem defines guess list item bound into html template.
type GuessListItem struct {
URL string
Name string
PhotoURL string
}
// PhotoProfile defiens profile with photo bound into html template.
type PhotoProfile struct {
// Embed a *model.Profile here to save typing in template.
*model.Profile
PhotoURL string
}
// Content defines the master content bound into html template.
type Content struct {
*PhotoProfile
GuessList []GuessListItem
}
func (g *Generator) generate(id int64, w io.Writer) error {
p := g.GenerateProfile(id)
var guessItems []GuessListItem
if g.Recommendation != nil {
guesses := g.Recommendation.NextGuess(id)
guessItems = make([]GuessListItem, len(guesses))
for i, v := range guesses {
gp := g.GenerateProfile(v)
guessItems[i].URL = fmt.Sprintf("http://%s/mock/album.zhenai.com/u/%d", config.ServerAddress, v)
guessItems[i].Name = gp.Name
guessItems[i].PhotoURL = gp.PhotoURL
}
}
return g.Tmpl.Execute(w, Content{
PhotoProfile: p,
GuessList: guessItems,
})
}
// GenerateProfile generates a photo profile given a user id.
func (g *Generator) GenerateProfile(id int64) *PhotoProfile {
r := rand.New(rand.NewSource(id))
n1 := elementFromSlice(r, []string{
"断念",
"街痞",
"浪瘾",
"猖狂",
"归戾",
"洒脱",
"故我",
"简白",
"酒虑",
"野性入骨",
"学霸的芯",
"一笑奈何",
"隐匿微笑",
"限时拥抱",
"何必怀念",
"厌与深情",
"寂寞成影",
"桀骜不驯",
"浪痞孤王",
"高冷绅士",
"心事痕迹",
"逍遥浪子",
"隐身守候",
"执手不离",
"独久厌闹",
"逆天飞翔",
"醉生忧愁",
"全球焦點",
"孤者何惧",
"君临天下",
"无戏配角",
"一身傲气",
"冷暖自知",
"海阔天空",
"今朝醉者",
"不良骚年",
"薄情寡义",
"称霸全服",
"久别无恙°",
"與我無關°",
"逍遥べ无痕",
"混過也愛過",
"与你度余生",
"一见不钟情",
"孤独比酒暖°",
"原来无话可说",
"陪你浪迹天涯",
})
n2 := elementFromSlice(r, []string{
"萌宝",
"丁当",
"遇到",
"病娇",
"莓哒",
"深碍",
"面码",
"娇喘",
"爱你",
"迁就",
"丸子",
"初心",
"baby",
"猫儿.",
"傻蛋.",
"呆萌i",
"如你*",
"肉嘟嘟",
"小气鬼",
"胆小鬼",
"欣欣雨",
"酷酷猫",
"柠小萌",
"小丸子",
"小可爱",
"小仙女",
"考莎啦",
"晴妹儿.",
"记得笑i",
"稚性女",
"小短腿i",
"万能萌妹",
"蓝莓布朗",
"天瓓少女",
"草莓裙摆",
})
p := &model.Profile{
Name: n1 + n2,
Gender: elementFromSlice(r, []string{"男", "女"}),
Age: r.Intn(100),
Height: r.Intn(200),
Weight: r.Intn(300),
Income: elementFromSlice(r, []string{
"1-2000元",
"2001-3000元",
"3001-5000元",
"5001-8000元",
"8001-10000元",
"10001-20000元",
"财务自由",
}),
Marriage: elementFromSlice(r, []string{"未婚", "离异"}),
Education: elementFromSlice(r, []string{
"小学",
"初中",
"高中",
"大学",
"硕士",
"博士及以上",
}),
Occupation: elementFromSlice(r, []string{
"人事/行政",
"程序员",
"产品经理",
"测试工程师",
"财务",
"总经理",
"金融",
"销售",
"其它",
}),
Hokou: elementFromSlice(r, []string{
"北京市", "上海市", "广州市", "深圳市",
"成都市", "杭州市", "武汉市", "重庆市", "南京市", "天津市", "苏州市", "西安市", "长沙市", "沈阳市", "青岛市", "郑州市", "大连市", "东莞市", "宁波市",
"其它",
}),
Xinzuo: elementFromSlice(r, []string{
"白羊座",
"金牛座",
"双子座",
"巨蟹座",
"狮子座",
"处女座",
"天秤座",
"天蝎座",
"射手座",
"魔羯座",
"水瓶座",
"双鱼座",
}),
House: elementFromSlice(r, []string{
"有房",
"租房",
"无房",
}),
Car: elementFromSlice(r, []string{
"无车",
"有车",
"有豪车",
}),
}
return &PhotoProfile{
Profile: p,
PhotoURL: fmt.Sprintf("https://picsum.photos/seed/%d/300/300", r.Intn(100000)),
}
}
func elementFromSlice(r *rand.Rand, s []string) string {
return s[r.Intn(len(s))]
}
================================================
FILE: mockserver/generator/profile/profile_test.go
================================================
package profile
import (
"bytes"
"html/template"
"math/rand"
"testing"
"github.com/google/go-cmp/cmp"
"imooc.com/ccmouse/learngo/crawler/engine"
"imooc.com/ccmouse/learngo/crawler/model"
"imooc.com/ccmouse/learngo/crawler/zhenai/parser"
"imooc.com/ccmouse/learngo/mockserver/config"
"imooc.com/ccmouse/learngo/mockserver/recommendation"
)
func TestGenerate(t *testing.T) {
config.ServerAddress = "localhost:8080"
g := Generator{
Tmpl: template.Must(template.ParseFiles("profile_tmpl.html")),
Recommendation: recommendation.Client{},
}
rand.Seed(34534)
var b bytes.Buffer
err := g.generate(12345, &b)
if err != nil {
t.Fatalf("Cannot generate content: %v.", err)
}
want := engine.Item{
Url: "http://localhost:8080/mock/album.zhenai.com/u/12345",
Type: "zhenai",
Id: "12345",
Payload: model.Profile{
Name: "逍遥べ无痕病娇",
Gender: "男",
Age: 36,
Height: 41,
Weight: 275,
Income: "8001-10000元",
Marriage: "未婚",
Education: "初中",
Occupation: "人事/行政",
Hokou: "成都市",
Xinzuo: "魔羯座",
House: "有房",
Car: "有豪车",
},
}
r := parser.NewProfileParser("逍遥べ无痕病娇").Parse(b.Bytes(), "http://localhost:8080/mock/album.zhenai.com/u/12345")
if len(r.Items) != 1 {
t.Errorf("want exactly 1 element, got %d items: %v", len(r.Items), r.Items)
} else {
if diff := cmp.Diff(want, r.Items[0]); diff != "" {
t.Errorf("generated data is incorrect: diff: -want +got\n%s", diff)
}
}
type reqData struct {
URL string
Parser string
Arg interface{}
}
wantReq := []reqData{
{
URL: "http://localhost:8080/mock/album.zhenai.com/u/12335",
Parser: "ParseProfile",
Arg: "酒虑肉嘟嘟",
},
{
URL: "http://localhost:8080/mock/album.zhenai.com/u/12340",
Parser: "ParseProfile",
Arg: "称霸全服胆小鬼",
},
}
var gotReq []reqData
for _, req := range r.Requests {
parser, arg := req.Parser.Serialize()
gotReq = append(gotReq, reqData{
URL: req.Url,
Parser: parser,
Arg: arg,
})
}
if diff := cmp.Diff(wantReq, gotReq); diff != "" {
t.Errorf("generated request is incorrect: diff: -want +got\n%s", diff)
}
}
================================================
FILE: mockserver/generator/profile/profile_tmpl.html
================================================
{{.Name}}的资料
个人信息简介
年龄: {{.Age}}岁
身高: {{.Height}}CM
月收入: {{.Income}}
婚况: {{.Marriage}}
学历: {{.Education}}
职业: {{.Occupation}}
籍贯: {{.Hokou}}
个人信息详情
详细资料
性别: {{.Gender}}
生肖: --
身高: {{.Height}}CM
星座: {{.Xinzuo}}
体重: {{.Weight}}KG
血型: --
体型: --
职业: {{.Occupation}}
公司: --
信仰: --
生活状况
住房条件: {{.House}}
是否购车: {{.Car}}
兴趣爱好
喜欢的活动: --
喜欢的食物: --
喜欢的体育运动: --
喜欢的地方: --
喜欢的音乐: --
喜欢的宠物: --
喜欢的影视节目: --
{{range .GuessList}}
{{end}}
================================================
FILE: mockserver/main.go
================================================
package main
import (
"html/template"
"log"
"math/rand"
"net/http"
"time"
"github.com/gin-gonic/gin"
"imooc.com/ccmouse/learngo/mockserver/config"
"imooc.com/ccmouse/learngo/mockserver/generator/city"
"imooc.com/ccmouse/learngo/mockserver/generator/citylist"
"imooc.com/ccmouse/learngo/mockserver/generator/profile"
"imooc.com/ccmouse/learngo/mockserver/recommendation"
)
const templateSuggestion = "Please make sure working directory is the root of the repository, where we have go.mod/go.sum. Suggested command line: go run mockserver/main.go"
func main() {
profileTemplate, err := template.ParseFiles("mockserver/generator/profile/profile_tmpl.html")
if err != nil {
log.Fatalf("Cannot create profile template: %v. %s", err, templateSuggestion)
}
profileGen := &profile.Generator{
Tmpl: profileTemplate,
Recommendation: recommendation.Client{},
}
cityTemplate, err := template.ParseFiles("mockserver/generator/city/city_tmpl.html")
if err != nil {
log.Fatalf("Cannot create city template: %v. %s", err, templateSuggestion)
}
cityGen := &city.Generator{
Tmpl: cityTemplate,
ProfileGen: profileGen,
}
cityListTemplate, err := template.ParseFiles("mockserver/generator/citylist/citylist_tmpl.html")
if err != nil {
log.Fatalf("Cannot create citylist template: %v. %s", err, templateSuggestion)
}
cityListGen := &citylist.Generator{
Tmpl: cityListTemplate,
}
rand.Seed(time.Now().Unix())
r := gin.Default()
r.GET("/", func(c *gin.Context) {
c.Redirect(http.StatusTemporaryRedirect, "/static/index.html")
})
r.Static("/static", "mockserver/static")
r.GET("mock/www.zhenai.com/zhenghun", cityListGen.HandleRequest)
r.GET("mock/www.zhenai.com/zhenghun/:city/:page", cityGen.HandleRequest)
r.GET("mock/www.zhenai.com/zhenghun/:city", cityGen.HandleRequest)
r.GET("mock/album.zhenai.com/u/:id", profileGen.HandleRequest)
log.Fatal(r.Run(config.ListenAddress))
}
================================================
FILE: mockserver/recommendation/rcmd.go
================================================
// Package recommendation provides a simple implementation mimicing a recommendation system.
package recommendation
import "math/rand"
// If given id is 20, we may generate 10, 15, 25, ...
// Intentionally desgined to increase duplication within recommendation cycles,
// to mimic real-world behvior.
const (
step = 5
offset = -10
maxGuess = 7
)
// Client defines the recommendation client.
type Client struct{}
// NextGuess returns the next batch of guesses for the given id.
func (Client) NextGuess(id int64) []int64 {
var res []int64
// We use a global random function because:
// 1. We want recommendation result to be different each time.
// 2. Only global random function is thread-safe.
guessCnt := rand.Intn(maxGuess)
for i, r := 0, id+offset; i < guessCnt; {
if r > 0 && r != id {
res = append(res, r)
}
i++
r += step
}
return res
}
================================================
FILE: mockserver/recommendation/rcmd_test.go
================================================
package recommendation
import (
"math"
"math/rand"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestNextGuess(t *testing.T) {
tests := []struct {
name string
id int64
want []int64
}{
{
name: "normal",
id: 12345,
want: []int64{12335, 12340, 12350, 12355},
},
{
name: "very small",
id: 8,
want: []int64{3, 13, 18},
},
{
name: "very large",
id: math.MaxInt64 - 7,
want: []int64{9223372036854775790, 9223372036854775795, 9223372036854775805},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rand.Seed(23465)
cl := Client{}
got := cl.NextGuess(tt.id)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("NextGuess(): diff: -want + got\n%s", diff)
}
})
}
}
================================================
FILE: mockserver/static/css/blog.css
================================================
/*!
Pure v1.0.1
Copyright 2013 Yahoo!
Licensed under the BSD License.
https://github.com/pure-css/pure/blob/master/LICENSE.md
*/
/*!
normalize.css v^3.0 | MIT License | git.io/normalize
Copyright (c) Nicolas Gallagher and Jonathan Neal
*/
/*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */
html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{font-size:2em;margin:.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;box-sizing:content-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}legend{border:0;padding:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}.hidden,[hidden]{display:none!important}.pure-img{max-width:100%;height:auto;display:block}.pure-g{letter-spacing:-.31em;text-rendering:optimizespeed;font-family:FreeSans,Arimo,"Droid Sans",Helvetica,Arial,sans-serif;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-align-content:flex-start;-ms-flex-line-pack:start;align-content:flex-start}@media all and (-ms-high-contrast:none),(-ms-high-contrast:active){table .pure-g{display:block}}.opera-only :-o-prefocus,.pure-g{word-spacing:-.43em}.pure-u{display:inline-block;zoom:1;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-g [class*=pure-u]{font-family:sans-serif}.pure-u-1,.pure-u-1-1,.pure-u-1-12,.pure-u-1-2,.pure-u-1-24,.pure-u-1-3,.pure-u-1-4,.pure-u-1-5,.pure-u-1-6,.pure-u-1-8,.pure-u-10-24,.pure-u-11-12,.pure-u-11-24,.pure-u-12-24,.pure-u-13-24,.pure-u-14-24,.pure-u-15-24,.pure-u-16-24,.pure-u-17-24,.pure-u-18-24,.pure-u-19-24,.pure-u-2-24,.pure-u-2-3,.pure-u-2-5,.pure-u-20-24,.pure-u-21-24,.pure-u-22-24,.pure-u-23-24,.pure-u-24-24,.pure-u-3-24,.pure-u-3-4,.pure-u-3-5,.pure-u-3-8,.pure-u-4-24,.pure-u-4-5,.pure-u-5-12,.pure-u-5-24,.pure-u-5-5,.pure-u-5-6,.pure-u-5-8,.pure-u-6-24,.pure-u-7-12,.pure-u-7-24,.pure-u-7-8,.pure-u-8-24,.pure-u-9-24{display:inline-block;zoom:1;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-u-1-24{width:4.1667%}.pure-u-1-12,.pure-u-2-24{width:8.3333%}.pure-u-1-8,.pure-u-3-24{width:12.5%}.pure-u-1-6,.pure-u-4-24{width:16.6667%}.pure-u-1-5{width:20%}.pure-u-5-24{width:20.8333%}.pure-u-1-4,.pure-u-6-24{width:25%}.pure-u-7-24{width:29.1667%}.pure-u-1-3,.pure-u-8-24{width:33.3333%}.pure-u-3-8,.pure-u-9-24{width:37.5%}.pure-u-2-5{width:40%}.pure-u-10-24,.pure-u-5-12{width:41.6667%}.pure-u-11-24{width:45.8333%}.pure-u-1-2,.pure-u-12-24{width:50%}.pure-u-13-24{width:54.1667%}.pure-u-14-24,.pure-u-7-12{width:58.3333%}.pure-u-3-5{width:60%}.pure-u-15-24,.pure-u-5-8{width:62.5%}.pure-u-16-24,.pure-u-2-3{width:66.6667%}.pure-u-17-24{width:70.8333%}.pure-u-18-24,.pure-u-3-4{width:75%}.pure-u-19-24{width:79.1667%}.pure-u-4-5{width:80%}.pure-u-20-24,.pure-u-5-6{width:83.3333%}.pure-u-21-24,.pure-u-7-8{width:87.5%}.pure-u-11-12,.pure-u-22-24{width:91.6667%}.pure-u-23-24{width:95.8333%}.pure-u-1,.pure-u-1-1,.pure-u-24-24,.pure-u-5-5{width:100%}.pure-button{display:inline-block;zoom:1;line-height:normal;white-space:nowrap;vertical-align:middle;text-align:center;cursor:pointer;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-button::-moz-focus-inner{padding:0;border:0}.pure-button-group{letter-spacing:-.31em;text-rendering:optimizespeed}.opera-only :-o-prefocus,.pure-button-group{word-spacing:-.43em}.pure-button-group .pure-button{letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-button{font-family:inherit;font-size:100%;padding:.5em 1em;color:#444;color:rgba(0,0,0,.8);border:1px solid #999;border:none transparent;background-color:#e6e6e6;text-decoration:none;border-radius:2px}.pure-button-hover,.pure-button:focus,.pure-button:hover{background-image:-webkit-gradient(linear,left top,left bottom,from(transparent),color-stop(40%,rgba(0,0,0,.05)),to(rgba(0,0,0,.1)));background-image:-webkit-linear-gradient(transparent,rgba(0,0,0,.05) 40%,rgba(0,0,0,.1));background-image:linear-gradient(transparent,rgba(0,0,0,.05) 40%,rgba(0,0,0,.1))}.pure-button:focus{outline:0}.pure-button-active,.pure-button:active{-webkit-box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;border-color:#000}.pure-button-disabled,.pure-button-disabled:active,.pure-button-disabled:focus,.pure-button-disabled:hover,.pure-button[disabled]{border:none;background-image:none;opacity:.4;cursor:not-allowed;-webkit-box-shadow:none;box-shadow:none;pointer-events:none}.pure-button-hidden{display:none}.pure-button-primary,.pure-button-selected,a.pure-button-primary,a.pure-button-selected{background-color:#0078e7;color:#fff}.pure-button-group .pure-button{margin:0;border-radius:0;border-right:1px solid #111;border-right:1px solid rgba(0,0,0,.2)}.pure-button-group .pure-button:first-child{border-top-left-radius:2px;border-bottom-left-radius:2px}.pure-button-group .pure-button:last-child{border-top-right-radius:2px;border-bottom-right-radius:2px;border-right:none}.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form select,.pure-form textarea{padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;vertical-align:middle;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input:not([type]){padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input[type=color]{padding:.2em .5em}.pure-form input[type=color]:focus,.pure-form input[type=date]:focus,.pure-form input[type=datetime-local]:focus,.pure-form input[type=datetime]:focus,.pure-form input[type=email]:focus,.pure-form input[type=month]:focus,.pure-form input[type=number]:focus,.pure-form input[type=password]:focus,.pure-form input[type=search]:focus,.pure-form input[type=tel]:focus,.pure-form input[type=text]:focus,.pure-form input[type=time]:focus,.pure-form input[type=url]:focus,.pure-form input[type=week]:focus,.pure-form select:focus,.pure-form textarea:focus{outline:0;border-color:#129fea}.pure-form input:not([type]):focus{outline:0;border-color:#129fea}.pure-form input[type=checkbox]:focus,.pure-form input[type=file]:focus,.pure-form input[type=radio]:focus{outline:thin solid #129fea;outline:1px auto #129fea}.pure-form .pure-checkbox,.pure-form .pure-radio{margin:.5em 0;display:block}.pure-form input[type=color][disabled],.pure-form input[type=date][disabled],.pure-form input[type=datetime-local][disabled],.pure-form input[type=datetime][disabled],.pure-form input[type=email][disabled],.pure-form input[type=month][disabled],.pure-form input[type=number][disabled],.pure-form input[type=password][disabled],.pure-form input[type=search][disabled],.pure-form input[type=tel][disabled],.pure-form input[type=text][disabled],.pure-form input[type=time][disabled],.pure-form input[type=url][disabled],.pure-form input[type=week][disabled],.pure-form select[disabled],.pure-form textarea[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input:not([type])[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input[readonly],.pure-form select[readonly],.pure-form textarea[readonly]{background-color:#eee;color:#777;border-color:#ccc}.pure-form input:focus:invalid,.pure-form select:focus:invalid,.pure-form textarea:focus:invalid{color:#b94a48;border-color:#e9322d}.pure-form input[type=checkbox]:focus:invalid:focus,.pure-form input[type=file]:focus:invalid:focus,.pure-form input[type=radio]:focus:invalid:focus{outline-color:#e9322d}.pure-form select{height:2.25em;border:1px solid #ccc;background-color:#fff}.pure-form select[multiple]{height:auto}.pure-form label{margin:.5em 0 .2em}.pure-form fieldset{margin:0;padding:.35em 0 .75em;border:0}.pure-form legend{display:block;width:100%;padding:.3em 0;margin-bottom:.3em;color:#333;border-bottom:1px solid #e5e5e5}.pure-form-stacked input[type=color],.pure-form-stacked input[type=date],.pure-form-stacked input[type=datetime-local],.pure-form-stacked input[type=datetime],.pure-form-stacked input[type=email],.pure-form-stacked input[type=file],.pure-form-stacked input[type=month],.pure-form-stacked input[type=number],.pure-form-stacked input[type=password],.pure-form-stacked input[type=search],.pure-form-stacked input[type=tel],.pure-form-stacked input[type=text],.pure-form-stacked input[type=time],.pure-form-stacked input[type=url],.pure-form-stacked input[type=week],.pure-form-stacked label,.pure-form-stacked select,.pure-form-stacked textarea{display:block;margin:.25em 0}.pure-form-stacked input:not([type]){display:block;margin:.25em 0}.pure-form-aligned .pure-help-inline,.pure-form-aligned input,.pure-form-aligned select,.pure-form-aligned textarea,.pure-form-message-inline{display:inline-block;vertical-align:middle}.pure-form-aligned textarea{vertical-align:top}.pure-form-aligned .pure-control-group{margin-bottom:.5em}.pure-form-aligned .pure-control-group label{text-align:right;display:inline-block;vertical-align:middle;width:10em;margin:0 1em 0 0}.pure-form-aligned .pure-controls{margin:1.5em 0 0 11em}.pure-form .pure-input-rounded,.pure-form input.pure-input-rounded{border-radius:2em;padding:.5em 1em}.pure-form .pure-group fieldset{margin-bottom:10px}.pure-form .pure-group input,.pure-form .pure-group textarea{display:block;padding:10px;margin:0 0 -1px;border-radius:0;position:relative;top:-1px}.pure-form .pure-group input:focus,.pure-form .pure-group textarea:focus{z-index:3}.pure-form .pure-group input:first-child,.pure-form .pure-group textarea:first-child{top:1px;border-radius:4px 4px 0 0;margin:0}.pure-form .pure-group input:first-child:last-child,.pure-form .pure-group textarea:first-child:last-child{top:1px;border-radius:4px;margin:0}.pure-form .pure-group input:last-child,.pure-form .pure-group textarea:last-child{top:-2px;border-radius:0 0 4px 4px;margin:0}.pure-form .pure-group button{margin:.35em 0}.pure-form .pure-input-1{width:100%}.pure-form .pure-input-3-4{width:75%}.pure-form .pure-input-2-3{width:66%}.pure-form .pure-input-1-2{width:50%}.pure-form .pure-input-1-3{width:33%}.pure-form .pure-input-1-4{width:25%}.pure-form .pure-help-inline,.pure-form-message-inline{display:inline-block;padding-left:.3em;color:#666;vertical-align:middle;font-size:.875em}.pure-form-message{display:block;color:#666;font-size:.875em}@media only screen and (max-width :480px){.pure-form button[type=submit]{margin:.7em 0 0}.pure-form input:not([type]),.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form label{margin-bottom:.3em;display:block}.pure-group input:not([type]),.pure-group input[type=color],.pure-group input[type=date],.pure-group input[type=datetime-local],.pure-group input[type=datetime],.pure-group input[type=email],.pure-group input[type=month],.pure-group input[type=number],.pure-group input[type=password],.pure-group input[type=search],.pure-group input[type=tel],.pure-group input[type=text],.pure-group input[type=time],.pure-group input[type=url],.pure-group input[type=week]{margin-bottom:0}.pure-form-aligned .pure-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.pure-form-aligned .pure-controls{margin:1.5em 0 0 0}.pure-form .pure-help-inline,.pure-form-message,.pure-form-message-inline{display:block;font-size:.75em;padding:.2em 0 .8em}}.pure-menu{-webkit-box-sizing:border-box;box-sizing:border-box}.pure-menu-fixed{position:fixed;left:0;top:0;z-index:3}.pure-menu-item,.pure-menu-list{position:relative}.pure-menu-list{list-style:none;margin:0;padding:0}.pure-menu-item{padding:0;margin:0;height:100%}.pure-menu-heading,.pure-menu-link{display:block;text-decoration:none;white-space:nowrap}.pure-menu-horizontal{width:100%;white-space:nowrap}.pure-menu-horizontal .pure-menu-list{display:inline-block}.pure-menu-horizontal .pure-menu-heading,.pure-menu-horizontal .pure-menu-item,.pure-menu-horizontal .pure-menu-separator{display:inline-block;zoom:1;vertical-align:middle}.pure-menu-item .pure-menu-item{display:block}.pure-menu-children{display:none;position:absolute;left:100%;top:0;margin:0;padding:0;z-index:3}.pure-menu-horizontal .pure-menu-children{left:0;top:auto;width:inherit}.pure-menu-active>.pure-menu-children,.pure-menu-allow-hover:hover>.pure-menu-children{display:block;position:absolute}.pure-menu-has-children>.pure-menu-link:after{padding-left:.5em;content:"\25B8";font-size:small}.pure-menu-horizontal .pure-menu-has-children>.pure-menu-link:after{content:"\25BE"}.pure-menu-scrollable{overflow-y:scroll;overflow-x:hidden}.pure-menu-scrollable .pure-menu-list{display:block}.pure-menu-horizontal.pure-menu-scrollable .pure-menu-list{display:inline-block}.pure-menu-horizontal.pure-menu-scrollable{white-space:nowrap;overflow-y:hidden;overflow-x:auto;-webkit-overflow-scrolling:touch;padding:.5em 0}.pure-menu-horizontal .pure-menu-children .pure-menu-separator,.pure-menu-separator{background-color:#ccc;height:1px;margin:.3em 0}.pure-menu-horizontal .pure-menu-separator{width:1px;height:1.3em;margin:0 .3em}.pure-menu-horizontal .pure-menu-children .pure-menu-separator{display:block;width:auto}.pure-menu-heading{text-transform:uppercase;color:#565d64}.pure-menu-link{color:#777}.pure-menu-children{background-color:#fff}.pure-menu-disabled,.pure-menu-heading,.pure-menu-link{padding:.5em 1em}.pure-menu-disabled{opacity:.5}.pure-menu-disabled .pure-menu-link:hover{background-color:transparent}.pure-menu-active>.pure-menu-link,.pure-menu-link:focus,.pure-menu-link:hover{background-color:#eee}.pure-menu-selected>.pure-menu-link,.pure-menu-selected>.pure-menu-link:visited{color:#000}.pure-table{border-collapse:collapse;border-spacing:0;empty-cells:show}.pure-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.pure-table td,.pure-table th{border-width:0 0 0 1px;font-size:inherit;margin:0;overflow:visible;padding:.5em 1em}.pure-table thead{background-color:#e0e0e0;color:#000;text-align:left;vertical-align:bottom}.pure-table td{background-color:transparent}.pure-table-odd td{background-color:#f2f2f2}.pure-table-striped tr:nth-child(2n-1) td{background-color:#f2f2f2}.pure-table-bordered td{border-bottom:1px solid #cbcbcb}.pure-table-bordered tbody>tr:last-child>td{border-bottom-width:0}.pure-table-horizontal td,.pure-table-horizontal th{border-width:0 0 1px 0;border-bottom:1px solid #cbcbcb}.pure-table-horizontal tbody>tr:last-child>td{border-bottom-width:0}
* {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
a {
text-decoration: none;
color: rgb(61, 146, 201);
}
a:hover,
a:focus {
text-decoration: underline;
}
h3 {
font-weight: 100;
}
/* LAYOUT CSS */
.pure-img-responsive {
max-width: 100%;
height: auto;
}
#layout {
padding: 0;
}
.header {
text-align: center;
top: auto;
margin: 3em auto;
}
.sidebar {
background: rgb(61, 79, 93);
color: #fff;
}
.brand-title,
.brand-tagline {
margin: 0;
}
.brand-title {
text-transform: uppercase;
}
.brand-tagline {
font-weight: 300;
color: rgb(176, 202, 219);
}
.nav-list {
margin: 0;
padding: 0;
list-style: none;
}
.nav-item {
display: inline-block;
*display: inline;
zoom: 1;
}
.nav-item a {
background: transparent;
border: 2px solid rgb(176, 202, 219);
color: #fff;
margin-top: 1em;
letter-spacing: 0.05em;
text-transform: uppercase;
font-size: 85%;
}
.nav-item a:hover,
.nav-item a:focus {
border: 2px solid rgb(61, 146, 201);
text-decoration: none;
}
.content-subhead {
text-transform: uppercase;
color: #aaa;
padding: 0.4em 0;
font-size: 80%;
font-weight: 500;
letter-spacing: 0.1em;
}
.content {
padding: 2em 1em 0;
}
.post {
padding-bottom: 2em;
}
.post-title {
font-size: 2em;
color: #222;
margin-bottom: 0.2em;
}
.post-avatar {
border-radius: 50px;
float: right;
margin-left: 1em;
}
.post-description {
font-family: Georgia, "Cambria", serif;
color: #444;
line-height: 1.8em;
}
.post-meta {
color: #999;
font-size: 90%;
margin: 0;
}
.post-category {
margin: 0 0.1em;
padding: 0.3em 1em;
color: #fff;
background: #999;
font-size: 80%;
}
.post-category-design {
background: #5aba59;
}
.post-category-pure {
background: #4d85d1;
}
.post-category-yui {
background: #8156a7;
}
.post-category-js {
background: #df2d4f;
}
.post-images {
margin: 1em 0;
}
.post-image-meta {
margin-top: -3.5em;
margin-left: 1em;
color: #fff;
text-shadow: 0 1px 1px #333;
}
.footer {
padding: 1em 0;
}
.footer a {
color: #ccc;
font-size: 80%;
}
.footer .pure-menu a:hover,
.footer .pure-menu a:focus {
background: none;
}
@media (min-width: 48em) {
.content {
padding: 2em 3em 0;
margin-left: 5%;
}
.header {
margin: 80% 2em 0;
text-align: right;
}
.sidebar {
position: fixed;
top: 0;
bottom: 0;
}
}
.city-photo {
width: 80%;
height: 80%;
}
.pager {
margin-right: 10px;
}
body {
color: #777;
}
.pure-img-responsive {
max-width: 100%;
height: auto;
}
/*
Add transition to containers so they can push in and out.
*/
#layout,
#menu,
.menu-link {
-webkit-transition: all 0.2s ease-out;
-moz-transition: all 0.2s ease-out;
-ms-transition: all 0.2s ease-out;
-o-transition: all 0.2s ease-out;
transition: all 0.2s ease-out;
}
/*
This is the parent `` that contains the menu and the content area.
*/
#layout {
position: relative;
left: 0;
padding-left: 0;
}
#layout.active #menu {
left: 150px;
width: 150px;
}
#layout.active .menu-link {
left: 150px;
}
/*
The content `
` is where all your content goes.
*/
.content {
padding: 0 2em;
margin-bottom: 50px;
line-height: 1.6em;
}
.header {
margin: 0;
color: #333;
text-align: center;
padding: 2.5em 2em 0;
border-bottom: 1px solid #eee;
}
.header h1 {
margin: 0.2em 0;
font-size: 3em;
font-weight: 300;
}
.header h2 {
font-weight: 300;
color: #ccc;
padding: 0;
margin-top: 0;
}
.content-subhead {
margin: 50px 0 20px 0;
font-weight: 300;
color: #888;
}
/*
The `#menu` `
` is the parent `
` that contains the `.pure-menu` that
appears on the left side of the page.
*/
#menu {
margin-left: -150px; /* "#menu" width */
width: 150px;
position: fixed;
top: 0;
left: 0;
bottom: 0;
z-index: 1000; /* so the menu or its navicon stays above all content */
background: #191818;
overflow-y: auto;
-webkit-overflow-scrolling: touch;
}
/*
All anchors inside the menu should be styled like this.
*/
#menu a {
color: #999;
border: none;
padding: 0.6em 0 0.6em 0.6em;
}
/*
Remove all background/borders, since we are applying them to #menu.
*/
#menu .pure-menu,
#menu .pure-menu ul {
border: none;
background: transparent;
}
/*
Add that light border to separate items into groups.
*/
#menu .pure-menu ul,
#menu .pure-menu .menu-item-divided {
border-top: 1px solid #333;
}
/*
Change color of the anchor links on hover/focus.
*/
#menu .pure-menu li a:hover,
#menu .pure-menu li a:focus {
background: #333;
}
/*
This styles the selected menu item `
`.
*/
#menu .pure-menu-selected,
#menu .pure-menu-heading {
background: #1f8dd6;
}
/*
This styles a link within a selected menu item ` `.
*/
#menu .pure-menu-selected a {
color: #fff;
}
/*
This styles the menu heading.
*/
#menu .pure-menu-heading {
font-size: 110%;
color: #fff;
margin: 0;
}
/* -- Dynamic Button For Responsive Menu -------------------------------------*/
/*
The button to open/close the Menu is custom-made and not part of Pure. Here's
how it works:
*/
/*
`.menu-link` represents the responsive menu toggle that shows/hides on
small screens.
*/
.menu-link {
position: fixed;
display: block; /* show this only on small screens */
top: 0;
left: 0; /* "#menu width" */
background: #000;
background: rgba(0,0,0,0.7);
font-size: 10px; /* change this value to increase/decrease button size */
z-index: 10;
width: 2em;
height: auto;
padding: 2.1em 1.6em;
}
.menu-link:hover,
.menu-link:focus {
background: #000;
}
.menu-link span {
position: relative;
display: block;
}
.menu-link span,
.menu-link span:before,
.menu-link span:after {
background-color: #fff;
width: 100%;
height: 0.2em;
}
.menu-link span:before,
.menu-link span:after {
position: absolute;
margin-top: -0.6em;
content: " ";
}
.menu-link span:after {
margin-top: 0.6em;
}
/* -- Responsive Styles (Media Queries) ------------------------------------- */
/*
Hides the menu at `48em`, but modify this based on your app's needs.
*/
@media (min-width: 48em) {
.header,
.content {
padding-left: 2em;
padding-right: 2em;
}
#layout {
padding-left: 150px; /* left col width "#menu" */
left: 0;
}
#menu {
left: 150px;
}
.menu-link {
position: fixed;
left: 150px;
display: none;
}
#layout.active .menu-link {
left: 150px;
}
}
@media (max-width: 48em) {
/* Only apply this when the window is small. Otherwise, the following
case results in extra padding on the left:
* Make the window small.
* Tap the menu to trigger the active state.
* Make the window large again.
*/
#layout.active {
position: relative;
left: 150px;
}
}
.profile-photo {
width: 100%;
height: 100%;
}
.guess-photo {
width: 80%;
height: 80%;
}
* {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
/*
* -- BASE STYLES --
* Most of these are inherited from Base, but I want to change a few.
*/
body {
line-height: 1.7em;
color: #7f8c8d;
font-size: 13px;
}
h1,
h2,
h3,
h4,
h5,
h6,
label {
color: #34495e;
}
.pure-img-responsive {
max-width: 100%;
height: auto;
}
/*
* -- LAYOUT STYLES --
* These are some useful classes which I will need
*/
.l-box {
padding: 1em;
}
.l-box-lrg {
padding: 2em;
border-bottom: 1px solid rgba(0,0,0,0.1);
}
.is-center {
text-align: center;
}
/*
* -- PURE FORM STYLES --
* Style the form inputs and labels
*/
.pure-form label {
margin: 1em 0 0;
font-weight: bold;
font-size: 100%;
}
.pure-form input[type] {
border: 2px solid #ddd;
box-shadow: none;
font-size: 100%;
width: 100%;
margin-bottom: 1em;
}
/*
* -- PURE BUTTON STYLES --
* I want my pure-button elements to look a little different
*/
.pure-button {
background-color: #1f8dd6;
color: white;
padding: 0.5em 2em;
border-radius: 5px;
}
a.pure-button-primary {
background: white;
color: #1f8dd6;
border-radius: 5px;
font-size: 120%;
}
/*
* -- MENU STYLES --
* I want to customize how my .pure-menu looks at the top of the page
*/
.home-menu {
padding: 0.5em;
text-align: center;
box-shadow: 0 1px 1px rgba(0,0,0, 0.10);
}
.home-menu {
background: #2d3e50;
}
.pure-menu.pure-menu-fixed {
/* Fixed menus normally have a border at the bottom. */
border-bottom: none;
/* I need a higher z-index here because of the scroll-over effect. */
z-index: 4;
}
.home-menu .pure-menu-heading {
color: white;
font-weight: 400;
font-size: 120%;
}
.home-menu .pure-menu-selected a {
color: white;
}
.home-menu a {
color: #6FBEF3;
}
.home-menu li a:hover,
.home-menu li a:focus {
background: none;
border: none;
color: #AECFE5;
}
/*
* -- SPLASH STYLES --
* This is the blue top section that appears on the page.
*/
.splash-container {
background: #1f8dd6;
z-index: 1;
overflow: hidden;
/* The following styles are required for the "scroll-over" effect */
width: 100%;
height: 88%;
top: 0;
left: 0;
position: fixed !important;
}
.splash {
/* absolute center .splash within .splash-container */
width: 80%;
height: 50%;
margin: auto;
position: absolute;
top: 100px; left: 0; bottom: 0; right: 0;
text-align: center;
}
/* This is the main heading that appears on the blue section */
.splash-head {
font-size: 20px;
font-weight: bold;
color: white;
border: 3px solid white;
padding: 1em 1.6em;
font-weight: 100;
border-radius: 30px;
line-height: 1em;
}
/* This is the subheading that appears on the blue section */
.splash-subhead {
color: white;
letter-spacing: 0.05em;
opacity: 0.8;
}
/*
* -- CONTENT STYLES --
* This represents the content area (everything below the blue section)
*/
.content-wrapper {
/* These styles are required for the "scroll-over" effect */
position: absolute;
top: 87%;
width: 100%;
min-height: 12%;
z-index: 2;
background: white;
}
/* We want to give the content area some more padding */
.content {
padding: 1em 1em 3em;
}
/* This is the class used for the main content headers () */
.content-head {
font-weight: 400;
text-transform: uppercase;
letter-spacing: 0.1em;
margin: 2em 0 1em;
}
/* This is a modifier class used when the content-head is inside a ribbon */
.content-head-ribbon {
color: white;
}
/* This is the class used for the content sub-headers () */
.content-subhead {
color: #1f8dd6;
}
.content-subhead i {
margin-right: 7px;
}
/* This is the class used for the dark-background areas. */
.ribbon {
background: #2d3e50;
color: #aaa;
}
/*
* -- TABLET (AND UP) MEDIA QUERIES --
* On tablets and other medium-sized devices, we want to customize some
* of the mobile styles.
*/
@media (min-width: 48em) {
/* We increase the body font size */
body {
font-size: 16px;
}
/* We can align the menu header to the left, but float the
menu items to the right. */
.home-menu {
text-align: left;
}
.home-menu ul {
float: right;
}
/* We increase the height of the splash-container */
/* .splash-container {
height: 500px;
}*/
/* We decrease the width of the .splash, since we have more width
to work with */
.splash {
width: 50%;
height: 50%;
}
.splash-head {
font-size: 250%;
}
/* We remove the border-separator assigned to .l-box-lrg */
.l-box-lrg {
border: none;
}
}
.crawler-prompt{
color: white;
}
================================================
FILE: mockserver/static/index.html
================================================
Google资深工程师深度讲解Go语言-课程实战项目爬虫测试站
Google资深工程师 深度讲解Go语言
课程实战项目爬虫测试站
进入
爬虫设置
================================================
FILE: mockserver/static/instructions.html
================================================
模拟相亲网站使用方法
本模拟相亲网站可针对性响应原本发到第三方网站的请求。
原本发送到类似http://www.zhenai.com/...的请求在本服务器/mock/www.zhanai.com/...被响应。
响应的数据可被本课程原针对zhenai.com的爬虫爬取。
例:http://www.zhenai.com/zhenghun/aomen --> http://localhost:8080/mock/www.zhenai.com/zhenghun/aomen
例:http://album.zhenai.com/u/9166010045394758737-->
http://localhost:8080/mock/album.zhenai.com/u/9166010045394758737
爬虫入口请设置为:
当然不要忘了我们在main.go
中配置正确的Parser。
在视频中我们会从 http://www.zhenai.com/zhenghun 这个地址入手去分析对方网站的结构,采用本模拟网站时,我们只需相应的访问 即可。
不过视频中我们对于抓取数据的URL的验证过于严格。原本我们会采用正则表达式匹配http://www.zhenai.com/...等,现在需要放宽限制,只需匹配www.zhenai.com等特征串即可,不是必须要http://
即把视频正则表达式中类似 http://album.zhenai.com/u/[0-9]+ 改为 .*album\.zhenai\.com/u/[0-9]+
我将在后面的章节中详细讲述有关正则表达式,在讲到时作相应的修改便可。
具体修改请参考
https://git.imooc.com/coding-180/coding-180/commit/1c9e644e901c6c84de99ad20c14e73c45abc06ec