Repository: wyh267/shortService Branch: master Commit: f330724b84f9 Files: 18 Total size: 19.2 KB Directory structure: gitextract__l2jne5p/ ├── .gitignore ├── README.md ├── config.ini ├── install.sh ├── src/ │ ├── shortService/ │ │ ├── CountThread.go │ │ ├── OriginalProcessor.go │ │ ├── Server.go │ │ └── ShortProcessor.go │ └── shortlib/ │ ├── Configure.go │ ├── LRU.go │ ├── LRU_test.go │ ├── Processor.go │ ├── RedisAdaptor.go │ ├── Router.go │ └── Utils.go ├── start.sh ├── stop.sh └── test.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.o *.swp *github.com* *bin/ *pkg/ *.log* *jzlservice/ .DS_Store nohup.out ================================================ FILE: README.md ================================================ ## 短链接服务 #### 短连接的原理 很多人一定想的是**短连接是通过一定的算法将长链接变成短连接的，然后访问的时候再还原**，恩，非常高大上，但是仔细想想，怎么可能，那得多牛逼的压缩算法，多长的url都可以压缩为几个字节，而且还能还原，还是无损压缩。 所以，实际上，短连接生成核心就两个字：**数数**，就是不停的自增一个数，然后有个表保存每个数和原始链接的对应关系，访问短连接的时候将原是连接取出来。 知道了原理就好弄了，最简单的办法，就是用一个数组来存储，数组的索引就是短链接，数组的值就是原始链接，恩，完美，由于数组下标是短链接，那么获取短链接的时间复杂度是O(1)，同时生成短链接的时间复杂度也是O(1) #### 短链接服务的实现 实现一个短链接服务，用数组固然可能，但也显得太LOW了吧，所以为了实现这个服务，从以下几个部分来实现。 首先，给两个概念 - **解析短链接**，就是请求是短连接，返回一个跳转的原始链接 - **生成短链接**，就是有个长链接，返回生成的短链接 ##### 存储 持久化的部分使用Redis数据库来实现，很明显，key-value的结构很适合存在Redis中 这部分主要在 shortlib/RedisAdaptor.go中 ##### 计数器 数数的功能可以用Redis的自增功能实现，这样也保证了原子性，同样这部分也可以自己实现，因为go语言开线程很容易，专门开一个线程实现这个功能，通过channl来接受请求，保证是串行的就行了，不就是数数嘛，大家都会 这部分在shortlib/RedisAdaptor.go和shortService/CountThread.go中，具体实现的时候通过配置文件的参数，返回一个高阶函数，调用的时候自动分配到不同的函数实现。 ##### 缓存服务 Redis固然很快，但是我们还需要更快，要是热门数据存在内存中就更快了，而且还有个问题，就是热门的url要是有人不停的申请短连接会造成浪费，为了防止这个问题，自己实现了一个LRU模块，解析短链接的时候，命中了话直接返回结果，否则从Redis返回数据，如果是申请短链接的话，如果在LRU中，那不再重新生成短链接了。 这部分主要在 shortlib/LRU.go中。 ##### 对外服务 这一部分用的go的http框架，很容易实现高并发，没啥好说的，现在编程高并发不是问题了，连语言都自带这框架了。 这部分包括shortlib/Router.go , shortService/OriginalProcessor.go,shortService/ShortProcessor.go 这几个文件。 ================================================ FILE: config.ini ================================================ [server] port = 26719 hostname = http://t.cn/ #前缀域名 [service] threads = 200 counter = redis # inner 使用内部还是Redis进行计数 [redis] redishost = 10.254.33.20 redisport = 32079 status = false # 是否初始化Redis计数器 ================================================ FILE: install.sh ================================================ #!/bin/zsh go install shortService ================================================ FILE: src/shortService/CountThread.go ================================================ /************************************************************************* > File Name: CountThread.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 一 6/15 16:19:18 2015 ************************************************************************/ package main import ( // "fmt" "shortlib" ) func CountThread(count_chan_in chan shortlib.CountChannl) { var count int64 count = 1000 for { select { case ok := <-count_chan_in: count = count + 1 ok.CountOutChan <- count } } } ================================================ FILE: src/shortService/OriginalProcessor.go ================================================ /************************************************************************* > File Name: OriginalProcessor.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package main import ( "encoding/json" "errors" "io" "net/http" "shortlib" ) type OriginalProcessor struct { *shortlib.BaseProcessor Count_Channl chan shortlib.CountChannl } const POST string = "POST" const TOKEN string = "token" const ORIGINAL_URL string = "original" const SHORT_URL string = "short" /* * * * { "original" : "http://XX.XX.com/XXTTYD", "token" : "DFEdafaeaqh43da" } * * */ func (this *OriginalProcessor) ProcessRequest(method, request_url string, params map[string]string, body []byte, w http.ResponseWriter, r *http.Request) error { if method != POST { return errors.New("Create short url must be POST the information") } var bodyInfo map[string]interface{} err := json.Unmarshal(body, &bodyInfo) if err != nil { return err } token, has_token := bodyInfo[TOKEN].(string) original_url, has_original_url := bodyInfo[ORIGINAL_URL].(string) if !has_token || !has_original_url { return errors.New("Post info errors") } if !shortlib.IsAllowToken(token) { return errors.New("Token is not allow") } if !shortlib.IsNormalUrl(original_url) { return errors.New("Url is not normal") } short_url, err := this.createUrl(original_url) if err != nil { return err } response, err := this.createResponseJson(short_url) if err != nil { return err } //add head information header := w.Header() header.Add("Content-Type", "application/json") header.Add("charset", "UTF-8") io.WriteString(w, response) return nil } // //生成short url // // func (this *OriginalProcessor) createUrl(original_url string) (string, error) { short, err := this.Lru.GetShortURL(original_url) if err == nil { // fmt.Printf("[INFO] Match the short url : %v ===> %v\n",original_url,short) return short, nil } /* count, err := this.RedisCli.NewShortUrlCount() if err != nil { return "", err } count_c := make(chan int64) ch:=shortlib.CountChannl{0,count_c} this.Count_Channl <- ch count := <- count_c */ count, err := this.CountFunction() if err != nil { return "", err } short_url, err := shortlib.TransNumToString(count) if err != nil { return "", err } //将对应关系添加到LRU缓存中 this.Lru.SetURL(original_url, short_url) return short_url, nil } func (this *OriginalProcessor) createResponseJson(short_url string) (string, error) { json_res := make(map[string]interface{}) json_res[SHORT_URL] = this.HostName + short_url res, err := json.Marshal(json_res) if err != nil { return "", err } return string(res), nil } ================================================ FILE: src/shortService/Server.go ================================================ /************************************************************************* > File Name: Server.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package main import ( "flag" "fmt" "net/http" "os" "shortlib" ) func main() { var configFile string flag.StringVar(&configFile, "conf", "config.ini", "configure file full path") flag.Parse() //读取配置文件 fmt.Printf("[INFO] Read configure file...\n") configure, err := shortlib.NewConfigure(configFile) if err != nil { fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err) return } //启动Redis客户端 fmt.Printf("[INFO] Start Redis Client...\n") redis_cli, err := shortlib.NewRedisAdaptor(configure) if err != nil { fmt.Printf("[ERROR] Redis init fail..\n") return } //是否初始化Redis计数器，如果为ture就初始化计数器 if configure.GetRedisStatus() { err = redis_cli.InitCountService() if err != nil { fmt.Printf("[ERROR] Init Redis key count fail...\n") } } //不使用redis的情况下，启动短链接计数器 count_channl := make(chan shortlib.CountChannl, 1000) go CountThread(count_channl) countfunction := shortlib.CreateCounter(configure.GetCounterType(), count_channl, redis_cli) //启动LRU缓存 fmt.Printf("[INFO] Start LRU Cache System...\n") lru, err := shortlib.NewLRU(redis_cli) if err != nil { fmt.Printf("[ERROR]LRU init fail...\n") } //初始化两个短连接服务 fmt.Printf("[INFO] Start Service...\n") baseprocessor := &shortlib.BaseProcessor{redis_cli, configure, configure.GetHostInfo(), lru, countfunction} original := &OriginalProcessor{baseprocessor, count_channl} short := &ShortProcessor{baseprocessor} //启动http handler router := &shortlib.Router{configure, map[int]shortlib.Processor{ 0: short, 1: original, }} //启动服务 port, _ := configure.GetPort() addr := fmt.Sprintf(":%d", port) fmt.Printf("[INFO]Service Starting addr :%v,port :%v\n", addr, port) err = http.ListenAndServe(addr, router) if err != nil { //logger.Error("Server start fail: %v", err) os.Exit(1) } } ================================================ FILE: src/shortService/ShortProcessor.go ================================================ /************************************************************************* > File Name: ShortProcessor.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package main import ( "fmt" "net/http" "shortlib" ) type ShortProcessor struct { *shortlib.BaseProcessor } func (this *ShortProcessor) ProcessRequest(method, request_url string, params map[string]string, body []byte, w http.ResponseWriter, r *http.Request) error { err := shortlib.IsShortUrl(request_url) if err != nil { return err } original_url, err := this.GetOriginalURL(request_url) if err != nil { return err } fmt.Printf("REQUEST_URL: %v --- ORIGINAL_URL : %v \n", request_url, original_url) http.Redirect(w, r, original_url, http.StatusMovedPermanently) return nil } func (this *ShortProcessor) GetOriginalURL(request_url string) (string, error) { original_url, err := this.Lru.GetOriginalURL(request_url) //没有从LRU获取到地址 if err != nil { return "", err } return original_url, nil } ================================================ FILE: src/shortlib/Configure.go ================================================ /************************************************************************* > File Name: Configure.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package shortlib import ( "errors" "github.com/ewangplay/config" "strconv" ) type Configure struct { ConfigureMap map[string]string } func NewConfigure(filename string) (*Configure, error) { config := &Configure{} config.ConfigureMap = make(map[string]string) err := config.ParseConfigure(filename) if err != nil { return nil, err } return config, nil } func (this *Configure) loopConfigure(sectionName string, cfg *config.Config) error { if cfg.HasSection(sectionName) { section, err := cfg.SectionOptions(sectionName) if err == nil { for _, v := range section { options, err := cfg.String(sectionName, v) if err == nil { this.ConfigureMap[v] = options } } return nil } return errors.New("Parse Error") } return errors.New("No Section") } func (this *Configure) ParseConfigure(filename string) error { cfg, err := config.ReadDefault(filename) if err != nil { return err } this.loopConfigure("server", cfg) this.loopConfigure("service", cfg) this.loopConfigure("redis", cfg) return nil } //服务信息 func (this *Configure) GetPort() (int, error) { portstr, ok := this.ConfigureMap["port"] if ok == false { return 9090, errors.New("No Port set, use default") } port, err := strconv.Atoi(portstr) if err != nil { return 9090, err } return port, nil } func (this *Configure) GetRedisHost() (string, error) { redishost, ok := this.ConfigureMap["redishost"] if ok == false { return "127.0.0.1", errors.New("No redishost,use defualt") } return redishost, nil } func (this *Configure) GetRedisPort() (string, error) { redisport, ok := this.ConfigureMap["redisport"] if ok == false { return "6379", errors.New("No redisport,use defualt") } return redisport, nil } func (this *Configure) GetRedisStatus() bool { status, ok := this.ConfigureMap["status"] if ok == false { return true } if status == "true" { return true } return false } func (this *Configure) GetHostInfo() string { host_name, ok := this.ConfigureMap["hostname"] if ok == false { return "http://wusay.org/" } return host_name } func (this *Configure) GetCounterType() string { count_type, ok := this.ConfigureMap["counter"] if ok == false { return "inner" } return count_type } ================================================ FILE: src/shortlib/LRU.go ================================================ /************************************************************************* > File Name: LRU.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 一 6/15 17:07:37 2015 ************************************************************************/ package shortlib import ( "container/list" "errors" // "fmt" ) type UrlElement struct { Original string Short string } type LRU struct { HashShortUrl map[string]*list.Element HashOriginUrl map[string]*list.Element ListUrl *list.List RedisCli *RedisAdaptor } func NewLRU(redis_cli *RedisAdaptor) (*LRU, error) { lru := &LRU{make(map[string]*list.Element), make(map[string]*list.Element), list.New(), redis_cli} return lru, nil } func (this *LRU) GetOriginalURL(short_url string) (string, error) { element, ok := this.HashShortUrl[short_url] //没有找到key,从Redis获取 if !ok { original_url, err := this.RedisCli.GetUrl(short_url) //Redis也没有相应的短连接，无法提供服务 if err != nil { return "", errors.New("No URL") } //更新LRU缓存 ele := this.ListUrl.PushFront(UrlElement{original_url, short_url}) this.HashShortUrl[short_url] = ele this.HashOriginUrl[original_url] = ele return original_url, nil } //调整位置 this.ListUrl.MoveToFront(element) ele, _ := element.Value.(UrlElement) return ele.Original, nil } func (this *LRU) GetShortURL(original_url string) (string, error) { element, ok := this.HashOriginUrl[original_url] //没有找到key，返回错误，重新生成url if !ok { return "", errors.New("No URL") } //调整位置 this.ListUrl.MoveToFront(element) ele, _ := element.Value.(UrlElement) /* fmt.Printf("Short_Url : %v \n",short_url) for iter:=this.ListUrl.Front();iter!=nil;iter=iter.Next(){ fmt.Printf("Element:%v ElementAddr:%v\n",iter.Value,iter) } fmt.Printf("\n\n\n") for key,value := range this.HashUrl{ fmt.Printf("Key:%v ==== Value:%v\n",key,value) } */ return ele.Short, nil } func (this *LRU) SetURL(original_url, short_url string) error { ele := this.ListUrl.PushFront(UrlElement{original_url, short_url}) this.HashShortUrl[short_url] = ele this.HashOriginUrl[original_url] = ele //将数据存入Redis中 //fmt.Printf("SET TO REDIS :: short : %v ====> original : %v \n",short_url,original_url) err := this.RedisCli.SetUrl(short_url, original_url) if err != nil { return err } return nil } func (this *LRU) checkList() error { return nil } ================================================ FILE: src/shortlib/LRU_test.go ================================================ /************************************************************************* > File Name: LRU_test.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 一 6/15 19:18:06 2015 ************************************************************************/ package shortlib import ( "testing" ) func Test_SetURL(t *testing.T) { lru, _ := NewLRU(nil) err := lru.SetURL("key6", "value6") err = lru.SetURL("key5", "value5") err = lru.SetURL("key4", "value4") err = lru.SetURL("key3", "value3") err = lru.SetURL("key2", "value2") err = lru.SetURL("key1", "value1") lru.GetShortURL("key3") if err != nil { t.Error("Fail....", err) } else { t.Log("OK...\n") } } ================================================ FILE: src/shortlib/Processor.go ================================================ /************************************************************************* > File Name: Processor.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package shortlib import ( "net/http" ) type Processor interface { /* * 基础接口 * 参数：方法，url参数，请求体 * 返回：需要返回的http response */ ProcessRequest(method, request_url string, params map[string]string, body []byte, w http.ResponseWriter, r *http.Request) error } type BaseProcessor struct { RedisCli *RedisAdaptor Configure *Configure HostName string Lru *LRU CountFunction CreateCountFunc } ================================================ FILE: src/shortlib/RedisAdaptor.go ================================================ /************************************************************************* > File Name: RedisAdaptor.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 二 6/ 9 15:29:05 2015 ************************************************************************/ package shortlib import ( //"errors" "fmt" "github.com/garyburd/redigo/redis" ) type RedisAdaptor struct { conn redis.Conn config *Configure } const SHORT_URL_COUNT_KEY string = "short_url_count" func NewRedisAdaptor(config *Configure) (*RedisAdaptor, error) { redis_cli := &RedisAdaptor{} redis_cli.config = config host, _ := config.GetRedisHost() port, _ := config.GetRedisPort() connStr := fmt.Sprintf("%v:%v", host, port) fmt.Printf(connStr) conn, err := redis.Dial("tcp", connStr) if err != nil { return nil, err } redis_cli.conn = conn return redis_cli, nil } func (this *RedisAdaptor) Release() { this.conn.Close() } func (this *RedisAdaptor) InitCountService() error { _, err := this.conn.Do("SET", SHORT_URL_COUNT_KEY, 0) if err != nil { return err } return nil } func (this *RedisAdaptor) NewShortUrlCount() (int64, error) { count, err := redis.Int64(this.conn.Do("INCR", SHORT_URL_COUNT_KEY)) if err != nil { return 0, err } return count, nil } func (this *RedisAdaptor) SetUrl(short_url, original_url string) error { key := fmt.Sprintf("short:%v", short_url) _, err := this.conn.Do("SET", key, original_url) if err != nil { return err } return nil } func (this *RedisAdaptor) GetUrl(short_url string) (string, error) { key := fmt.Sprintf("short:%v", short_url) original_url, err := redis.String(this.conn.Do("GET", key)) if err != nil { return "", err } return original_url, nil } ================================================ FILE: src/shortlib/Router.go ================================================ /************************************************************************* > File Name: Router.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 16:00:54 2015 ************************************************************************/ package shortlib import ( "fmt" "io" "io/ioutil" "net/http" "regexp" ) type Router struct { Configure *Configure Processors map[int]Processor } const ( SHORT_URL = 0 ORIGINAL_URL = 1 UNKOWN_URL = 2 ) //路由设置 //数据分发 func (this *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := TimeNow() request_url := r.RequestURI[1:] action, err := this.ParseUrl(request_url) if err != nil { fmt.Printf("[ERROR]parse url fail : %v \n", err) } err = r.ParseForm() if err != nil { return } params := make(map[string]string) for k, v := range r.Form { params[k] = v[0] } body, err := ioutil.ReadAll(r.Body) if err != nil && err != io.EOF { return } if r.Method == "GET" { action = 0 } else { action = 1 } processor, _ := this.Processors[action] err = processor.ProcessRequest(r.Method, request_url, params, body, w, r) if err != nil { fmt.Printf("[ERROR] : %v\n", err) } if action == 0 { DuringTime(start, "REDIRECT URL ") } else { DuringTime(start, "CREATE SHORTURL ") } return } func (this *Router) ParseUrl(url string) (action int, err error) { if this.isShortUrl(url) { return SHORT_URL, nil } else { return ORIGINAL_URL, nil } } func (this *Router) isShortUrl(url string) bool { short_url_pattern := "XXXX" url_reg_exp, err := regexp.Compile(short_url_pattern) if err != nil { return false } short_match := url_reg_exp.FindStringSubmatch(url) if short_match == nil { return false } return true } ================================================ FILE: src/shortlib/Utils.go ================================================ /************************************************************************* > File Name: Utils.go > Author: Wu Yinghao > Mail: wyh817@gmail.com > Created Time: 日 6/14 18:05:47 2015 ************************************************************************/ package shortlib import ( "container/list" "fmt" "time" ) type CountChannl struct { Ok int64 CountOutChan chan int64 } type CreateCountFunc func() (int64, error) func IsAllowToken(token string) bool { return true } func IsNormalUrl(url string) bool { return true } func TransNumToString(num int64) (string, error) { startTime := TimeNow() var base int64 base = 62 baseHex := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" output_list := list.New() for num/base != 0 { output_list.PushFront(num % base) num = num / base } output_list.PushFront(num % base) str := "" for iter := output_list.Front(); iter != nil; iter = iter.Next() { str = str + string(baseHex[int(iter.Value.(int64))]) } DuringTime(startTime, "TransNumToString") return str, nil } func TransStringToNum(str string) (int64, error) { return 0, nil } func TimeNow() time.Time { return time.Now() } func DuringTime(start time.Time, taskname string) { endTime := time.Now() fmt.Printf("[INFO] [ %v ] COST Time %v \n", taskname, endTime.Sub(start)) } func IsShortUrl(short_url string) error { return nil } func CreateCounter(count_type string, count_chan chan CountChannl, rediscli *RedisAdaptor) CreateCountFunc { if count_type == "inner" { return func() (int64, error) { count_c := make(chan int64) ch := CountChannl{0, count_c} count_chan <- ch count := <-count_c return count, nil } } else { return func() (int64, error) { count, err := rediscli.NewShortUrlCount() if err != nil { return 0, err } return count, nil } } } ================================================ FILE: start.sh ================================================ #!/bin/sh # 在后台启动snsscheduler-server服务 nohup ./bin/ProxyServer & ================================================ FILE: stop.sh ================================================ #!/bin/sh killall ProxyServer ================================================ FILE: test.sh ================================================ #!/bin/sh go test ProxyServer