Repository: alanhou/golang-streaming Branch: master Commit: 5b17bafd6192 Files: 41 Total size: 67.7 KB Directory structure: gitextract_uhks32hd/ ├── .gitignore ├── README.md ├── video_server/ │ ├── api/ │ │ ├── auth.go │ │ ├── dbops/ │ │ │ ├── api.go │ │ │ ├── api_test.go │ │ │ ├── conn.go │ │ │ └── internal.go │ │ ├── defs/ │ │ │ ├── apidef.go │ │ │ └── errs.go │ │ ├── handlers.go │ │ ├── main.go │ │ ├── response.go │ │ ├── session/ │ │ │ └── ops.go │ │ └── utils/ │ │ └── uuid.go │ ├── build.sh │ ├── scheduler/ │ │ ├── dbops/ │ │ │ ├── api.go │ │ │ ├── conn.go │ │ │ └── internal.go │ │ ├── handlers.go │ │ ├── main.go │ │ ├── response.go │ │ └── taskrunner/ │ │ ├── defs.go │ │ ├── runner.go │ │ ├── runner_test.go │ │ ├── tasks.go │ │ └── trmain.go │ ├── streamserver/ │ │ ├── defs.go │ │ ├── handlers.go │ │ ├── limiter.go │ │ ├── main.go │ │ ├── response.go │ │ └── upload.html │ ├── templates/ │ │ ├── home.html │ │ ├── scripts/ │ │ │ └── home.js │ │ └── userhome.html │ └── web/ │ ├── client.go │ ├── defs.go │ ├── handlers.go │ └── main.go └── webserver/ ├── main.go └── main_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.iml .idea/ ================================================ FILE: README.md ================================================ # Go语言实战流媒体视频网站 学习笔记链接：https://alanhou.org/golang-video-streaming/ ## 第1章 课程介绍 介绍这门课程大纲，技术堆栈以及环境 * 1-1 prestudy * 1-2 课程介绍及知识要点预习 ## 第2章 一个例子了解golang工具链 通过一个简单的webservice具体从golang的工具链，到test，全面介绍golang在工程项目里需要掌握的知识点。 * 2-1 一个例子了解golang常用工具链 * 2-2 golang项目中test的写法 * 2-3 golang项目中benchmark的写法 * 2-4 章节总结 ## 第3章 流媒体网站架构以及API模块的实现 本章通过实战演练，从网站的整体架构设计，到服务划分，数据库设计，到api模块的实现，全面讲述golang对webservice的实现以及代码分层架构的思想，同时辅以test cases的全程编写与指导，全面了解工程化golang项目的实现。 * 3-1 流媒体网站整体介绍与架构梳理 * 3-2 api设计与架构 试看 * 3-3 api实现之详细设计(上) * 3-4 api实现之详细设计(中) * 3-5 api实现之详细设计(下) * 3-6 api之http handler层 * 3-7 api之数据库层设计 * 3-8 api之数据库层实现_数据库连接 * 3-9 api之数据库层实现_实现User * 3-10 api之数据库层实现_编写User Test Case * 3-11 api之数据库层实现_User部分代码优化 * 3-12 api之数据库层实现_实现和验证Video * 3-13 api之数据库层实现_实现Comments * 3-14 api之数据库层实现_Comments Test Case * 3-15 api之session处理与实现(上) * 3-16 api之session处理与实现(下) * 3-17 api之http middleware的实现与handler收尾(上) * 3-18 api之http middleware的实现与handler收尾(下) ## 第4章 stream模块 通过stream server的实现过程，着重讲述通过golang实现流式播放，上传文件，以及利用channel实现流控等实用知识点，进一步加深对golang的掌握。 * 4-1 stream server * 4-2 streaming的架构搭建 * 4-3 token bucket * 4-4 流控模块的实现 试看 * 4-5 在http middleware中嵌入流控 * 4-6 streamHandler实现 * 4-7 验证streamHandler * 4-8 uploadHandler实现 * 4-9 验证uploadHandler ## 第5章 scheduler模块 通过对生产者消费者模型在scheduler中的实现，全面了解golang是如何处理并发场景，以及如何在并发场景下通过channel实现消息同步。 * 5-1 scheduler介绍 * 5-2 代码架构搭建 * 5-3 runner的生产消费者模型实现 * 5-4 runner的使用与测试 * 5-5 task示例的实现 * 5-6 timer的实现 * 5-7 api实现以及scheduler完成 ## 第6章 前端服务和模版引擎渲染 讲述如何使用golang的模版引擎来渲染html文件，如何通过原生proxy和api两种模式实现后端服务接口透传并避免跨域访问，以及整个前台在实现业务上的js逻辑代码。 * 6-1 大前端和golang模版引擎介绍 * 6-2 前端代码架构搭建 * 6-3 静态页面渲染* * 6-4 build脚本和homeHandler * 6-5 userHomeHandler * 6-6 api透传模块实现 * 6-7 proxy转发的实现 * 6-8 UI部分的预览 * 6-9 API service补全与讲解 * 6-10 UI之html讲解 * 6-11 js部分实现 ## 第7章 网站上云 通过对网站部分架构的改造和代码重构，使之更符合cloud native架构，辅以阿里云计算存储网络等服务，最终实现网站上云，打通网站上线最后一公里。 * 7-1 云原生讲解 * 7-2 云存储改造之OSS方案分析 * 7-3 云存储改造之OSS适配 * 7-4 公共配置实现 * 7-5 用vendor处理公共配置包 * 7-6 SLB讲解与配置 * 7-7 SLB之添加session容错 * 7-8 ECS云主机和安全组配置 * 7-9 scheduler的改造 * 7-10 部署脚本以及db初始化 * 7-11 部署演示以及完成效果展示 * 7-12 课程总结（回顾，延伸和优化） credit：慕课网 ================================================ FILE: video_server/api/auth.go ================================================ package main import ( "golang-streaming/video_server/api/defs" "golang-streaming/video_server/api/session" "net/http" ) var HEADER_FIELD_SESSION = "X-Session-Id" var HEADER_FIELD_UNAME = "X-User-Name" func ValidateUserSession(r *http.Request) bool { sid := r.Header.Get(HEADER_FIELD_SESSION) if len(sid) == 0 { return false } uname, ok := session.IsSessionExpired(sid) if ok { return false } r.Header.Add(HEADER_FIELD_UNAME, uname) return true } func ValidateUser(w http.ResponseWriter, r *http.Request) bool { uname := r.Header.Get(HEADER_FIELD_UNAME) if len(uname) == 0 { sendErrorResponse(w, defs.ErrorNotAuthUser) return false } return true } ================================================ FILE: video_server/api/dbops/api.go ================================================ package dbops import ( "golang-streaming/video_server/api/defs" "golang-streaming/video_server/api/utils" "database/sql" _ "github.com/go-sql-driver/mysql" "log" "time" ) func AddUserCredential(loginName string, pwd string) error { stmtIns, err := dbConn.Prepare("INSERT INTO users (login_name, pwd) VALUES (?, ?)") if err != nil { return err } _, err = stmtIns.Exec(loginName, pwd) if err != nil { return err } stmtIns.Close() return nil } func GetUserCredential(loginName string) (string, error) { stmtOut, err := dbConn.Prepare("SELECT pwd FROM users WHERE login_name = ?") if err != nil { log.Printf("%s", err) return "", err } var pwd string err = stmtOut.QueryRow(loginName).Scan(&pwd) if err != nil && err != sql.ErrNoRows { return "", err } defer stmtOut.Close() return pwd, nil } func DeleteUser(loginName string, pwd string) error { stmtDel, err := dbConn.Prepare("DELETE FROM users WHERE login_name = ? AND pwd = ?") if err != nil { log.Printf("DeleteUser error: %s", err) return err } _, err = stmtDel.Exec(loginName, pwd) if err != nil { return err } defer stmtDel.Close() return nil } func GetUser(loginName string) (*defs.User, error) { stmtOut, err:=dbConn.Prepare("SELECT id, pwd FROM users WHERE login_name=?") if err!=nil { log.Printf("%s", err) return nil, err } var id int var pwd string err=stmtOut.QueryRow(loginName).Scan(&id, &pwd) if err!= nil && err!=sql.ErrNoRows { return nil, err } if err==sql.ErrNoRows { return nil, nil } res:=&defs.User{Id: id, LoginName: loginName, Pwd: pwd} defer stmtOut.Close() return res, nil } func AddNewVideo(aid int, name string) (*defs.VideoInfo, error) { // create uuid vid, err := utils.NewUUID() if err != nil { return nil, err } t := time.Now() ctime := t.Format("Jan 02 2006, 15:04:05") // M D y, HH:MM:SS stmtIns, err := dbConn.Prepare(`INSERT INTO video_info (id, author_id, name, display_ctime) VALUES(?, ?, ?, ?)`) if err != nil { return nil, err } _, err = stmtIns.Exec(vid, aid, name, ctime) if err != nil { return nil, err } res := &defs.VideoInfo{Id: vid, AuthorId: aid, Name: name, DisplayCtime:ctime} defer stmtIns.Close() return res, nil } func GetVideoInfo(vid string) (*defs.VideoInfo, error) { // create uuid stmtOut, err := dbConn.Prepare("SELECT author_id, name, display_ctime FROM video_info WHERE id=?") var aid int var dct string var name string err = stmtOut.QueryRow(vid).Scan(&aid, &name, &dct) if err != nil && err != sql.ErrNoRows { return nil, err } if err == sql.ErrNoRows { return nil, nil } defer stmtOut.Close() res := &defs.VideoInfo{Id: vid, AuthorId: aid, Name: name, DisplayCtime: dct} return res, nil } func DeleteVideoInfo(vid string) error { stmtDel, err := dbConn.Prepare("DELETE FROM video_info WHERE id = ?") if err != nil { return err } _, err = stmtDel.Exec(vid) if err != nil { return err } defer stmtDel.Close() return nil } func AddNewComments(vid string, aid int, content string) error { id, err := utils.NewUUID() if err != nil { return err } stmtIns, err := dbConn.Prepare("INSERT INTO comments (id, video_id, author_id, content) VALUES (?, ?, ?, ?)") if err != nil { return err } _, err = stmtIns.Exec(id, vid, aid, content) if err != nil { return err } defer stmtIns.Close() return nil } func ListComments(vid string, from, to int) ([]*defs.Comment, error) { stmtOut, err := dbConn.Prepare(`SELECT comments.id, users.login_name, comments.content FROM comments INNER JOIN users ON comments.author_id = users.id WHERE comments.video_id = ? AND comments.time > FROM_UNIXTIME(?) AND comments.time <= FROM_UNIXTIME(?) ORDER BY comments.time DESC`) var res []*defs.Comment rows, err := stmtOut.Query(vid, from, to) if err != nil { return res, err } for rows.Next() { var id, name, content string if err := rows.Scan(&id, &name, &content); err != nil { return res, err } c := &defs.Comment{Id: id, VideoId: vid, Author: name, Content: content} res = append(res, c) } defer stmtOut.Close() return res, nil } func ListVideoInfo(uname string, from, to int) ([]*defs.VideoInfo, error) { stmtOut, err:=dbConn.Prepare(`SELECT video_info.id, video_info.author_id, video_info.name, video_info.display_ctime FROM video_info INNER JOIN users ON video_info.author_id = users.id WHERE users.login_name=? AND video_info.create_time > FROM_UNIXTIME(?) AND video_info.create_time<=FROM_UNIXTIME(?) OREDER BY video_info.create_time DESC`) var res []*defs.VideoInfo if err!=nil{ return res, err } rows, err:=stmtOut.Query(uname, from, to) if err!=nil { log.Printf("%s", err) return res, err } for rows.Next() { var id, name, ctime string var aid int if err:=rows.Scan(&id, &aid, &name, &ctime); err!=nil{ return res, err } vi:=&defs.VideoInfo{Id:id, AuthorId:aid, Name:name, DisplayCtime: ctime} res = append(res, vi) } defer stmtOut.Close() return res, nil } ================================================ FILE: video_server/api/dbops/api_test.go ================================================ package dbops import ( "fmt" "strconv" "testing" "time" ) //init(dblogin, truncate tables) -> run tests -> clear data(truncate tables) var tempvid string func clearTables() { dbConn.Exec("truncate users") dbConn.Exec("truncate video_info") dbConn.Exec("truncate comments") dbConn.Exec("truncate sessions") } func TestMain(m *testing.M) { clearTables() m.Run() clearTables() } func TestUserWorkFlow(t *testing.T) { t.Run("Add", testAddUser) t.Run("Get", testGetUser) t.Run("Del", testDeleteUser) t.Run("Reget", testRegetUser) } func testAddUser(t *testing.T) { err := AddUserCredential("alan", "123") if err != nil { t.Errorf("Error of AddUser: %v", err) } } func testGetUser(t *testing.T) { pwd, err := GetUserCredential("alan") if pwd != "123" || err != nil { t.Errorf("Error of GetUser") } } func testDeleteUser(t *testing.T) { err := DeleteUser("alan", "123") if err != nil { t.Errorf("Error of DeleteUser: %v", err) } } func testRegetUser(t *testing.T) { pwd, err := GetUserCredential("alan") if err != nil { t.Errorf("Error of RegetUser: %v", err) } if pwd != "" { t.Errorf("Deleting user test failed") } } func TestVideoWorkFlow(t *testing.T) { clearTables() t.Run("PrepareUser", testAddUser) t.Run("AddVideo", testAddVideoInfo) t.Run("GetVideo", testGetVideoInfo) t.Run("DelVideo", testDeleteVideoInfo) t.Run("RegetVideo", testRegetVideoInfo) } func testAddVideoInfo(t *testing.T) { vi, err := AddNewVideo(1, "my-video") if err != nil { t.Errorf("Error of AddVideoInfo: %v", err) } tempvid = vi.Id } func testGetVideoInfo(t *testing.T) { _, err := GetVideoInfo(tempvid) if err != nil { t.Errorf("Error of GetVideoInfo: %v", err) } } func testDeleteVideoInfo(t *testing.T) { err := DeleteVideoInfo(tempvid) if err != nil { t.Errorf("Error of DeleteVideoInfo: %v", err) } } func testRegetVideoInfo(t *testing.T) { vi, err := GetVideoInfo(tempvid) if err != nil || vi != nil { t.Errorf("Error of RegetVideoInfo: %v", err) } } func TestComments(t *testing.T) { clearTables() t.Run("AddUser", testAddUser) t.Run("AddComments", testAddComments) t.Run("ListComments", testListComments) } func testAddComments(t *testing.T) { vid := "12345" aid := 1 content := "I like this video" err := AddNewComments(vid, aid, content) if err != nil { t.Errorf("Error of AddComments: %v", err) } } func testListComments(t *testing.T) { vid := "12345" from := 1560960000 to, _ := strconv.Atoi(strconv.FormatInt(time.Now().UnixNano()/1000000000, 10)) res, err := ListComments(vid, from ,to) if err != nil { t.Errorf("Error of ListComments: %v", err) } for i, ele := range res { fmt.Printf("comment: %d, %v \n", i, ele) } } ================================================ FILE: video_server/api/dbops/conn.go ================================================ package dbops import ( "database/sql" _ "github.com/go-sql-driver/mysql" ) var ( dbConn *sql.DB err error ) func init() { dbConn, err = sql.Open("mysql", "root:@tcp(127.0.0.1:3306)/video_server?charset=utf8") if err != nil { panic(err.Error()) } } ================================================ FILE: video_server/api/dbops/internal.go ================================================ package dbops import ( "golang-streaming/video_server/api/defs" "database/sql" "log" "strconv" "sync" ) func InsertSession(sid string, ttl int64, uname string) error { ttlstr := strconv.FormatInt(ttl, 10) stmtIns, err := dbConn.Prepare("INSERT INTO sessions (session_id, TTL, login_name) VALUE (?, ?, ?)") if err != nil { return err } _, err = stmtIns.Exec(sid, ttlstr, uname) if err != nil { return err } defer stmtIns.Close() return nil } func RetrieveSession(sid string) (*defs.SimpleSession, error) { ss := &defs.SimpleSession{} stmtOut, err := dbConn.Prepare("SELECT TTL, login_name from sessions WHERE session_id = ?") if err != nil { return nil, err } var ttl string var uname string stmtOut.QueryRow(sid).Scan(&ttl, &uname) if err != nil && err != sql.ErrNoRows { return nil, err } if res, err := strconv.ParseInt(ttl, 10, 64); err == nil { ss.TTL = res ss.Username = uname } else { return nil, err } defer stmtOut.Close() return ss, nil } func RetrieveAllSessions() (*sync.Map, error) { m := &sync.Map{} stmtOut, err := dbConn.Prepare("SELECT * FROM sessions") if err != nil { log.Printf("%s", err) return nil, err } rows, err := stmtOut.Query() if err != nil { log.Printf("%s", err) return nil, err } for rows.Next() { var id string var ttlstr string var login_name string if err := rows.Scan(&id, &ttlstr, &login_name); err != nil { log.Printf("retrieve sessions error: %s", err) break } if ttl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil { ss := &defs.SimpleSession{Username: login_name, TTL: ttl} m.Store(id, ss) log.Printf("session id: 5s, ttl: %d", id, ss.TTL) } } return m, nil } func DeleteSession(sid string) error { stmtOut, err := dbConn.Prepare("DELETE FROM sessions WHERE session_id = ?") if err != nil { log.Printf("%s", err) return err } if _, err := stmtOut.Query(sid); err != nil { return err } return nil } ================================================ FILE: video_server/api/defs/apidef.go ================================================ package defs //requests type UserCredential struct { Username string `json:"username"` Pwd string `json:"pwd"` } type NewComment struct { AuthorId int `json:"author_id"` Content string `json:"content"` } type NewVideo struct { AuthorId int `json:"author_id"` Name string `json:"name"` } // response type SignedUp struct { Success bool `json:"success"` SessionId string `json:"session_id"` } type UserSession struct { Username string `json:"user_name"` SessionId string `json:"session_id"` } type UserInfo struct { Id int `json:"id""` } type SignedIn struct { Success bool `json:"success"` SessionId string `json:"session_id"` } type VideosInfo struct { Videos []*VideoInfo `json:"videos"` } type Comments struct { Comments []*Comment `json:"comments"` } // Data model type User struct { Id int LoginName string Pwd string } type VideoInfo struct { Id string `json:"id"` AuthorId int `json:"author_id"` Name string `json:"name"` DisplayCtime string `json:"display_ctime"` } type Comment struct { Id string `json:"id"` VideoId string `json:"video_id"` Author string `json:"author"` Content string `json:"content"` } type SimpleSession struct { Username string // login name TTL int64 } ================================================ FILE: video_server/api/defs/errs.go ================================================ package defs type Err struct { Error string `json:"error"` ErrorCode string `json:"error_code"` } type ErrResponse struct { HttpSC int Error Err } var ( ErrorRequestBodyParseFailed = ErrResponse{HttpSC: 400, Error: Err{Error: "Request body is not correct", ErrorCode: "001"}} ErrorNotAuthUser = ErrResponse{HttpSC: 401, Error: Err{Error: "User authentication failed", ErrorCode: "002"}} ErrorDBError = ErrResponse{HttpSC: 500, Error: Err{Error: "DB ops failed", ErrorCode: "003"}} ErrorInternalFaults = ErrResponse{HttpSC: 500, Error: Err{Error: "Internal service error", ErrorCode: "004"}} ) ================================================ FILE: video_server/api/handlers.go ================================================ package main import ( "encoding/json" "github.com/julienschmidt/httprouter" "golang-streaming/video_server/api/dbops" "golang-streaming/video_server/api/defs" "golang-streaming/video_server/api/session" "golang-streaming/video_server/api/utils" "io/ioutil" "log" "net/http" ) func CreateUser(w http.ResponseWriter, r *http.Request, p httprouter.Params) { res, _ := ioutil.ReadAll(r.Body) ubody := &defs.UserCredential{} if err := json.Unmarshal(res, ubody); err != nil { sendErrorResponse(w, defs.ErrorRequestBodyParseFailed) return } if err := dbops.AddUserCredential(ubody.Username, ubody.Pwd); err != nil { sendErrorResponse(w, defs.ErrorDBError) return } id := session.GenerateNewSessionId(ubody.Username) su := &defs.SignedUp{Success: true, SessionId: id} if resp, err := json.Marshal(su); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) return } else { sendNormalResponse(w, string(resp), 201) } } func Login(w http.ResponseWriter, r *http.Request, p httprouter.Params) { res, _ := ioutil.ReadAll(r.Body) log.Printf("%s", res) ubody := &defs.UserCredential{} if err := json.Unmarshal(res, ubody); err != nil { log.Printf("%s", err) // io.WriteString(w, "wrong") sendErrorResponse(w, defs.ErrorRequestBodyParseFailed) return } // Validate the request body uname := p.ByName("username") log.Printf("Login url name: %s", uname) log.Printf("Login body name: %s", ubody.Username) if uname != ubody.Username { sendErrorResponse(w, defs.ErrorNotAuthUser) return } log.Printf("%s", ubody.Username) pwd, err := dbops.GetUserCredential(ubody.Username) log.Printf("Login pwd: %s", pwd) if err != nil || len(pwd) == 0 || pwd != ubody.Pwd { sendErrorResponse(w, defs.ErrorNotAuthUser) return } id := session.GenerateNewSessionId(ubody.Username) si := &defs.SignedIn{Success: true, SessionId: id} if resp, err := json.Marshal(si); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) } else { sendNormalResponse(w, string(resp), 200) } //io.WriteString(w, "signed in") } func GetUserInfo(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if !ValidateUser(w, r) { log.Printf("Unauthorized user \n") } uname := p.ByName("username") u, err := dbops.GetUser(uname) if err != nil { log.Printf("Erorr in GetUserinfo: %s", err) sendErrorResponse(w, defs.ErrorDBError) return } ui := &defs.UserInfo{Id: u.Id} if resp, err := json.Marshal(ui); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) } else { sendNormalResponse(w, string(resp), 200) } } func AddNewVideo(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if !ValidateUser(w, r) { log.Printf("Unauthorized user \n") return } res, _ := ioutil.ReadAll(r.Body) nvbody := &defs.NewVideo{} if err := json.Unmarshal(res, nvbody); err != nil { log.Printf("%s", err) sendErrorResponse(w, defs.ErrorRequestBodyParseFailed) return } vi, err := dbops.AddNewVideo(nvbody.AuthorId, nvbody.Name) log.Printf("Author id : %d, name: %s \n", nvbody.AuthorId, nvbody.Name) if err != nil { log.Printf("Error in AddNewVideo: 5s", err) sendErrorResponse(w, defs.ErrorDBError) return } if resp, err := json.Marshal(vi); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) } else { sendNormalResponse(w, string(resp), 201) } } func ListAllVideos(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if !ValidateUser(w ,r) { return } uname := p.ByName("username") vs, err := dbops.ListVideoInfo(uname, 0, utils.GetCurrentTimestampSec()) if err != nil { log.Printf("Error in ListAllVideos: %s", err) sendErrorResponse(w, defs.ErrorDBError) return } vsi := &defs.VideosInfo{Videos: vs} if resp, err := json.Marshal(vsi); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) } else { sendNormalResponse(w, string(resp), 200) } } func DeleteVideo(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if !ValidateUser(w ,r) { return } vid := p.ByName("vid-id") err := dbops.DeleteVideoInfo(vid) if err != nil { log.Printf("Error in DeleteVideo: %s", err) sendErrorResponse(w, defs.ErrorDBError) return } sendNormalResponse(w, "", 204) } func PostComment(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if ValidateUser(w, r) { return } reqBody, _ := ioutil.ReadAll(r.Body) cbody := &defs.NewComment{} if err := json.Unmarshal(reqBody, cbody); err != nil { log.Printf("%s", err) sendErrorResponse(w, defs.ErrorRequestBodyParseFailed) return } vid := p.ByName("vid-id") if err := dbops.AddNewComments(vid, cbody.AuthorId, cbody.Content); err != nil { log.Printf("Error in PostComment: %s", err) sendErrorResponse(w, defs.ErrorDBError) } else { sendNormalResponse(w, "ok", 201) } } func ShowComments(w http.ResponseWriter, r *http.Request, p httprouter.Params) { if !ValidateUser(w, r) { return } vid := p.ByName("vid-id") cm, err := dbops.ListComments(vid, 0, utils.GetCurrentTimestampSec()) if err != nil { log.Printf("Error in ShowComments: %s", err) sendErrorResponse(w, defs.ErrorDBError) return } cms := &defs.Comments{Comments: cm} if resp, err := json.Marshal(cms); err != nil { sendErrorResponse(w, defs.ErrorInternalFaults) } else { sendNormalResponse(w, string(resp), 200) } } ================================================ FILE: video_server/api/main.go ================================================ package main import ( "github.com/julienschmidt/httprouter" "golang-streaming/video_server/api/session" "net/http" ) type middleWareHandler struct { r *httprouter.Router } func NewMiddleWareHandler(r *httprouter.Router) http.Handler { m := middleWareHandler{} m.r = r return m } func (m middleWareHandler) ServeHTTP(w http.ResponseWriter, r * http.Request) { ValidateUserSession(r) m.r.ServeHTTP(w, r) } func RegisterHandlers() *httprouter.Router { router := httprouter.New() router.POST("/user", CreateUser) router.POST("/user/:username", Login) router.GET("/user/:username", GetUserInfo) router.POST("/user/:username/videos", AddNewVideo) router.GET("/user/:username/videos", ListAllVideos) router.DELETE("/user/:username/videos/:vid-id", DeleteVideo) router.POST("/videos/:vid-id/comments", PostComment) router.GET("/videos/:vid-id/comments", ShowComments) return router } func Prepare() { session.LoadSessionsFromDB() } func main() { Prepare() r := RegisterHandlers() mh := NewMiddleWareHandler(r) http.ListenAndServe(":8000", mh) } ================================================ FILE: video_server/api/response.go ================================================ package main import ( "encoding/json" "golang-streaming/video_server/api/defs" "io" "net/http" ) func sendErrorResponse(w http.ResponseWriter, errResp defs.ErrResponse) { w.WriteHeader(errResp.HttpSC) resStr, _ := json.Marshal(&errResp.Error) io.WriteString(w, string(resStr)) } func sendNormalResponse(w http.ResponseWriter, resp string, sc int) { w.WriteHeader(sc) io.WriteString(w, resp) } ================================================ FILE: video_server/api/session/ops.go ================================================ package session import ( "golang-streaming/video_server/api/dbops" "golang-streaming/video_server/api/defs" "golang-streaming/video_server/api/utils" "sync" "time" ) var sessionMap *sync.Map func init() { sessionMap = &sync.Map{} } func noInMilli() int64 { return time.Now().UnixNano()/1000000 } func deleteExpiredSession(sid string) { sessionMap.Delete(sid) dbops.DeleteSession(sid) } func LoadSessionsFromDB() { r, err := dbops.RetrieveAllSessions() if err != nil { return } r.Range(func(k, v interface{}) bool { ss := v.(*defs.SimpleSession) sessionMap.Store(k, ss) return true }) } func GenerateNewSessionId(un string) string { id, _ := utils.NewUUID() ct := noInMilli() ttl := ct + 30 * 60 * 1000 // Server side session valid time: 30 min ss := &defs.SimpleSession{Username: un, TTL: ttl} sessionMap.Store(id, ss) dbops.InsertSession(id, ttl, un) return id } func IsSessionExpired(sid string) (string, bool) { ss, ok := sessionMap.Load(sid) if ok { ct := noInMilli() if ss.(*defs.SimpleSession).TTL < ct { deleteExpiredSession(sid) return "", true } return ss.(*defs.SimpleSession).Username, false } return "", true } ================================================ FILE: video_server/api/utils/uuid.go ================================================ package utils import ( "crypto/rand" "fmt" "io" "strconv" "time" ) func NewUUID() (string, error) { uuid := make([]byte, 16) n, err := io.ReadFull(rand.Reader, uuid) if n != len(uuid) || err != nil { return "", err } // variant bits; see section 4.1.1 uuid[8] = uuid[8]&^0xc0 | 0x80 // version 4 (pseudo-random); see section 4.1.3 uuid[6] = uuid[6]&^0xf0 | 0x40 return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6],uuid[6:8], uuid[8:10], uuid[10:]), nil } func GetCurrentTimestampSec() int { ts, _ := strconv.Atoi(strconv.FormatInt(time.Now().UnixNano()/1000000000, 10)) return ts } ================================================ FILE: video_server/build.sh ================================================ #! /bin/bash # Build web UI cd ~/go/src/golang-streaming/video_server/web/ go install cp ~/go/bin/web ~/go/bin/video_server_web_ui/web cp -R ~/go/src/golang-streaming/video_server/templates ~/go/bin/video_server_web_ui/web ================================================ FILE: video_server/scheduler/dbops/api.go ================================================ package dbops import "log" //1. user -> api service -> delete video //2. api service -> scheduler -> write video deletion record //3. timer //4. timer -> runner -> read wvdr -> exec -> delete video form folder func AddVideoDeletionRecord(vid string) error { stmtIns, err := dbConn.Prepare("INSERT INTO video_del_rec (video_id) VALUES (?)") if err != nil { return err } _, err = stmtIns.Exec(vid) if err != nil { log.Printf("AddVideoDeletionRecord error: %v", err) return err } defer stmtIns.Close() return nil } ================================================ FILE: video_server/scheduler/dbops/conn.go ================================================ package dbops import ( "database/sql" _ "github.com/go-sql-driver/mysql" ) var ( dbConn *sql.DB err error ) func init() { dbConn, err = sql.Open("mysql", "root:@tcp(127.0.0.1:3306)/video_server?charset=utf8") if err != nil { panic(err.Error()) } } ================================================ FILE: video_server/scheduler/dbops/internal.go ================================================ package dbops import "log" func ReadVideoDeletionRecord(count int) ([]string, error) { stmtOut, err := dbConn.Prepare("SELECT video_id FROM video_del_rec LIMIT ?") var ids []string if err != nil { return ids, err } rows, err := stmtOut.Query(count) if err != nil { log.Printf("Query VideoDeletionRecord error: %v", err) return ids, err } for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return ids, err } ids = append(ids, id) } defer stmtOut.Close() return ids, nil } func DelVideoDeletionRecord(vid string) error { stmtDel, err := dbConn.Prepare("DELETE FROM video_del_rec WHERE video_id = ?") if err != nil { return err } _, err = stmtDel.Exec(vid) if err != nil { log.Printf("Deleting VideoDeletionRecord: %v", err) return err } defer stmtDel.Close() return nil } ================================================ FILE: video_server/scheduler/handlers.go ================================================ package main import ( "github.com/julienschmidt/httprouter" "golang-streaming/video_server/scheduler/dbops" "net/http" ) func vidDelRecHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { vid := p.ByName("vid-id") if len(vid) == 0 { sendResponse(w, 400, "Video id should not be empty") return } err := dbops.AddVideoDeletionRecord(vid) if err != nil { sendResponse(w, 500, "Internal server error") return } sendResponse(w, 200, "") return } ================================================ FILE: video_server/scheduler/main.go ================================================ package main import ( "github.com/julienschmidt/httprouter" "golang-streaming/video_server/scheduler/taskrunner" "net/http" ) func RegisterHandlers() *httprouter.Router { router := httprouter.New() router.GET("/video-delete-record/:vid-id", vidDelRecHandler) return router } func main() { go taskrunner.Start() r := RegisterHandlers() http.ListenAndServe(":9001", r) } ================================================ FILE: video_server/scheduler/response.go ================================================ package main import ( "io" "net/http" ) func sendResponse(w http.ResponseWriter, sc int, resp string) { w.WriteHeader(sc) io.WriteString(w, resp) } ================================================ FILE: video_server/scheduler/taskrunner/defs.go ================================================ package taskrunner const ( READY_TO_DISPATCH = "d" READY_TO_EXCUTE = "e" CLOSE = "c" VIDEO_PATH = "./videos/" ) type controlChan chan string type dataChan chan interface{} type fn func(dc dataChan) error ================================================ FILE: video_server/scheduler/taskrunner/runner.go ================================================ package taskrunner type Runner struct { Controller controlChan Error controlChan Data dataChan dataSize int longLived bool Dispatcher fn Executor fn } func NewRunner(size int, longlived bool, d fn, e fn) *Runner { return &Runner{ Controller: make(chan string, 1), Error: make(chan string, 1), Data: make(chan interface{}, size), longLived: longlived, dataSize: size, Dispatcher: d, Executor: e, } } func (r *Runner) startDispatch() { defer func() { if !r.longLived { close(r.Controller) close(r.Data) close(r.Error) } }() for { select { case c := <- r.Controller: if c == READY_TO_DISPATCH { err := r.Dispatcher(r.Data) if err != nil { r.Error <- CLOSE } else { r.Controller <- READY_TO_EXCUTE } } if c == READY_TO_EXCUTE { err := r.Executor(r.Data) if err != nil { r.Error <- CLOSE } else { r.Controller <- READY_TO_DISPATCH } } case e := <- r.Error: if e == CLOSE { return } default: } } } func (r *Runner) StartAll() { r.Controller <- READY_TO_DISPATCH r.startDispatch() } ================================================ FILE: video_server/scheduler/taskrunner/runner_test.go ================================================ package taskrunner import ( "github.com/pkg/errors" "log" "testing" "time" ) func TestRuner(t *testing.T) { d := func(dc dataChan) error { for i := 0; i < 30; i++ { dc <- i log.Printf("Dispatcher sent: %v", i) } return nil } e := func(dc dataChan) error { forloop: for { select { case d := <- dc: log.Printf("Executor received: %v", d) default: break forloop } } return errors.New("Executor") } runner := NewRunner(30, false, d, e) go runner.StartAll() time.Sleep(3 * time.Second) } ================================================ FILE: video_server/scheduler/taskrunner/tasks.go ================================================ package taskrunner import ( "errors" "golang-streaming/video_server/scheduler/dbops" "log" "os" "sync" ) func VideoClearDispatcher(dc dataChan) error { res, err := dbops.ReadVideoDeletionRecord(3) // 简单起见读3条 if err != nil { log.Printf("Video clear dispatcher error: %v", err) return err } if len(res) == 0 { return errors.New("All tasks finished") } for _, id := range res { dc <- id } return nil } func deleteVideo(vid string) error { err := os.Remove(VIDEO_PATH + vid) if err != nil && !os.IsNotExist(err) { log.Printf("Deleting video error: %v", err) return err } return nil } func VideoClearExecutor(dc dataChan) error { errMap := &sync.Map{} var err error forloop: for { select { case vid := <- dc: go func(id interface{}) { if err := deleteVideo(id.(string)); err != nil { errMap.Store(id, err) return } if err := dbops.DelVideoDeletionRecord(id.(string)); err != nil { errMap.Store(id, err) return } }(vid) default: break forloop } } errMap.Range(func(k, v interface{}) bool { err = v.(error) if err != nil { return false } return true }) return err } ================================================ FILE: video_server/scheduler/taskrunner/trmain.go ================================================ package taskrunner import "time" type Worker struct { ticker *time.Ticker runner *Runner } func NewWorker(interval time.Duration, r *Runner) *Worker { return &Worker { ticker : time.NewTicker(interval * time.Second), runner: r, } } func (w *Worker) startWorker() { for { select { case <- w.ticker.C: go w.runner.StartAll() } } } func Start() { // Start video file cleaning r := NewRunner(3,true, VideoClearDispatcher, VideoClearExecutor) w := NewWorker(3, r) go w.startWorker() } ================================================ FILE: video_server/streamserver/defs.go ================================================ package main const ( VIDEO_DIR = "./videos/" MAX_UPLOAD_SIZE = 50 * 1024 * 1024 // 50MB ) ================================================ FILE: video_server/streamserver/handlers.go ================================================ package main import ( "github.com/julienschmidt/httprouter" "html/template" "io" "io/ioutil" "log" "net/http" "os" "time" ) func testPageHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { t, _ := template.ParseFiles("./videos/upload.html") t.Execute(w, nil) } func streamHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { vid := p.ByName("vid-id") vl := VIDEO_DIR + vid video, err := os.Open(vl) if err != nil { log.Printf("Error when try to open file: %v", err) sendErrorResponse(w, http.StatusInternalServerError, "Internal Error") return } w.Header().Set("Content-Type", "video/mp4") http.ServeContent(w, r, "", time.Now(), video) defer video.Close() } func uploadHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) { r.Body = http.MaxBytesReader(w, r.Body, MAX_UPLOAD_SIZE) if err := r.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil { sendErrorResponse(w, http.StatusBadRequest, "File is too big") return } file, _, err := r.FormFile("file") if err != nil { sendErrorResponse(w, http.StatusInternalServerError, "Internal Error") return } data, err := ioutil.ReadAll(file) if err != nil { log.Printf("Read file error: %v", err) sendErrorResponse(w, http.StatusInternalServerError, "Internal Error") } fn := p.ByName("vid-id") err = ioutil.WriteFile(VIDEO_DIR + fn, data, 0666 ) if err != nil { log.Printf("Write file error: %v", err) sendErrorResponse(w, http.StatusInternalServerError, "Internal Error") return } w.WriteHeader(http.StatusCreated) io.WriteString(w, "Upload successfully") } ================================================ FILE: video_server/streamserver/limiter.go ================================================ package main import "log" type ConnLimiter struct { concurrentConn int bucket chan int } func NewConnLimiter(cc int) *ConnLimiter { return &ConnLimiter{ concurrentConn: cc, bucket: make(chan int, cc), } } func (cl *ConnLimiter) GetConn() bool { if len(cl.bucket) >= cl.concurrentConn { log.Printf("Reached the rate limitation.") return false } cl.bucket <- 1 return true } func (cl *ConnLimiter) ReleaseConn() { c := <- cl.bucket log.Printf("New connection coming: %d", c) } ================================================ FILE: video_server/streamserver/main.go ================================================ package main import ( "github.com/julienschmidt/httprouter" "net/http" ) type middleWareHandler struct { r *httprouter.Router l *ConnLimiter } func NewMiddleWareHandler(r *httprouter.Router, cc int) http.Handler { m := middleWareHandler{} m.r = r m.l = NewConnLimiter(cc) return m } func (m middleWareHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !m.l.GetConn() { sendErrorResponse(w, http.StatusTooManyRequests, "Too many requests") return } m.r.ServeHTTP(w, r) defer m.l.ReleaseConn() } func RegisterHandlers() *httprouter.Router { router := httprouter.New() router.GET("/videos/:vid-id", streamHandler) router.POST("/upload/:vid-id", uploadHandler) router.GET("/testpage", testPageHandler) return router } func main() { r := RegisterHandlers() mh := NewMiddleWareHandler(r, 100) // 流控值暂置为2便于测试 http.ListenAndServe(":9000", mh) } ================================================ FILE: video_server/streamserver/response.go ================================================ package main import ( "io" "net/http" ) func sendErrorResponse(w http.ResponseWriter, sc int, errMsg string) { w.WriteHeader(sc) io.WriteString(w, errMsg) } ================================================ FILE: video_server/streamserver/upload.html ================================================ Test Upload a File
{{/* 1. File input */}} {{/* 2. Submit button */}}
================================================ FILE: video_server/templates/home.html ================================================ Title
Home {{.Name}} About Upload Logout

Welcome to MOVIEDASH

Register

By creating an account you agree to our Terms & Privacy.

Already have an account? Sign in.

================================================ FILE: video_server/templates/scripts/home.js ================================================ $(document).ready(function() { DEFAULT_COOKIE_EXPIRE_TIME = 30; uname = ''; session = ''; uid = 0; currentVideo = null; listedVideos = null; session = getCookie('session'); uname = getCookie('username'); initPage(function() { if (listedVideos !== null) { currentVideo = listedVideos[0]; selectVideo(listedVideos[0]['id']); } $(".video-item").click(function() { var self = this.id listedVideos.forEach(function(item, index) { if (item['id'] === self) { currentVideo = item; return } }); selectVideo(self); }); $(".del-video-button").click(function() { var id = this.id.substring(4); deleteVideo(id, function(res, err) { if (err !== null) { //window.alert("encounter an error when try to delete video: " + id); popupErrorMsg("encounter an error when try to delete video: " + id); return; } popupNotificationMsg("Successfully deleted video: " + id) location.reload(); }); }); $("#submit-comment").on('click', function() { var content = $("#comments-input").val(); postComment(currentVideo['id'], content, function(res, err) { if (err !== null) { popupErrorMsg("encounter and error when try to post a comment: " + content); return; } if (res === "ok") { popupNotificationMsg("New comment posted") $("#comments-input").val(""); refreshComments(currentVideo['id']); } }); }); }); // home page event registry $("#regbtn").on('click', function(e) { $("#regbtn").text('Loading...') e.preventDefault() registerUser(function(res, err) { if (err != null) { $('#regbtn').text("Register") popupErrorMsg('encounter an error, pls check your username or pwd'); return; } var obj = JSON.parse(res); setCookie("session", obj["session_id"], DEFAULT_COOKIE_EXPIRE_TIME); setCookie("username", uname, DEFAULT_COOKIE_EXPIRE_TIME); $("#regsubmit").submit(); }); }); $("#signinbtn").on('click', function(e) { $("#signinbtn").text('Loading...') e.preventDefault(); signinUser(function(res, err) { if (err != null) { $('#signinbtn').text("Sign In"); //window.alert('encounter an error, pls check your username or pwd') popupErrorMsg('encounter an error, pls check your username or pwd'); return; } var obj = JSON.parse(res); setCookie("session", obj["session_id"], DEFAULT_COOKIE_EXPIRE_TIME); setCookie("username", uname, DEFAULT_COOKIE_EXPIRE_TIME); $("#signinsubmit").submit(); }); }); $("#signinhref").on('click', function() { $("#regsubmit").hide(); $("#signinsubmit").show(); }); $("#registerhref").on('click', function() { $("#regsubmit").show(); $("#signinsubmit").hide(); }); // userhome event register $("#upload").on('click', function() { $("#uploadvideomodal").show(); }); $("#uploadform").on('submit', function(e) { e.preventDefault() var vname = $('#vname').val(); createVideo(vname, function(res, err) { if (err != null ) { //window.alert('encounter an error when try to create video'); popupErrorMsg('encounter an error when try to create video'); return; } var obj = JSON.parse(res); var formData = new FormData(); formData.append('file', $('#inputFile')[0].files[0]); $.ajax({ url : 'http://' + window.location.hostname + ':8080/upload/' + obj['id'], type : 'POST', data : formData, //headers: {'Access-Control-Allow-Origin': 'http://127.0.0.1:9000'}, crossDomain: true, processData: false, // tell jQuery not to process the data contentType: false, // tell jQuery not to set contentType success : function(data) { console.log(data); $('#uploadvideomodal').hide(); location.reload(); //window.alert("hoa"); }, complete: function(xhr, textStatus) { if (xhr.status === 204) { window.alert("finish") return; } if (xhr.status === 400) { $("#uploadvideomodal").hide(); popupErrorMsg('file is too big'); return; } } }); }); }); $(".close").on('click', function() { $("#uploadvideomodal").hide(); }); $("#logout").on('click', function() { setCookie("session", "", -1) setCookie("username", "", -1) }); $(".video-item").click(function () { var url = 'http://' + window.location.hostname + ':9000/videos/'+ this.id var video = $("#curr-video"); video[0].attr('src', url); video.load(); }); }); function initPage(callback) { getUserId(function(res, err) { if (err != null) { window.alert("Encountered error when loading user id"); return; } var obj = JSON.parse(res); uid = obj['id']; //window.alert(obj['id']); listAllVideos(function(res, err) { if (err != null) { //window.alert('encounter an error, pls check your username or pwd'); popupErrorMsg('encounter an error, pls check your username or pwd'); return; } var obj = JSON.parse(res); listedVideos = obj['videos']; obj['videos'].forEach(function(item, index) { var ele = htmlVideoListElement(item['id'], item['name'], item['display_ctime']); $("#items").append(ele); }); callback(); }); }); } function setCookie(cname, cvalue, exmin) { var d = new Date(); d.setTime(d.getTime() + (exmin * 60 * 1000)); var expires = "expires="+d.toUTCString(); document.cookie = cname + "=" + cvalue + ";" + expires + ";path=/"; } function getCookie(cname) { var name = cname + "="; var ca = document.cookie.split(';'); for(var i = 0; i < ca.length; i++) { var c = ca[i]; while (c.charAt(0) == ' ') { c = c.substring(1); } if (c.indexOf(name) == 0) { return c.substring(name.length, c.length); } } return ""; } // DOM operations function selectVideo(vid) { var url = 'http://' + window.location.hostname + ':8080/videos/'+ vid var video = $("#curr-video"); $("#curr-video:first-child").attr('src', url); $("#curr-video-name").text(currentVideo['name']); $("#curr-video-ctime").text('Uploaded at: ' + currentVideo['display_ctime']); //currentVideoId = vid; refreshComments(vid); } function refreshComments(vid) { listAllComments(vid, function (res, err) { if (err !== null) { //window.alert("encounter an error when loading comments"); popupErrorMsg('encounter an error when loading comments'); return } var obj = JSON.parse(res); $("#comments-history").empty(); if (obj['comments'] === null) { $("#comments-total").text('0 Comments'); } else { $("#comments-total").text(obj['comments'].length + ' Comments'); } obj['comments'].forEach(function(item, index) { var ele = htmlCommentListElement(item['id'], item['author'], item['content']); $("#comments-history").append(ele); }); }); } function popupNotificationMsg(msg) { var x = document.getElementById("snackbar"); $("#snackbar").text(msg); x.className = "show"; setTimeout(function(){ x.className = x.className.replace("show", ""); }, 2000); } function popupErrorMsg(msg) { var x = document.getElementById("errorbar"); $("#errorbar").text(msg); x.className = "show"; setTimeout(function(){ x.className = x.className.replace("show", ""); }, 2000); } function htmlCommentListElement(cid, author, content) { var ele = $('
', { id: cid }); ele.append( $('
', { class: 'comment-author', text: author + ' says:' }) ); ele.append( $('
', { class: 'comment', text: content }) ); ele.append('
'); return ele; } function htmlVideoListElement(vid, name, ctime) { var ele = $('', { href: '#' }); ele.append( $('
', { text: name }) ); ele.append( $('
', { text: ctime }) ); var res = $('
', { id: vid, class: 'video-item' }).append(ele); res.append( $('