Repository: himananiito/livedl Branch: master Commit: a8720f1e358e Files: 50 Total size: 258.1 KB Directory structure: gitextract_0ls_u7mq/ ├── .gitignore ├── Dockerfile ├── LICENSE ├── Readme.md ├── build/ │ └── windows/ │ ├── Dockerfile │ └── docker-compose.yml ├── build-386.ps1 ├── build.ps1 ├── changelog.txt ├── livedl-logger.go ├── readme-gen.pl ├── replacelocal.pl ├── src/ │ ├── amf/ │ │ ├── amf.go │ │ ├── amf0/ │ │ │ └── amf0.go │ │ ├── amf3/ │ │ │ └── amf3.go │ │ └── amf_t/ │ │ └── amf_t.go │ ├── buildno/ │ │ ├── buildno.go │ │ └── funcs.go │ ├── cryptoconf/ │ │ └── cryptoconf.go │ ├── defines/ │ │ └── constant.go │ ├── files/ │ │ └── files.go │ ├── flvs/ │ │ └── flv.go │ ├── go.mod │ ├── go.sum │ ├── gorman/ │ │ └── gorman.go │ ├── httpbase/ │ │ └── httpbase.go │ ├── httpsub/ │ │ └── httpsub.go │ ├── livedl.go │ ├── log4gui/ │ │ └── log4gui.go │ ├── niconico/ │ │ ├── jikken.gox │ │ ├── nico.go │ │ ├── nico_db.go │ │ ├── nico_hls.go │ │ ├── nico_mem_db.go │ │ └── nico_rtmp.go │ ├── objs/ │ │ └── objs.go │ ├── options/ │ │ └── options.go │ ├── procs/ │ │ ├── base/ │ │ │ └── base.go │ │ ├── ffmpeg/ │ │ │ └── ffmpeg.go │ │ ├── kill.go │ │ ├── streamlink/ │ │ │ └── streamlink.go │ │ └── youtube_dl/ │ │ └── youtube-dl.go │ ├── rtmps/ │ │ ├── message.go │ │ └── rtmp.go │ ├── twitcas/ │ │ └── twicas.go │ ├── youtube/ │ │ ├── comment.go │ │ ├── youtube.go │ │ └── youtube.gox │ └── zip2mp4/ │ └── zip2mp4.go └── updatebuildno.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ /testrec/ *.flv *.mp4 *.ts *.mkv *.part *.mpg *.webm *~ *.exe *.dll *.exe.config *.xml *.m3u8 *.bin *.zip *.txt *.conf *.db *.pem *.ass *.sqlite *.sqlite3 *-journal *.sqlite3-shm *.sqlite3-wal Nico/* !changelog.txt *.orig lv* ================================================ FILE: Dockerfile ================================================ FROM golang:1.16-alpine as builder RUN apk add --no-cache \ build-base \ git COPY . /tmp/livedl RUN cd /tmp/livedl/src && \ go build livedl.go FROM alpine:3.8 RUN apk add --no-cache \ ca-certificates \ ffmpeg \ openssl COPY --from=builder /tmp/livedl/src/livedl /usr/local/bin/ WORKDIR /livedl VOLUME /livedl ENTRYPOINT [ "livedl", "--no-chdir" ] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2018 himananiito Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Readme.md ================================================ # livedl 新配信(HTML5)に対応したニコ生録画ツール。ニコ生以外のサイトにも対応予定 ## 使い方 https://himananiito.hatenablog.jp/entry/livedl を参照 ## Windowsでのビルド(exeを作成するためにDockerを利用) ### Step1 `docker-compose` が実行できるようにDocker Desktop for Windowsをインストールする。 ### Step2 ターミナルで ``` build\windows ``` に移動する。 ### Step3 ``` docker-compose up --build ``` を実行するとプロジェクトのトップディレクトリに `livedl.exe` が作成される。 ## Linux(Ubuntu)でのビルド方法 ``` cat /etc/os-release NAME="Ubuntu" VERSION="16.04.2 LTS (Xenial Xerus)" ``` ### Go実行環境のインストール　（無い場合） ``` https://golang.org/doc/install に従う ``` ### gitをインストール　（無い場合） ``` sudo apt-get install git ``` ### gccなどのビルドツールをインストール　（無い場合） ``` sudo apt-get install build-essential ``` ### livedlのソースを取得 ``` git clone https://github.com/himananiito/livedl.git ``` ### livedlのコンパイル ディレクトリを移動 ``` cd livedl ``` #### (オプション)最新のコードをビルドする場合 ``` git checkout master ``` ビルドする ``` go build src/livedl.go ``` ``` ./livedl -h livedl (20180807.22-linux) ``` ## Windows(32bit及び64bit上での32bit向け)コンパイル方法 ### gccのインストール gcc には必ず以下を使用すること。 http://tdm-gcc.tdragon.net/download 環境変数で（例）`C:\TDM-GCC-64\bin`が他のgccより優先されるように設定すること。 ### 必要なgoのモジュール linuxの説明に倣ってインストールする。 ### コンパイル PowerSellで、`build-386.ps1` を実行する。または以下を実行する。 ``` set-item env:GOARCH -value 386 set-item env:CGO_ENABLED -value 1 go build -o livedl.x86.exe src/livedl.go ``` ### 32bit環境で`x509: certificate signed by unknown authority`が出る 動けばいいのであればオプションで以下を指定する。 `-http-skip-verify=on` ## コンテナで実行 ### livedlのソースを取得 ``` git clone https://github.com/himananiito/livedl.git cd livedl git checkout master # Or another version that supports docker (contains Dockerfile) ``` ### イメージ作成 ``` docker build -t livedl . ``` ### イメージの使い方 - 出力フォルダを/livedlにマウント ``` docker run --rm -it -v "$(pwd):/livedl" livedl "https://live.nicovideo.jp/watch/..." ``` 以上 ================================================ FILE: build/windows/Dockerfile ================================================ FROM golang:1.16-alpine RUN apk add mingw-w64-gcc COPY ./src/ /livedl/src/ RUN \ cd /livedl/src/ && \ GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -o livedl.exe livedl.go CMD cp /livedl/src/livedl.exe /mnt/ ================================================ FILE: build/windows/docker-compose.yml ================================================ version: '3' services: livedl-win: build: context: ../.. dockerfile: ./build/windows/Dockerfile volumes: - ../..:/mnt ================================================ FILE: build-386.ps1 ================================================ set-item env:GOARCH -value 386 set-item env:CGO_ENABLED -value 1 go build -o livedl.x86.exe src/livedl.go ================================================ FILE: build.ps1 ================================================ rm livedl.exe go run updatebuildno.go go build src/livedl.go .\build-386.ps1 go build livedl-logger.go # hide local path perl replacelocal.pl # Generate Readme.txt perl readme-gen.pl # livedl test run(nico) $process = Start-Process -FilePath livedl.exe -ArgumentList '-nicotestrun -nicotesttimeout 7 -nicotestfmt "testrec/?UNAME?/?PID?-?UNAME?-?TITLE?"' -PassThru $process.WaitForExit(1000 * 61) $process.Kill() $process = Start-Process -FilePath livedl.x86.exe -ArgumentList '-nicotestrun -nicotesttimeout 7 -nicotestfmt "testrec/?UNAME?/?PID?-?UNAME?-?TITLE?"' -PassThru $process.WaitForExit(1000 * 30) $process.Kill() $dir = "livedl" $zip = "$dir.zip" if(Test-Path -PathType Leaf $zip) { rm $zip } if(Test-Path -PathType Container $dir) { rmdir -Recurse $dir } mkdir $dir cp livedl.exe $dir cp livedl.x86.exe $dir cp livedl-logger.exe $dir cp Readme.txt $dir cp livedl-gui.exe $dir cp livedl-gui.exe.config $dir cp Newtonsoft.Json.dll $dir cp Newtonsoft.Json.xml $dir Compress-Archive -Path $dir -DestinationPath $zip if(Test-Path -PathType Container $dir) { rmdir -Recurse $dir } ================================================ FILE: changelog.txt ================================================ 更新履歴 20181215.35 ・-nico-ts-start-minオプションの追加 ・win32bit版のビルドを追加 ・-http-skip-verifyオプションを保存できるようにした ・ライセンスをMITにした 20181107.34 ・[ニコ生] (暫定)TEMPORARILY_CROWDEDで録画終了するようにした ・ファイル名が半角ドットで終わる場合に全角ドットにした ・[YouTubeLive] コメントの改行をCRLFにした ・[ニコ生TS] タイムシフトの録画を指定した再生時間(秒)から開始するオプション追加(merged) ・[ニコ生TS] 32bitで終了しない問題を修正(merged) 20181008.33 ・[Youtube] チャットが取得できない問題を修正 ・[Youtube] Streamlinkでダウンロードできない場合にyoutube-dlを使うようにした ・[Youtube] コメントファイルを書き出せるようにした。 ・#15 [ニコ生コメント] 出力をCRLFにした。/hbコマンドを出さないオプションを追加 20181003.32 ・#14 ★緊急 [ニコ生] 新配信録画のプレイリスト取得にウェイトが入らない問題を修正 ・#9 [ニコ生TS] プレイリストの最後で無限ループしてしまう問題を修正 ・YoutubeLiveコメント対応中(未完了) ・[実験的] -yt-api-key オプションの追加（未使用） 20180925.31 ・#8 [ツイキャス] 「c:」から始まるユーザ名が録画できない問題を修正 ・#11 [ツイキャス] 実行直後またはリトライ中にエラーで終了する問題を修正 ・#10 [ツイキャス] -tcas-retry-intervalが効かない問題を修正 ・#12 [ニコ生] タイムシフトで先頭のセグメント(seqno=0)が取得できない問題を修正 ================================================ FILE: livedl-logger.go ================================================ package main import ( "fmt" "bufio" "regexp" "sync" "os" "os/exec" ) func main() { args := os.Args[1:] var vid string for _, s := range args { if ma := regexp.MustCompile(`(lv\d{9,})`).FindStringSubmatch(s); len(ma) > 0 { vid = ma[1] } } args = append(args, "-nicoDebug") cmd := exec.Command("livedl", args...) if vid == "" { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Run() } else { stdout, err := cmd.StdoutPipe() if err != nil { fmt.Println(err) return } stderr, err := cmd.StderrPipe() if err != nil { fmt.Println(err) return } name := fmt.Sprintf("log/%s.txt", vid) os.MkdirAll("log", os.ModePerm) f, err := os.Create(name) if err != nil { fmt.Println(err) return } defer f.Close() var mtx sync.Mutex append := func(s string) { mtx.Lock() defer mtx.Unlock() f.WriteString(s) } go func() { rdr := bufio.NewReader(stdout) for { s, err := rdr.ReadString('\n') if err != nil { return } fmt.Print(s) append(s) } defer stdout.Close() }() go func() { rdr := bufio.NewReader(stderr) for { s, err := rdr.ReadString('\n') if err != nil { return } append(s) } defer stderr.Close() }() cmd.Run() } } ================================================ FILE: readme-gen.pl ================================================ use strict; use warnings; use v5.20; open my $f, "-|", "livedl", "-h" or die; undef $/; my $s = <$f>; close $f; $s =~ s{livedl\s*\((\d+\.\d+)[^\r\n]*}{livedl ($1)} or die; my $ver = $1; $s =~ s{chdir:[^\n]*\n}{}; open my $g, "changelog.txt" or die; my $t = <$g>; close $g; $t =~ s{\$latest}{$ver} or die; open my $h, ">", "changelog.txt" or die; print $h $t; close $h; open my $o, ">", "Readme.txt" or die; say $o $s; say $o ""; say $o $t; close $o; ================================================ FILE: replacelocal.pl ================================================ # perl # livedl.exe内のローカルパスの文字列を隠す use strict; use v5.20; for my $file("livedl.exe", "livedl.x86.exe", "livedl-logger.exe") { open my $f, "<:raw", $file or die; undef $/; my $s = <$f>; close $f; say "$0: $file"; my %h = (); while($s =~ m{(?<=\0)[^\0]{5,512}\.go(?=\0)|(?<=[[:cntrl:]])_/[A-Z]_/[^\0]{5,512}}g) { my $s = $&; if($s =~ m{\A(.*(?:/Users/.+?/go/src|/Go/src))(/.*)\z}s or $s =~ m{\A(.*(?=/livedl/src/))(/.*)\z}s) { my($all, $p, $f) = ($s, $1, $2); my $p2 = $p; $p2 =~ s{.}{*}gs; #$h{$all} = $p2 . $f; #say $p; $h{$p} = $p2; } } for my $k (sort{$a cmp $b} keys %h) { my $k2 = $k; $k2 =~ s{/}{\\}g; my $r = quotemeta $k; my $r2 = quotemeta $k2; say "$k => $h{$k}"; $s =~ s{$r}{$h{$k}}g; $s =~ s{$r2}{$h{$k}}g; } open $f, ">:raw", $file or die; print $f $s; close $f; sleep 1; } ================================================ FILE: src/amf/amf.go ================================================ package amf import ( "bytes" "io" "github.com/himananiito/livedl/amf/amf0" "github.com/himananiito/livedl/amf/amf_t" ) func SwitchToAmf3() amf_t.SwitchToAmf3 { return amf_t.SwitchToAmf3{} } func EncodeAmf0(data []interface{}, asEcmaArray bool) ([]byte, error) { return amf0.Encode(data, asEcmaArray) } func Amf0EcmaArray(data map[string]interface{}) amf_t.AMF0EcmaArray { return amf_t.AMF0EcmaArray{ Data: data, } } // paddingHint: zero padded before AMF data func DecodeAmf0(data []byte, paddingHint ...bool) (res []interface{}, err error) { rdr := bytes.NewReader(data) var seek1 bool for _, h := range paddingHint { if h { seek1 = true break } } if seek1 { rdr.Seek(1, io.SeekStart) } res, err = amf0.DecodeAll(rdr) if err != nil { if seek1 { // retry rdr.Seek(0, io.SeekStart) res, err = amf0.DecodeAll(rdr) } } return } ================================================ FILE: src/amf/amf0/amf0.go ================================================ package amf0 import ( "bytes" "encoding/binary" "fmt" "io" "log" "math" "github.com/himananiito/livedl/amf/amf3" "github.com/himananiito/livedl/amf/amf_t" ) func encodeNumber(num float64, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(0); err != nil { return } bits := math.Float64bits(num) bytes := make([]byte, 8) binary.BigEndian.PutUint64(bytes, bits) if _, err = buff.Write(bytes); err != nil { return } return } func encodeBoolean(b bool, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(1); err != nil { return } var val byte if b { val = 1 } if err = buff.WriteByte(val); err != nil { return } return } func encodeUtf8(s string, buff *bytes.Buffer) (err error) { bs := []byte(s) if len(bs) > 0xffff { err = fmt.Errorf("string too large") return } b0 := make([]byte, 2) binary.BigEndian.PutUint16(b0, uint16(len(bs))) if _, err = buff.Write(b0); err != nil { return } if _, err = buff.Write(bs); err != nil { return } return } func encodeString(s string, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(2); err != nil { return } err = encodeUtf8(s, buff) return } func encodeObject(obj map[string]interface{}, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(3); err != nil { return } for k, v := range obj { if err = encodeUtf8(k, buff); err != nil { return } if _, err = encode(v, false, buff); err != nil { return } } if _, err = buff.Write([]byte{0, 0, 9}); err != nil { return } return } func encodeNull(buff *bytes.Buffer) error { return buff.WriteByte(5) } func encodeSwitchToAmf3(buff *bytes.Buffer) error { return buff.WriteByte(0x11) } func encodeEcmaArray(data map[string]interface{}, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(8); err != nil { return } buf4 := make([]byte, 4) binary.BigEndian.PutUint32(buf4, uint32(len(data))) if _, err = buff.Write(buf4); err != nil { return } for k, v := range data { if err = encodeUtf8(k, buff); err != nil { return } if _, err = encode(v, true, buff); err != nil { return } } if _, err = buff.Write([]byte{0, 0, 9}); err != nil { return } return } func encode(data interface{}, asEcmaArray bool, buff *bytes.Buffer) (toAmf3 bool, err error) { switch data.(type) { case string: err = encodeString(data.(string), buff) case float64: err = encodeNumber(data.(float64), buff) case int: err = encodeNumber(float64(data.(int)), buff) case bool: err = encodeBoolean(data.(bool), buff) case map[string]interface{}: if asEcmaArray { err = encodeEcmaArray(data.(map[string]interface{}), buff) } else { err = encodeObject(data.(map[string]interface{}), buff) } case []interface{}: m := make(map[string]interface{}) for i, d := range data.([]interface{}) { k := fmt.Sprintf("%d", i) m[k] = d } err = encodeEcmaArray(m, buff) case nil: err = encodeNull(buff) case amf_t.SwitchToAmf3: toAmf3 = true err = encodeSwitchToAmf3(buff) default: log.Fatalf("amf0/encode %#v", data) } return } func Encode(data []interface{}, asEcmaArray bool) (b []byte, err error) { buff := bytes.NewBuffer(nil) for i, d := range data { var toAmf3 bool if toAmf3, err = encode(d, asEcmaArray, buff); err != nil { return } if toAmf3 { b2, e := amf3.Encode(data[i+1:]) if e != nil { err = e return } b = append(b, buff.Bytes()...) b = append(b, b2...) return } } b = buff.Bytes() return } type objectEnd struct{} func decodeString(rdr *bytes.Reader) (str string, err error) { buf := make([]byte, 2) if _, err = io.ReadFull(rdr, buf); err != nil { return } len := (int(buf[0]) << 8) | int(buf[1]) if len > 0 { buf := make([]byte, len) if _, err = io.ReadFull(rdr, buf); err != nil { return } str = string(buf) } return } func decodeNumber(rdr *bytes.Reader) (res float64, err error) { buf := make([]byte, 8) if _, err = io.ReadFull(rdr, buf); err != nil { return } u64 := binary.BigEndian.Uint64(buf) res = math.Float64frombits(u64) return } func decodeBoolean(rdr *bytes.Reader) (res bool, err error) { buf := make([]byte, 1) if _, err = io.ReadFull(rdr, buf); err != nil { return } if buf[0] == 0 { res = false } else { res = true } return } func decodeObject(rdr *bytes.Reader) (res map[string]interface{}, err error) { res = make(map[string]interface{}) for { key, e := decodeString(rdr) if e != nil { err = e return } val, e := decodeOne(rdr) if e != nil { err = e return } if key == "" { switch val.(type) { case objectEnd: return default: log.Fatalf("decodeObject: parse error; Not object-end, %+s", val) } } res[key] = val } return } func decodeEcmaArray(rdr *bytes.Reader) (res map[string]interface{}, err error) { buf := make([]byte, 4) if _, err = io.ReadFull(rdr, buf); err != nil { return } //count := binary.BigEndian.Uint32(buf) //log.Printf("decodeEcmaArray: Count: %v", count) res, err = decodeObject(rdr) return } func decodeStrictArray(rdr *bytes.Reader) (res []interface{}, err error) { buf := make([]byte, 4) if _, err = io.ReadFull(rdr, buf); err != nil { return } count := binary.BigEndian.Uint32(buf) for i := uint32(0); i < count; i++ { re, e := decodeOne(rdr) if e != nil { err = e return } res = append(res, re) } return } func decodeOne(rdr *bytes.Reader) (res interface{}, err error) { buf := make([]byte, 1) if _, err = io.ReadFull(rdr, buf); err != nil { return } switch buf[0] { case 0: // Number res, err = decodeNumber(rdr) case 1: // Boolean res, err = decodeBoolean(rdr) case 2: // String res, err = decodeString(rdr) case 3: res, err = decodeObject(rdr) case 5: // Null res = nil case 6: // undefined res = nil case 8: // ECMA Array res, err = decodeEcmaArray(rdr) case 9: // Object End res = objectEnd{} case 10: res, err = decodeStrictArray(rdr) case 0x11: // Switch to AMF3 dat, e := amf3.DecodeAll(rdr) if e != nil { err = e return } res = amf_t.AMF3{Data: dat} default: err = fmt.Errorf("Not implemented: type=%d", buf[0]) } return } func DecodeAll(rdr *bytes.Reader) (res []interface{}, err error) { for rdr.Len() > 0 { re, e := decodeOne(rdr) if e != nil { err = e return } switch re.(type) { case amf_t.AMF3: res = append(res, re.(amf_t.AMF3).Data...) default: res = append(res, re) } } return } ================================================ FILE: src/amf/amf3/amf3.go ================================================ package amf3 import ( "bytes" "io" "log" "fmt" ) func decodeU29(rdr *bytes.Reader) (res int, err error) { for i := 0; i < 4; i++ { var num byte if num, err = rdr.ReadByte(); err != nil { return } var flg bool var val uint8 if i == 3 { val = num } else { flg = (num & 0x80) == 0x80 val = (num & 0x7f) } switch i { case 0: res = int(val) case 3: res = (res << 8) | int(val) default: res = (res << 7) | int(val) } if (! flg) { break } } return } // UTF-8-vr func decodeString(rdr *bytes.Reader) (str string, err error) { // UTF-8-vr = U29S-ref // UTF-8-vr = U29S-value *(UTF8-char) u29, err := decodeU29(rdr) if err != nil { return } flag := (u29 & 1) != 0 len := u29 >> 1 if (! flag) { // string reference table index log.Fatalf("[FIXME] not implemented: UTF-8-vr = U29S-ref") } else { buf := make([]byte, len) if _, err = io.ReadFull(rdr, buf); err != nil { return } str = string(buf) } return } func assocOrUtf8Empty(rdr *bytes.Reader) (key string, val interface{}, err error) { key, err = decodeString(rdr) if err != nil { return } if key == "" { //fmt.Printf("assocOrUtf8Empty: string is empty\n") return } val, err = decodeOne(rdr) if err != nil { return } //log.Fatalf("assocOrUtf8Empty: key=%v, val=%v", key, val) return } func decodeOne(rdr *bytes.Reader) (res interface{}, err error) { format, err := rdr.ReadByte() if err != nil { return } switch format { case 6: // string-marker res, err = decodeString(rdr) if err != nil { return } case 9: // array-marker // array-marker U29O-ref // # array-marker U29A-value (UTF-8-empty | * (assoc-value) UTF-8-empty) * (value-type) // array-marker U29A-value * (assoc-value) UTF-8-empty * (value-type) // array-marker U29A-value UTF-8-empty * (value-type) u29, _ := decodeU29(rdr) flag := u29 & 1 != 0 count := u29 >> 1 if (! flag) { log.Fatalf("[FIXME] not implemented: array-type = array-marker U29O-ref") } if count == 0 { // [FIXME] condition OK? // associative, terminated by empty string assoc := make(map[string]interface{}) for { k, v, e := assocOrUtf8Empty(rdr) if e != nil { //fmt.Printf("## amf3 associative: %+v\n", e) err = e return } if k == "" { break } assoc[k] = v //log.Printf("AMF3 array: %v = %v", k, v) } res = assoc } //log.Fatalf("AMF3 array: len: %d", count) default: log.Printf("%v\n", res) log.Fatalf("Not implemented: %d", format) } return } func DecodeAll(rdr *bytes.Reader) (res []interface{}, err error) { for rdr.Len() > 0 { re, e := decodeOne(rdr) if e != nil { err = e return } res = append(res, re) } return } func encodeU29(num int, buff *bytes.Buffer) (err error) { if (0 <= num && num <= 0x7f) { if err = buff.WriteByte( byte(num & 0x7f) ); err != nil { return } } else if (0x80 <= num && num <= 0x3fff) { if err = buff.WriteByte( byte(0x80 | ((num >> 7) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(num & 0x7f) ); err != nil { return } } else if (0x4000 <= num && num <= 0x1fffff) { if err = buff.WriteByte( byte(0x80 | ((num >> 14) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(0x80 | ((num >> 7) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(num & 0x7f) ); err != nil { return } } else if (0x200000 <= num && num <= 0x3fffffff) { if err = buff.WriteByte( byte(0x80 | ((num >> 22) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(0x80 | ((num >> 15) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(0x80 | ((num >> 7) & 0x7f)) ); err != nil { return } if err = buff.WriteByte( byte(num & 0xff) ); err != nil { return } } else { err = fmt.Errorf("u29 overflow") } return } func encodeU28Flag(num int, flag bool, buff *bytes.Buffer) (err error) { if flag { err = encodeU29(((num << 1) | 1), buff) } else { err = encodeU29((num << 1), buff) } return } func encodeArray(data []interface {}, buff *bytes.Buffer) (err error) { // array-marker if err = buff.WriteByte(9); err != nil { return } // U29A-value; count of the dense portin of the Array if err = encodeU28Flag(len(data), true, buff); err != nil { return } // UTF-8-empty if err = buff.WriteByte(1); err != nil { return } for _, v := range data { if err = encode(v, buff); err != nil { return } } return } func encodeStringArray(data []string, buff *bytes.Buffer) error { var list []interface{} for _, v := range data { list = append(list, v) } return encodeArray(list, buff) } func encodeString(data string, buff *bytes.Buffer) (err error) { if err = buff.WriteByte(6); err != nil { return } bstr := []byte(data) // U29S-value if err = encodeU28Flag(len(bstr), true, buff); err != nil { return } if _, err = buff.Write(bstr); err != nil { return } return } func encode(data interface{}, buff *bytes.Buffer) (err error) { switch data.(type) { case string: err = encodeString(data.(string), buff) case []string: err = encodeStringArray(data.([]string), buff) default: log.Fatalf("amf0/encode %#v", data) } return } func Encode(data []interface{}) (b []byte, err error) { buff := bytes.NewBuffer(nil) for _, data := range data { if err = encode(data, buff); err != nil { return } } b = buff.Bytes() return } ================================================ FILE: src/amf/amf_t/amf_t.go ================================================ package amf_t type AMF3 struct { Data []interface{} } type SwitchToAmf3 struct { } type AMF0EcmaArray struct { Data map[string]interface {} } ================================================ FILE: src/buildno/buildno.go ================================================ package buildno var BuildDate = "20181215" var BuildNo = "35" ================================================ FILE: src/buildno/funcs.go ================================================ package buildno import ( "fmt" "runtime" ) func GetBuildNo() string { return fmt.Sprintf( "%v.%v-%s-%s", BuildDate, BuildNo, runtime.GOOS, runtime.GOARCH, ) } ================================================ FILE: src/cryptoconf/cryptoconf.go ================================================ package cryptoconf import ( "golang.org/x/crypto/sha3" "crypto/aes" "crypto/cipher" "crypto/rand" "io" "io/ioutil" "os" "encoding/json" "fmt" "log" ) func Set(dataSet map[string]string, fileName, pass string) (err error) { var data map[string]interface{} if _, test := os.Stat(fileName); test == nil { data, err = Load(fileName, pass) if err != nil { return } } else { data = map[string]interface{}{} } for key, val := range dataSet { data[key] = val } digest := sha3.Sum256([]byte(pass)) block, err := aes.NewCipher(digest[:]) if err != nil { log.Fatalln(err) } aesgcm, err := cipher.NewGCM(block) if err != nil { log.Fatalln(err.Error()) } nonceSize := aesgcm.NonceSize() // Never use more than 2^32 random nonces with a given key because of the risk of a repeat. nonce := make([]byte, nonceSize) if _, err := io.ReadFull(rand.Reader, nonce); err != nil { log.Fatalln(err.Error()) } plaintext, err := json.Marshal(data) if err != nil { return } ciphertext := aesgcm.Seal(nonce, nonce, plaintext, nil) //fmt.Printf("%#v\n", ciphertext) file, err := os.Create(fileName) if err != nil { return } defer file.Close() if _, err = file.Write(ciphertext); err != nil { return } return } func Load(file, pass string) (data map[string]interface{}, err error) { b, err := ioutil.ReadFile(file) if err != nil { err = nil return } digest := sha3.Sum256([]byte(pass)) block, err := aes.NewCipher(digest[:]) if err != nil { log.Fatalln(err) } aesgcm, err := cipher.NewGCM(block) if err != nil { log.Fatalln(err.Error()) } nonceSize := aesgcm.NonceSize() nonce, ciphertext := b[:nonceSize], b[nonceSize:] plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) if err != nil { err = fmt.Errorf("Password wrong for config: %s", file) return } ////fmt.Printf("%s\n", plaintext) data = map[string]interface{}{} err = json.Unmarshal(plaintext, &data) return } ================================================ FILE: src/defines/constant.go ================================================ package defines var Twitter = "@himananiito" var Email = "himananiito@yahoo.co.jp" ================================================ FILE: src/files/files.go ================================================ package files import ( "fmt" "os" "path/filepath" "strings" "regexp" ) func RemoveExtention(fileName string) string { e := filepath.Ext(fileName) base := strings.TrimSuffix(fileName, e) return base } func ChangeExtention(fileName, ext string) string { e := filepath.Ext(fileName) base := strings.TrimSuffix(fileName, e) return base + "." + ext } func MkdirByFileName(fileName string) (err error) { dir := filepath.Dir(fileName) err = os.MkdirAll(dir, os.ModePerm) if err != nil { fmt.Println(err) return } return } func GetFileNameNext(name string) (fileName string, err error) { fileName = name _, test := os.Stat(fileName) if test == nil { // file Exists ext := filepath.Ext(fileName) base := strings.TrimSuffix(fileName, ext) var i int for i = 2; i < 10000000 ; i++ { fileName = fmt.Sprintf("%s-%d%s", base, i, ext) _, test := os.Stat(fileName) if test != nil { return } } err = fmt.Errorf("too many files: %s", name) } return } func ReplaceForbidden(name string) (fileName string) { fileName = name fileName = regexp.MustCompile(`\\`).ReplaceAllString(fileName, "￥") fileName = regexp.MustCompile(`/`).ReplaceAllString(fileName, "∕") fileName = regexp.MustCompile(`:`).ReplaceAllString(fileName, "：") fileName = regexp.MustCompile(`\*`).ReplaceAllString(fileName, "＊") fileName = regexp.MustCompile(`\?`).ReplaceAllString(fileName, "？") fileName = regexp.MustCompile(`"`).ReplaceAllString(fileName, `゛`) fileName = regexp.MustCompile(`<`).ReplaceAllString(fileName, "＜") fileName = regexp.MustCompile(`>`).ReplaceAllString(fileName, "＞") fileName = regexp.MustCompile(`\|`).ReplaceAllString(fileName, "｜") fileName = regexp.MustCompile(`）`).ReplaceAllString(fileName, ")") fileName = regexp.MustCompile(`（`).ReplaceAllString(fileName, "(") fileName = regexp.MustCompile(`\p{Zs}+`).ReplaceAllString(fileName, " ") fileName = regexp.MustCompile(`\A\p{Zs}+|\p{Zs}+\z`).ReplaceAllString(fileName, "") // 末尾が.であるようなファイルは作れない fileName = regexp.MustCompile(`\.\p{Zs}*\z`).ReplaceAllString(fileName, "．") return } ================================================ FILE: src/flvs/flv.go ================================================ package flvs import ( "os" "fmt" "io" "encoding/binary" "bytes" "bufio" ) type Flv struct { filename string file *os.File writer *bufio.Writer startAt int audioTimestamp int videoTimestamp int } func (flv *Flv) Flush() { if flv.writer != nil { flv.writer.Flush() } } func (flv *Flv) Close() { flv.Flush() if flv.file != nil { flv.file.Close() } } func Open(name string) (flv *Flv, err error) { file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0777) if err != nil { return } flv = &Flv { filename: name, file: file, audioTimestamp: -1, videoTimestamp: -1, } stat, err := file.Stat() if err != nil { } // FLV header sz := stat.Size() if sz == 0 { if err = flv.writeHeader(); err != nil { flv.Close() return } } if err = flv.testHeader(); err != nil { flv.Close() return } flv.lastPacketTimestamp() if _, err = flv.file.Seek(0, 2); err != nil { return } ts := flv.GetLastTimestamp() if ts != 0 { fmt.Printf("[info] Seek point: %d\n", ts) } flv.writer = bufio.NewWriterSize(file, 256*1024) return } func (flv *Flv) AudioExists() bool { return flv.audioTimestamp >= 0 } func (flv *Flv) VideoExists() bool { return flv.videoTimestamp >= 0 } func (flv *Flv) testHeader() (err error) { if _, err = flv.file.Seek(0, 0); err != nil { return } b := make([]byte, 9) _, err = io.ReadFull(flv.file, b); if err != nil { return } if "FLV" != string(b[0:3]) { err = fmt.Errorf("magic number is not FLV") return } offset := binary.BigEndian.Uint32(b[5:9]) flv.startAt = int(offset) return } func intToBE24(num int) (data []byte) { tmp := make([]byte, 4) binary.BigEndian.PutUint32(tmp, uint32(num)) data = append(data, tmp[1:]...) return } func intToBE32(num int) (data []byte) { tmp := make([]byte, 4) binary.BigEndian.PutUint32(tmp, uint32(num)) data = append(data, tmp[:]...) return } func (flv *Flv) writePacket(tag byte, rdr *bytes.Buffer, ts int) (err error) { buff := bytes.NewBuffer(nil) dataSize := intToBE24(rdr.Len()) tagSize := intToBE32(11 + rdr.Len()) // TagType if err = buff.WriteByte(tag); err != nil { return } // DataSize if _, err = buff.Write(dataSize); err != nil { return } // Timestamp tsBytes := intToBE32(ts) if _, err = buff.Write(tsBytes[1:4]); err != nil { return } // (TimestampExtended) if err = buff.WriteByte(tsBytes[0]); err != nil { return } // StreamID if _, err = buff.Write([]byte{0, 0, 0}); err != nil { return } // header if _, err = io.Copy(flv.writer, buff); err != nil { return } // data if _, err = io.Copy(flv.writer, rdr); err != nil { return } // PreviousTagSize if _, err = flv.writer.Write(tagSize); err != nil { return } return } func (flv *Flv) WriteAudio(rdr *bytes.Buffer, ts int) (err error) { if ts > flv.audioTimestamp { flv.audioTimestamp = ts err = flv.writePacket(8, rdr, ts) } return } func (flv *Flv) WriteVideo(rdr *bytes.Buffer, ts int) (err error) { if ts > flv.videoTimestamp { flv.videoTimestamp = ts err = flv.writePacket(9, rdr, ts) } return } func (flv *Flv) WriteMetaData(rdr *bytes.Buffer, ts int) (err error) { err = flv.writePacket(18, rdr, ts) return } func (flv *Flv) GetLastTimestamp() int { var min int if flv.audioTimestamp > flv.videoTimestamp { min = flv.videoTimestamp } else { min = flv.audioTimestamp } if min < 0 { return 0 } return min } func (flv *Flv) lastPacketTimestamp() (err error) { defer flv.file.Seek(0, 2) if _, err = flv.file.Seek(-4, 2); err != nil { fmt.Printf("flv.lastPacketTimestamp: %#v\n", err) return } b0 := make([]byte, 4) b1 := make([]byte, 11) var audioFound bool var videoFound bool for !(audioFound && videoFound) { _, err = io.ReadFull(flv.file, b0); if err != nil { return } size := binary.BigEndian.Uint32(b0) //fmt.Printf("size: %d\n", size) if size == 0 { break } if _, err = flv.file.Seek(-(int64(size) + 4), 1); err != nil { return } _, err = io.ReadFull(flv.file, b1); if err != nil { return } ts := (int(b1[7]) << 24) | (int(b1[4]) << 16) | (int(b1[5]) << 8) | (int(b1[6]) ) //fmt.Printf("ts: %d\n", ts) if b1[0] == 8 { flv.audioTimestamp = ts audioFound = true } else if b1[0] == 9 { flv.videoTimestamp = ts videoFound = true } if _, err = flv.file.Seek(-(11 + 4), 1); err != nil { return } } return } func (flv *Flv) writeHeader() (err error) { _, err = flv.file.Write([]byte{ 'F', 'L', 'V', 1, // FLV version 1 5, // Audio+Video tags are present 0, 0, 0, 9, // DataOffset = 9 0, 0, 0, 0, // PreviousTagSize0 }) return } ================================================ FILE: src/go.mod ================================================ module github.com/himananiito/livedl go 1.16 require ( github.com/gin-gonic/gin v1.7.1 github.com/gorilla/websocket v1.4.2 github.com/mattn/go-sqlite3 v1.14.7 golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e ) replace github.com/himananiito/livedl => ./ ================================================ FILE: src/go.sum ================================================ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.1 h1:qC89GU3p8TvKWMAVhEpmpB2CIb1hnqt2UdKZaP93mS8= github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e h1:8foAy0aoO5GkqCvAEJ4VC4P3zksTg4X4aJCDpZzmgQI= golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= ================================================ FILE: src/gorman/gorman.go ================================================ package gorman import ( "sync" ) type GoroutineManager struct { channels map[chan struct{}] struct{} mtxChan sync.Mutex mtxWg sync.Mutex wg sync.WaitGroup codeChecker func(code int) } func NewManager() *GoroutineManager { return &GoroutineManager{ channels: map[chan struct{}] struct{}{}, } } func WithChecker(f func(int)) *GoroutineManager { return &GoroutineManager{ channels: map[chan struct{}] struct{}{}, codeChecker: f, } } func (gm *GoroutineManager) addChan(c chan struct{}) { gm.mtxChan.Lock() defer gm.mtxChan.Unlock() gm.channels[c] = struct{}{} } func (gm *GoroutineManager) delChan(c chan struct{}) { gm.mtxChan.Lock() defer gm.mtxChan.Unlock() delete(gm.channels, c) } func (gm *GoroutineManager) Cancel() { gm.mtxChan.Lock() defer gm.mtxChan.Unlock() for c, _ := range gm.channels { close(c) delete(gm.channels, c) } } func (gm *GoroutineManager) Count() int { gm.mtxChan.Lock() defer gm.mtxChan.Unlock() return len(gm.channels) } func (gm *GoroutineManager) Go(f func(<-chan struct{}) int) { gm.wg.Add(1) stopChan := make(chan struct{}, 1) gm.addChan(stopChan) go func(){ defer gm.wg.Done() code := f(stopChan) gm.delChan(stopChan) if gm.codeChecker != nil { gm.codeChecker(code) } }() } func (gm *GoroutineManager) RegisterCodeChecker(f func(int)) { gm.codeChecker = f } func (gm *GoroutineManager) Wait() { gm.wg.Wait() } ================================================ FILE: src/httpbase/httpbase.go ================================================ package httpbase import ( "bytes" "crypto/tls" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "io" "io/ioutil" "net/http" "net/url" "os" "strings" "time" "github.com/himananiito/livedl/buildno" "github.com/himananiito/livedl/defines" ) func GetUserAgent() string { return fmt.Sprintf( "livedl/%s (contact: twitter=%s, email=%s)", buildno.GetBuildNo(), defines.Twitter, defines.Email, ) } var Client = &http.Client{ Timeout: time.Duration(5) * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) (err error) { if req != nil && via != nil && len(via) > 0 { if len(via) >= 10 { return errors.New("stopped after 10 redirects") } req.Header = via[0].Header } return nil }, } func checkTransport() bool { if Client.Transport == nil { Client.Transport = &http.Transport{} } switch Client.Transport.(type) { case *http.Transport: return true } return false } func checkTLSClientConfig() bool { if !checkTransport() { return false } if Client.Transport.(*http.Transport).TLSClientConfig == nil { Client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{} } return true } func SetRootCA(file string) (err error) { if !checkTLSClientConfig() { err = fmt.Errorf("SetRootCA: check failed") return } dat, err := ioutil.ReadFile(file) if err != nil { return } // try decode pem var nDecode int for len(dat) > 0 { block, d := pem.Decode(dat) if block == nil { break } dat = d nDecode++ if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { continue } addCert(block.Bytes) } if nDecode < 1 { addCert(dat) } return } func addCert(dat []byte) (err error) { certs, err := x509.ParseCertificates(dat) if err != nil { return } if certs == nil { err = fmt.Errorf("ParseCertificates failed") return } if len(certs) > 0 { if Client.Transport.(*http.Transport).TLSClientConfig.RootCAs == nil { Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = x509.NewCertPool() } } for _, cert := range certs { Client.Transport.(*http.Transport).TLSClientConfig.RootCAs.AddCert(cert) } return } func SetSkipVerify(skip bool) (err error) { if checkTLSClientConfig() { Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify = skip } else { err = fmt.Errorf("SetSkipVerify(%#v): check failed", skip) } return } func SetProxy(rawurl string) (err error) { if !checkTransport() { return fmt.Errorf("SetProxy(%#v): check failed", rawurl) } u, err := url.Parse(rawurl) if err != nil { return } Client.Transport.(*http.Transport).Proxy = http.ProxyURL(u) return } func httpBase(method, uri string, header map[string]string, body io.Reader) (resp *http.Response, err, neterr error) { req, err := http.NewRequest(method, uri, body) if err != nil { return } req.Header.Set("User-Agent", GetUserAgent()) for k, v := range header { req.Header.Set(k, v) } resp, neterr = Client.Do(req) if neterr != nil { if strings.Contains(neterr.Error(), "x509: certificate signed by unknown") { fmt.Println(neterr) os.Exit(10) } return } return } func Get(uri string, header map[string]string) (*http.Response, error, error) { return httpBase("GET", uri, header, nil) } func PostForm(uri string, header map[string]string, val url.Values) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } header["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8" return httpBase("POST", uri, header, strings.NewReader(val.Encode())) } func reqJson(method, uri string, header map[string]string, data interface{}) ( *http.Response, error, error) { encoded, err := json.Marshal(data) if err != nil { return nil, err, nil } if header == nil { header = make(map[string]string) } header["Content-Type"] = "application/json" return httpBase(method, uri, header, bytes.NewReader(encoded)) } func PostJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { return reqJson("POST", uri, header, data) } func PutJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { return reqJson("PUT", uri, header, data) } func PostData(uri string, header map[string]string, data io.Reader) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } return httpBase("POST", uri, header, data) } func GetBytes(uri string, header map[string]string) (code int, buff []byte, err, neterr error) { resp, err, neterr := Get(uri, header) if err != nil { return } if neterr != nil { return } defer resp.Body.Close() buff, neterr = ioutil.ReadAll(resp.Body) if neterr != nil { return } code = resp.StatusCode return } ================================================ FILE: src/httpsub/httpsub.go ================================================ package httpsub import ( "net/http" "os" "sync" "log" "io" "fmt" "bytes" ) type SubDownloader struct { method string uri string data []byte Header map[string]string RangeSize int64 BuffSize int64 fileName string file *os.File numConcurrent int chRunning chan bool mtx sync.Mutex wg sync.WaitGroup chLength chan int64 } func (sub *SubDownloader) Concurrent(c int) { sub.numConcurrent = c } func Get(uri, fileName string) (sub *SubDownloader) { sub = &SubDownloader{ method: "GET", uri: uri, fileName: fileName, } return } func (sub *SubDownloader) Close() { sub.mtx.Lock() defer sub.mtx.Unlock() if sub.file != nil { sub.file.Close() sub.file = nil } } func (sub *SubDownloader) open() { f, err := os.Create(sub.fileName) if err != nil { log.Fatal(err) } sub.file = f } func (sub *SubDownloader) write(pos int64, rdr io.Reader) (err error) { sub.mtx.Lock() defer sub.mtx.Unlock() //fmt.Printf("write %d\n", pos) if sub.file == nil { sub.open() } if _, err = sub.file.Seek(pos, 0); err != nil { log.Fatalln(err) } if _, err = io.Copy(sub.file, rdr); err != nil { log.Fatalln(err) } return } func (sub *SubDownloader) subrange(pos int64) { //fmt.Printf("start subrange pos(%d), size(%d) \n", pos, sub.RangeSize) sub.wg.Add(1) sub.chRunning <- true go func() { defer func() { <-sub.chRunning sub.wg.Done() }() data := bytes.NewBuffer(sub.data) req, _ := http.NewRequest(sub.method, sub.uri, data) req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", pos, pos + sub.RangeSize - 1)) client := new(http.Client) resp, err := client.Do(req) if err != nil { return } defer resp.Body.Close() switch resp.StatusCode { case 206: default: log.Fatalf("StatusCode is %v\n", resp.StatusCode) } sub.chLength <- resp.ContentLength buff := new(bytes.Buffer) wbytes := int64(0) for { n, _ := io.CopyN(buff, resp.Body, sub.BuffSize) //fmt.Printf("buff size is %d\n", buff.Len()) if n > 0 { sub.write(pos + wbytes, buff) wbytes += n } else { return } } }() } func (sub *SubDownloader) Wait() { sub.chRunning = make(chan bool, sub.numConcurrent) sub.chLength = make(chan int64, 10) if sub.RangeSize <= 0 { sub.RangeSize = 10*1000*1000 } if sub.BuffSize <= 0 { sub.BuffSize = 3*1000*1000 } pos := int64(0) for { sub.subrange(pos) length := <-sub.chLength fmt.Printf("Downloading %v: %v-%v\n", sub.fileName, pos, pos + length - 1) if length == sub.RangeSize { pos += length } else { break } } sub.wg.Wait() sub.Close() } ================================================ FILE: src/livedl.go ================================================ package main import ( "fmt" "os" "path/filepath" "regexp" "strings" "time" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/niconico" "github.com/himananiito/livedl/options" "github.com/himananiito/livedl/twitcas" "github.com/himananiito/livedl/youtube" "github.com/himananiito/livedl/zip2mp4" ) func main() { var baseDir string if regexp.MustCompile(`\AC:\\.*\\Temp\\go-build[^\\]*\\[^\\]+\\exe\\[^\\]*\.exe\z`).MatchString(os.Args[0]) { // go runで起動時 pwd, e := os.Getwd() if e != nil { fmt.Println(e) return } baseDir = pwd } else { //pa, e := filepath.Abs(os.Args[0]) pa, e := os.Executable() if e != nil { fmt.Println(e) return } // symlinkを追跡する for { sl, e := os.Readlink(pa) if e != nil { break } pa = sl } baseDir = filepath.Dir(pa) } opt := options.ParseArgs() // chdir if not disabled if !opt.NoChdir { fmt.Printf("chdir: %s\n", baseDir) if e := os.Chdir(baseDir); e != nil { fmt.Println(e) return } } // http if opt.HttpRootCA != "" { if err := httpbase.SetRootCA(opt.HttpRootCA); err != nil { fmt.Println(err) return } } if opt.HttpSkipVerify { if err := httpbase.SetSkipVerify(true); err != nil { fmt.Println(err) return } } if opt.HttpProxy != "" { if err := httpbase.SetProxy(opt.HttpProxy); err != nil { fmt.Println(err) return } } switch opt.Command { default: fmt.Printf("Unknown command: %v\n", opt.Command) os.Exit(1) case "TWITCAS": var doneTime int64 for { done, dbLocked := twitcas.TwitcasRecord(opt.TcasId, "") if dbLocked { break } if !opt.TcasRetry { break } if opt.TcasRetryTimeoutMinute < 0 { } else if done { doneTime = time.Now().Unix() } else { if doneTime == 0 { doneTime = time.Now().Unix() } else { delta := time.Now().Unix() - doneTime var minutes int if opt.TcasRetryTimeoutMinute == 0 { minutes = options.DefaultTcasRetryTimeoutMinute } else { minutes = opt.TcasRetryTimeoutMinute } if minutes > 0 { if delta > int64(minutes*60) { break } } } } var interval int if opt.TcasRetryInterval <= 0 { interval = options.DefaultTcasRetryInterval } else { interval = opt.TcasRetryInterval } select { case <-time.After(time.Duration(interval) * time.Second): } } case "YOUTUBE": err := youtube.Record(opt.YoutubeId, opt.YtNoStreamlink, opt.YtNoYoutubeDl) if err != nil { fmt.Println(err) } case "NICOLIVE": hlsPlaylistEnd, dbname, err := niconico.Record(opt) if err != nil { fmt.Println(err) os.Exit(1) } if hlsPlaylistEnd && opt.NicoAutoConvert { done, nMp4s, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb) if err != nil { fmt.Println(err) os.Exit(1) } if done { if nMp4s == 1 { if 1 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } } else if 1 < nMp4s { if 2 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } } } } case "NICOLIVE_TEST": if err := niconico.TestRun(opt); err != nil { fmt.Println(err) os.Exit(1) } case "ZIP2MP4": if err := zip2mp4.Convert(opt.ZipFile); err != nil { fmt.Println(err) os.Exit(1) } case "DB2MP4": if strings.HasSuffix(opt.DBFile, ".yt.sqlite3") { zip2mp4.YtComment(opt.DBFile) } else if opt.ExtractChunks { if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb); err != nil { fmt.Println(err) os.Exit(1) } } else { if _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb); err != nil { fmt.Println(err) os.Exit(1) } } } return } ================================================ FILE: src/log4gui/log4gui.go ================================================ package log4gui import ( "fmt" "encoding/json" ) func print(k, v string) { bs, e := json.Marshal(map[string]string{ k: v, }) if(e != nil) { fmt.Println(e) return } fmt.Println("$" + string(bs) + "$") } func Info(s string) { print("Info", s) } func Error(s string) { print("Error", s) } ================================================ FILE: src/niconico/jikken.gox ================================================ package niconico import ( "fmt" "os" "time" "os/signal" "syscall" "net/http" "io/ioutil" "log" "encoding/json" "bytes" "../options" "../obj" "../files" ) func getActionTrackId() (actionTrackId string, err error) { uri := "https://public.api.nicovideo.jp/v1/action-track-ids.json" req, _ := http.NewRequest("POST", uri, nil) req.Header.Set("Content-Type", "application/json") client := new(http.Client) resp, e := client.Do(req) if e != nil { err = e return } defer resp.Body.Close() bs, err := ioutil.ReadAll(resp.Body) if err != nil { log.Println(err) } var props interface{} if err = json.Unmarshal(bs, &props); err != nil { return } //obj.PrintAsJson(props) data, ok := obj.FindString(props, "data") if (! ok) { err = fmt.Errorf("actionTrackId not found") } actionTrackId = data return } func jikkenWatching(opt options.Option, actionTrackId string, isArchive bool) (props interface{}, err error) { str, _ := json.Marshal(OBJ{ "actionTrackId": actionTrackId, "isBroadcaster": false, "isLowLatencyStream": true, "streamCapacity": "superhigh", "streamProtocol": "https", "streamQuality": "auto", // high, auto }) if err != nil { log.Println(err) return } data := bytes.NewReader(str) var uri string if isArchive { uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching-archive", opt.NicoLiveId) } else { uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) } req, _ := http.NewRequest("POST", uri, data) //if opt.NicoSession != "" { req.Header.Set("Cookie", "user_session=" + opt.NicoSession) //} req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") req.Header.Set("Origin", "https://cas.nicovideo.jp") req.Header.Set("X-Connection-Environment", "ethernet") req.Header.Set("X-Frontend-Id", "91") client := new(http.Client) resp, e := client.Do(req) if e != nil { err = e return } defer resp.Body.Close() bs, err := ioutil.ReadAll(resp.Body) if err != nil { log.Println(err) } if err = json.Unmarshal([]byte(bs), &props); err != nil { return } //obj.PrintAsJson(props) return } func jikkenPut(opt options.Option, actionTrackId string) (forbidden, notOnAir bool, err error) { str, _ := json.Marshal(OBJ{ "actionTrackId": actionTrackId, "isBroadcaster": false, }) if err != nil { log.Println(err) } fmt.Printf("\n%s\n\n", str) data := bytes.NewReader(str) uri := fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) req, _ := http.NewRequest("PUT", uri, data) //if opt.NicoSession != "" { req.Header.Set("Cookie", "user_session=" + opt.NicoSession) //} req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") req.Header.Set("Origin", "https://cas.nicovideo.jp") req.Header.Set("X-Frontend-Id", "91") client := new(http.Client) resp, e := client.Do(req) if e != nil { err = e return } defer resp.Body.Close() bs, err := ioutil.ReadAll(resp.Body) if err != nil { log.Println(err) } var props interface{} if err = json.Unmarshal([]byte(bs), &props); err != nil { return } //obj.PrintAsJson(props) if code, ok := obj.FindString(props, "meta", "errorCode"); ok { switch code { case "FORBIDDEN": forbidden = true return case "PROGRAM_NOT_ONAIR": notOnAir = true return } } return } func jikkenHousou(nicoliveProgramId, title, userId, nickname, communityId string, opt options.Option, isArchive bool) (err error) { chInterrupt := make(chan os.Signal, 10) signal.Notify(chInterrupt, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) actionTrackId, err := getActionTrackId() if err != nil { log.Println(err) } media := &NicoMedia{} defer func() { if media.zipWriter != nil { media.zipWriter.Close() } }() title = files.ReplaceForbidden(title) nickname = files.ReplaceForbidden(nickname) media.fileName = fmt.Sprintf("%s-%s-%s.zip", nicoliveProgramId, nickname, title) var nLast int L_main: for { select { case <-chInterrupt: break L_main default: } props, e := jikkenWatching(opt, actionTrackId, isArchive) if e != nil { err = e log.Println(err) return } if uri, ok := obj.FindString(props, "data", "streamServer", "url"); ok { //fmt.Println(uri) is403, e := media.SetPlaylist(uri) if is403 { break L_main } if e != nil { err = e log.Println(e) return } } L_loc: for i := 0; true; i++ { select { case <-chInterrupt: break L_main default: } is403, e := media.GetMedias() if is403 { n := media.getNumChunk() if n != nLast { nLast = n break L_loc } else { break L_main } } if e != nil { log.Println(e) return } if i > 60 { forbidden, notOnAir, e := jikkenPut(opt, actionTrackId) if e != nil { err = e log.Println(e) return } if notOnAir { break L_main } if forbidden { break L_loc } i = 0 } select { case <-chInterrupt: break L_main case <-time.After(1 * time.Second): } } } if media.zipWriter != nil { media.zipWriter.Close() } signal.Stop(chInterrupt) return } ================================================ FILE: src/niconico/nico.go ================================================ package niconico import ( "bufio" "encoding/xml" "fmt" "io/ioutil" "net" "net/http" _ "net/http/pprof" "net/url" "os" "os/signal" "regexp" "runtime" "strconv" "strings" "syscall" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/options" ) func NicoLogin(opt options.Option) (err error) { id, pass, _, _ := options.LoadNicoAccount(opt.NicoLoginAlias) if id == "" || pass == "" { err = fmt.Errorf("Login ID/Password not set. Use -nico-login \",\"") return } resp, err, neterr := httpbase.PostForm( "https://account.nicovideo.jp/api/v1/login", nil, url.Values{"mail_tel": {id}, "password": {pass}, "site": {"nicoaccountsdk"}}, ) if err != nil { return } if neterr != nil { err = neterr return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return } if ma := regexp.MustCompile(`(.+?)`).FindSubmatch(body); len(ma) > 0 { options.SetNicoSession(opt.NicoLoginAlias, string(ma[1])) fmt.Println("login success") } else { err = fmt.Errorf("login failed: session_key not found") return } return } func Record(opt options.Option) (hlsPlaylistEnd bool, dbName string, err error) { for i := 0; i < 2; i++ { // load session info if opt.NicoSession == "" || i > 0 { _, _, opt.NicoSession, _ = options.LoadNicoAccount(opt.NicoLoginAlias) } if !opt.NicoRtmpOnly { var done bool var notLogin bool var reserved bool done, hlsPlaylistEnd, notLogin, reserved, dbName, err = NicoRecHls(opt) if done { return } if err != nil { return } if notLogin { fmt.Println("not_login") if err = NicoLogin(opt); err != nil { return } continue } if reserved { continue } } if !opt.NicoHlsOnly { notLogin, e := NicoRecRtmp(opt) if e != nil { err = e return } if notLogin { fmt.Println("not_login") if err = NicoLogin(opt); err != nil { return } continue } } break } return } func TestRun(opt options.Option) (err error) { go func() { fmt.Println(http.ListenAndServe("localhost:6060", nil)) }() if false { ch := make(chan os.Signal, 10) signal.Notify(ch, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) go func() { <-ch os.Exit(0) }() } opt.NicoRtmpIndex = map[int]bool{ 0: true, } var nextId func() string if opt.NicoLiveId == "" { // niconama alert if opt.NicoTestTimeout <= 0 { opt.NicoTestTimeout = 12 } resp, e, nete := httpbase.Get("https://live.nicovideo.jp/api/getalertinfo", nil) if e != nil { err = e return } if nete != nil { err = nete return } defer resp.Body.Close() switch resp.StatusCode { case 200: default: err = fmt.Errorf("StatusCode is %v", resp.StatusCode) return } type Alert struct { User string `xml:"user_id"` UserHash string `xml:"user_hash"` Addr string `xml:"ms>addr"` Port string `xml:"ms>port"` Thread string `xml:"ms>thread"` } status := &Alert{} dat, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() err = xml.Unmarshal(dat, status) if err != nil { fmt.Println(string(dat)) fmt.Printf("error: %v", err) return } raddr, e := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%s", status.Addr, status.Port)) if e != nil { fmt.Printf("%v\n", e) return } conn, e := net.DialTCP("tcp", nil, raddr) if e != nil { err = e return } defer conn.Close() msg := fmt.Sprintf(`%c`, status.Thread, 0) if _, err = conn.Write([]byte(msg)); err != nil { fmt.Println(err) return } rdr := bufio.NewReader(conn) chLatest := make(chan string, 1000) go func() { for { s, e := rdr.ReadString(0) if e != nil { fmt.Println(e) err = e return } //fmt.Println(s) if ma := regexp.MustCompile(`>(\d+),\S+,\S+<`).FindStringSubmatch(s); len(ma) > 0 { L0: for { select { case <-chLatest: default: break L0 } } chLatest <- ma[1] } } }() nextId = func() string { L1: for { select { case <-chLatest: default: break L1 } } return <-chLatest } } else { // start from NicoLiveId var id int64 if ma := regexp.MustCompile(`\Alv(\d+)\z`).FindStringSubmatch(opt.NicoLiveId); len(ma) > 0 { if id, err = strconv.ParseInt(ma[1], 10, 64); err != nil { fmt.Println(err) return } } else { fmt.Println("TestRun: NicoLiveId not specified") return } nextId = func() (s string) { s = fmt.Sprintf("%d", id) id++ return } } if opt.NicoTestTimeout <= 0 { opt.NicoTestTimeout = 3 } //chErr := make(chan error) var NFCount int var endCount int for { opt.NicoLiveId = fmt.Sprintf("lv%s", nextId()) fmt.Fprintf(os.Stderr, "start test: %s\n", opt.NicoLiveId) fmt.Fprintf(os.Stderr, "# NumGoroutine: %d\n", runtime.NumGoroutine()) var msg string _, _, err = Record(opt) if err != nil { if ma := regexp.MustCompile(`\AError\s+code:\s*(\S+)`).FindStringSubmatch(err.Error()); len(ma) > 0 { msg = ma[1] switch ma[1] { case "notfound", "closed", "comingsoon", "timeshift_ticket_exhaust": case "deletedbyuser", "deletedbyvisor", "violated": case "usertimeshift", "tsarchive", "require_community_member", "noauth", "full", "premium_only", "selected-country": default: fmt.Fprintf(os.Stderr, "unknown: %s\n", ma[1]) return } } else if strings.Contains(err.Error(), "closed network") { msg = "OK" } else { fmt.Fprintln(os.Stderr, err) return } } else { msg = "OK" } fmt.Fprintf(os.Stderr, "%s: %s\n---------\n", opt.NicoLiveId, msg) endCount++ if endCount > 100 { break } if msg == "notfound" { NFCount++ } else { NFCount = 0 } if NFCount >= 10 { return } } return } ================================================ FILE: src/niconico/nico_db.go ================================================ package niconico import ( "database/sql" "fmt" "log" "os" "path/filepath" "strings" "time" "github.com/himananiito/livedl/files" ) var SelMedia = `SELECT seqno, bandwidth, size, data FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL ORDER BY seqno` var SelComment = `SELECT vpos, date, date_usec, IFNULL(no, -1) AS no, IFNULL(anonymity, 0) AS anonymity, user_id, content, IFNULL(mail, "") AS mail, IFNULL(premium, 0) AS premium, IFNULL(score, 0) AS score, thread, IFNULL(origin, "") AS origin, IFNULL(locale, "") AS locale FROM comment ORDER BY date2` func (hls *NicoHls) dbOpen() (err error) { db, err := sql.Open("sqlite3", hls.dbName) if err != nil { return } hls.db = db _, err = hls.db.Exec(` PRAGMA synchronous = OFF; PRAGMA journal_mode = WAL; `) if err != nil { return } err = hls.dbCreate() if err != nil { hls.db.Close() } return } func (hls *NicoHls) dbCreate() (err error) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() // table media _, err = hls.db.Exec(` CREATE TABLE IF NOT EXISTS media ( seqno INTEGER PRIMARY KEY NOT NULL UNIQUE, current INTEGER, position REAL, notfound INTEGER, bandwidth INTEGER, size INTEGER, data BLOB ) `) if err != nil { return } _, err = hls.db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS media0 ON media(seqno); CREATE INDEX IF NOT EXISTS media1 ON media(position); ---- for debug ---- CREATE INDEX IF NOT EXISTS media100 ON media(size); CREATE INDEX IF NOT EXISTS media101 ON media(notfound); `) if err != nil { return } // table comment _, err = hls.db.Exec(` CREATE TABLE IF NOT EXISTS comment ( vpos INTEGER NOT NULL, date INTEGER NOT NULL, date_usec INTEGER NOT NULL, date2 INTEGER NOT NULL, no INTEGER, anonymity INTEGER, user_id TEXT NOT NULL, content TEXT NOT NULL, mail TEXT, premium INTEGER, score INTEGER, thread TEXT, origin TEXT, locale TEXT, hash TEXT UNIQUE NOT NULL )`) if err != nil { return } _, err = hls.db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS comment0 ON comment(hash); ---- for debug ---- CREATE INDEX IF NOT EXISTS comment100 ON comment(date2); CREATE INDEX IF NOT EXISTS comment101 ON comment(no); `) if err != nil { return } // kvs media _, err = hls.db.Exec(` CREATE TABLE IF NOT EXISTS kvs ( k TEXT PRIMARY KEY NOT NULL UNIQUE, v BLOB ) `) if err != nil { return } _, err = hls.db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS kvs0 ON kvs(k); `) if err != nil { return } //hls.__dbBegin() return } // timeshift func (hls *NicoHls) dbSetPosition() { hls.dbExec(`UPDATE media SET position = ? WHERE seqno=?`, hls.playlist.position, hls.playlist.seqNo, ) } // timeshift func (hls *NicoHls) dbGetLastPosition() (res float64) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() hls.db.QueryRow("SELECT position FROM media ORDER BY POSITION DESC LIMIT 1").Scan(&res) return } //func (hls *NicoHls) __dbBegin() { // return /////////////////////////////////////////// //hls.db.Exec(`BEGIN TRANSACTION`) //} //func (hls *NicoHls) __dbCommit(t time.Time) { // return /////////////////////////////////////////// //// Never hls.dbMtx.Lock() //var start int64 //hls.db.Exec(`COMMIT; BEGIN TRANSACTION`) //if t.UnixNano() - hls.lastCommit.UnixNano() > 500000000 { // log.Printf("Commit: %s\n", hls.dbName) //} //hls.lastCommit = t //} func (hls *NicoHls) dbCommit() { // hls.dbMtx.Lock() // defer hls.dbMtx.Unlock() // hls.__dbCommit(time.Now()) } func (hls *NicoHls) dbExec(query string, args ...interface{}) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN]dbExec: %d(ms):%s\n", debug_Now(), t, query) } }() } if _, err := hls.db.Exec(query, args...); err != nil { fmt.Printf("dbExec %#v\n", err) //hls.db.Exec("COMMIT") hls.db.Close() os.Exit(1) } } func (hls *NicoHls) dbKVSet(k string, v interface{}) { query := `INSERT OR REPLACE INTO kvs (k,v) VALUES (?,?)` hls.startDBGoroutine(func(sig <-chan struct{}) int { hls.dbExec(query, k, v) return OK }) } func (hls *NicoHls) dbInsertReplaceOrIgnore(table string, data map[string]interface{}, replace bool) { var keys []string var qs []string var args []interface{} for k, v := range data { keys = append(keys, k) qs = append(qs, "?") args = append(args, v) } var replaceOrIgnore string if replace { replaceOrIgnore = "REPLACE" } else { replaceOrIgnore = "IGNORE" } query := fmt.Sprintf( `INSERT OR %s INTO %s (%s) VALUES (%s)`, replaceOrIgnore, table, strings.Join(keys, ","), strings.Join(qs, ","), ) hls.startDBGoroutine(func(sig <-chan struct{}) int { hls.dbExec(query, args...) return OK }) } func (hls *NicoHls) dbInsert(table string, data map[string]interface{}) { hls.dbInsertReplaceOrIgnore(table, data, false) } func (hls *NicoHls) dbReplace(table string, data map[string]interface{}) { hls.dbInsertReplaceOrIgnore(table, data, true) } // timeshift func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() var date2 int64 var no int hls.db.QueryRow("SELECT date2, no FROM comment ORDER BY date2 ASC LIMIT 1").Scan(&date2, &no) res_from = no if res_from <= 0 { res_from = 1 } if date2 == 0 { var endTime float64 hls.db.QueryRow(`SELECT v FROM kvs WHERE k = "endTime"`).Scan(&endTime) when = endTime } else { when = float64(date2) / (1000 * 1000) } return } func WriteComment(db *sql.DB, fileName string, skipHb bool) { rows, err := db.Query(SelComment) if err != nil { log.Println(err) return } defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") dir := filepath.Dir(fileName) base := filepath.Base(fileName) base, err = files.GetFileNameNext(base) if err != nil { fmt.Println(err) os.Exit(1) } fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) } defer f.Close() fmt.Fprintf(f, "%s\r\n", ``) fmt.Fprintf(f, "%s\r\n", ``) for rows.Next() { var vpos int64 var date int64 var date_usec int64 var no int64 var anonymity int64 var user_id string var content string var mail string var premium int64 var score int64 var thread string var origin string var locale string err = rows.Scan( &vpos, &date, &date_usec, &no, &anonymity, &user_id, &content, &mail, &premium, &score, &thread, &origin, &locale, ) if err != nil { log.Println(err) return } // skip /hb if (premium > 1) && skipHb && strings.HasPrefix(content, "/hb ") { continue } if vpos < 0 { continue } line := fmt.Sprintf( `= 0 { line += fmt.Sprintf(` no="%d"`, no) } if anonymity != 0 { line += fmt.Sprintf(` anonymity="%d"`, anonymity) } if mail != "" { mail = strings.Replace(mail, `"`, """, -1) mail = strings.Replace(mail, "&", "&", -1) mail = strings.Replace(mail, "<", "<", -1) line += fmt.Sprintf(` mail="%s"`, mail) } if origin != "" { origin = strings.Replace(origin, `"`, """, -1) origin = strings.Replace(origin, "&", "&", -1) origin = strings.Replace(origin, "<", "<", -1) line += fmt.Sprintf(` origin="%s"`, origin) } if premium != 0 { line += fmt.Sprintf(` premium="%d"`, premium) } if score != 0 { line += fmt.Sprintf(` score="%d"`, score) } if locale != "" { locale = strings.Replace(locale, `"`, """, -1) locale = strings.Replace(locale, "&", "&", -1) locale = strings.Replace(locale, "<", "<", -1) line += fmt.Sprintf(` locale="%s"`, locale) } line += ">" content = strings.Replace(content, "&", "&", -1) content = strings.Replace(content, "<", "<", -1) line += content line += "" fmt.Fprintf(f, "%s\r\n", line) } fmt.Fprintf(f, "%s\r\n", ``) } // ts func (hls *NicoHls) dbGetLastMedia(i int) (res []byte) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() hls.db.QueryRow("SELECT data FROM media WHERE seqno = ?", i).Scan(&res) return } func (hls *NicoHls) dbGetLastSeqNo() (res int64) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() hls.db.QueryRow("SELECT seqno FROM media ORDER BY seqno DESC LIMIT 1").Scan(&res) return } ================================================ FILE: src/niconico/nico_hls.go ================================================ package niconico import ( "context" "database/sql" "encoding/json" "fmt" "html" "io/ioutil" "log" "math" "net/http" _ "net/http/pprof" "net/url" "os" "os/signal" "regexp" "runtime" "strconv" "strings" "sync" "syscall" "time" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/objs" "github.com/himananiito/livedl/options" _ "github.com/mattn/go-sqlite3" "golang.org/x/crypto/sha3" ) type OBJ = map[string]interface{} type playlist struct { uri *url.URL uriMaster *url.URL uriTimeshiftMaster *url.URL bandwidth int nextTime time.Time format string withoutFormat bool seqNo int position float64 } type NicoHls struct { wsapi int startDelay int playlist playlist nicoliveProgramId string webSocketUrl string myUserId string commentStarted bool mtxCommentStarted sync.Mutex chInterrupt chan os.Signal nInterrupt int mtxInterrupt sync.Mutex mtxRestart sync.Mutex restartMain bool quality string errNumChunk int errRestartCnt int dbName string db *sql.DB dbMtx sync.Mutex lastCommit time.Time isTimeshift bool timeshiftStart float64 fastTimeshift bool ultrafastTimeshift bool fastTimeshiftOrig bool ultrafastTimeshiftOrig bool finish bool commentDone bool NicoSession string limitBw int limitBwOrig int nicoDebug bool msgErrorCount int msgErrorSeqNo int memdb *sql.DB memdbMtx sync.Mutex seqNo500 int cnt500 int bw500 int mtxWg sync.Mutex gmPlst *gorman.GoroutineManager gmCmnt *gorman.GoroutineManager gmDB *gorman.GoroutineManager gmMain *gorman.GoroutineManager } func debug_Now() string { return time.Now().Format("2006/01/02-15:04:05") } func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err error) { nicoliveProgramId, ok := prop["nicoliveProgramId"].(string) if !ok { err = fmt.Errorf("nicoliveProgramId is not string") return } webSocketUrl, ok := prop["//webSocketUrl"].(string) if !ok { err = fmt.Errorf("webSocketUrl is not string") return } wsapi := 2 if m := regexp.MustCompile(`/wsapi/v1/`).FindStringSubmatch(webSocketUrl); len(m) > 0 { wsapi = 1 log.Println("wsapi: 1") } myUserId, _ := prop["//myId"].(string) if myUserId == "" { myUserId = "NaN" } var timeshift bool if status, ok := prop["status"].(string); ok && status == "ENDED" { timeshift = true } if wsapi == 2 && false && !timeshift { if m := regexp.MustCompile(`/watch/([^?]+)`).FindStringSubmatch(webSocketUrl); len(m) > 0 { nicoliveProgramId = m[1] } webSocketUrl = strings.Replace(webSocketUrl, "/wsapi/v2/", "/wsapi/v1/", 1) wsapi = 1 log.Println("wsapi: 1") } var pid string if nicoliveProgramId, ok := prop["nicoliveProgramId"]; ok { pid, _ = nicoliveProgramId.(string) } var uname string // ユーザ名 var uid string // ユーザID var cname string // コミュ名 or チャンネル名 var cid string // コミュID or チャンネルID var pt string if providerType, ok := prop["providerType"]; ok { if pt, ok = providerType.(string); ok { if pt == "official" { uname = "official" uid = "official" cname = "official" cid = "official" } } } // ユーザ名 if userName, ok := prop["userName"]; ok { uname, _ = userName.(string) } // ユーザID if userPageUrl, ok := prop["userPageUrl"]; ok { if u, ok := userPageUrl.(string); ok { if m := regexp.MustCompile(`/user/(\d+)`).FindStringSubmatch(u); len(m) > 0 { uid = m[1] prop["userId"] = uid } } } if uid == "" && pt == "channel" { uid = "channel" } // コミュ名 if socName, ok := prop["socName"]; ok { cname, _ = socName.(string) } // コミュID if comId, ok := prop["comId"]; ok { cid, _ = comId.(string) } if cid == "" { if socId, ok := prop["socId"]; ok { cid, _ = socId.(string) } } var title string if t, ok := prop["title"]; ok { title, _ = t.(string) } var beginTime int64 if t, ok := prop["beginTime"]; ok { if bt, ok := t.(float64); ok { beginTime = int64(bt) } } tBegin := time.Unix(beginTime, 0) sYear := fmt.Sprintf("%04d", tBegin.Year()) sMonth := fmt.Sprintf("%02d", tBegin.Month()) sDay := fmt.Sprintf("%02d", tBegin.Day()) sDay8 := fmt.Sprintf("%04d%02d%02d", tBegin.Year(), tBegin.Month(), tBegin.Day()) sDay6 := fmt.Sprintf("%02d%02d%02d", tBegin.Year()%100, tBegin.Month(), tBegin.Day()) sHour := fmt.Sprintf("%02d", tBegin.Hour()) sMinute := fmt.Sprintf("%02d", tBegin.Minute()) sSecond := fmt.Sprintf("%02d", tBegin.Second()) sTime6 := fmt.Sprintf("%02d%02d%02d", tBegin.Hour(), tBegin.Minute(), tBegin.Second()) sTime4 := fmt.Sprintf("%02d%02d", tBegin.Hour(), tBegin.Minute()) // "${PID}-${UNAME}-${TITLE}" dbName := opt.NicoFormat dbName = strings.Replace(dbName, "?PID?", files.ReplaceForbidden(pid), -1) dbName = strings.Replace(dbName, "?UNAME?", files.ReplaceForbidden(uname), -1) dbName = strings.Replace(dbName, "?UID?", files.ReplaceForbidden(uid), -1) dbName = strings.Replace(dbName, "?CNAME?", files.ReplaceForbidden(cname), -1) dbName = strings.Replace(dbName, "?CID?", files.ReplaceForbidden(cid), -1) dbName = strings.Replace(dbName, "?TITLE?", files.ReplaceForbidden(title), -1) // date,time dbName = strings.Replace(dbName, "?YEAR?", sYear, -1) dbName = strings.Replace(dbName, "?MONTH?", sMonth, -1) dbName = strings.Replace(dbName, "?DAY?", sDay, -1) dbName = strings.Replace(dbName, "?DAY8?", sDay8, -1) dbName = strings.Replace(dbName, "?DAY6?", sDay6, -1) dbName = strings.Replace(dbName, "?HOUR?", sHour, -1) dbName = strings.Replace(dbName, "?MINUTE?", sMinute, -1) dbName = strings.Replace(dbName, "?SECOND?", sSecond, -1) dbName = strings.Replace(dbName, "?TIME6?", sTime6, -1) dbName = strings.Replace(dbName, "?TIME4?", sTime4, -1) if timeshift { dbName = dbName + "(TS)" } dbName = dbName + ".sqlite3" files.MkdirByFileName(dbName) hls = &NicoHls{ wsapi: wsapi, nicoliveProgramId: nicoliveProgramId, webSocketUrl: webSocketUrl, myUserId: myUserId, quality: "abr", dbName: dbName, isTimeshift: timeshift, fastTimeshift: opt.NicoFastTs || opt.NicoUltraFastTs, ultrafastTimeshift: opt.NicoUltraFastTs, NicoSession: opt.NicoSession, limitBw: opt.NicoLimitBw, limitBwOrig: opt.NicoLimitBw, nicoDebug: opt.NicoDebug, gmPlst: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), gmCmnt: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), gmDB: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), gmMain: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), timeshiftStart: opt.NicoTsStart, } hls.fastTimeshiftOrig = hls.fastTimeshift hls.ultrafastTimeshiftOrig = hls.ultrafastTimeshift for i := 0; i < 2; i++ { err := hls.dbOpen() if err != nil { if !strings.Contains(err.Error(), "able to open") { log.Fatalln(err) } } else if _, err := os.Stat(hls.dbName); err == nil { break } fmt.Printf("can't open: %s\n", hls.dbName) hls.dbName = fmt.Sprintf("%s.sqlite3", pid) } if err := hls.memdbOpen(); err != nil { fmt.Println(err) os.Exit(1) } // 放送情報をdbに入れる。自身のユーザ情報は入れない // dbに入れたくないデータはキーの先頭を//としている for k, v := range prop { if !strings.HasPrefix(k, "//") { hls.dbKVSet(k, v) } } return } func (hls *NicoHls) Close() { hls.dbCommit() if hls.db != nil { hls.db.Close() } if hls.memdb != nil { hls.memdb.Close() } } // Comment method func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { attrMap, ok := attr.(map[string]interface{}) if !ok { err = fmt.Errorf("[FIXME] commentHandler: not a map: %#v", attr) return } //fmt.Printf("%#v\n", attrMap) if vpos_f, ok := attrMap["vpos"].(float64); ok { vpos := int64(vpos_f) var date int64 if d, ok := attrMap["date"].(float64); ok { date = int64(d) } var date_usec int64 if d, ok := attrMap["date_usec"].(float64); ok { date_usec = int64(d) } date2 := (date * 1000 * 1000) + date_usec var user_id string if s, ok := attrMap["user_id"].(string); ok { user_id = s } var content string if s, ok := attrMap["content"].(string); ok { content = s } calc_s := fmt.Sprintf("%d,%d,%d,%s,%s", vpos, date, date_usec, user_id, content) hash := fmt.Sprintf("%x", sha3.Sum256([]byte(calc_s))) var thread string if d, ok := attrMap["thread"].(float64); ok { thread = fmt.Sprintf("%.f", d) } else if s, ok := attrMap["thread"].(string); ok { thread = s } hls.dbInsert("comment", map[string]interface{}{ "vpos": attrMap["vpos"], "date": attrMap["date"], "date_usec": attrMap["date_usec"], "date2": date2, "no": attrMap["no"], "anonymity": attrMap["anonymity"], "user_id": attrMap["user_id"], "content": attrMap["content"], "mail": attrMap["mail"], "premium": attrMap["premium"], "score": attrMap["score"], "thread": thread, "origin": attrMap["origin"], "locale": attrMap["locale"], "hash": hash, }) } else { if d, ok := attrMap["thread"].(float64); ok { hls.dbKVSet("comment/thread", fmt.Sprintf("%.f", d)) } else if s, ok := attrMap["thread"].(string); ok { hls.dbKVSet("comment/thread", s) } } return } // return code const ( OK = iota INTERRUPT MAIN_WS_ERROR MAIN_DISCONNECT MAIN_END_PROGRAM MAIN_INVALID_STREAM_QUALITY MAIN_TEMPORARILY_ERROR PLAYLIST_END PLAYLIST_403 PLAYLIST_ERROR DELAY COMMENT_WS_ERROR COMMENT_SAVE_ERROR COMMENT_DONE GOT_SIGNAL ERROR_SHUTDOWN NETWORK_ERROR ) func (hls *NicoHls) stopPCGoroutines() { hls.stopPGoroutines() hls.stopCGoroutines() } func (hls *NicoHls) stopAllGoroutines() { hls.stopPGoroutines() hls.stopCGoroutines() hls.stopMGoroutines() } func (hls *NicoHls) stopPGoroutines() { hls.gmPlst.Cancel() } func (hls *NicoHls) stopCGoroutines() { hls.gmCmnt.Cancel() } func (hls *NicoHls) stopMGoroutines() { hls.gmMain.Cancel() } func (hls *NicoHls) working() bool { return hls.gmPlst.Count() > 0 || hls.gmCmnt.Count() > 0 || hls.gmDB.Count() > 0 } func (hls *NicoHls) stopInterrupt() { if hls.chInterrupt != nil { signal.Stop(hls.chInterrupt) } } func (hls *NicoHls) startInterrupt() { if hls.chInterrupt == nil { hls.chInterrupt = make(chan os.Signal, 10) signal.Notify(hls.chInterrupt, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) } hls.startMGoroutine(func(sig <-chan struct{}) int { select { case <-hls.chInterrupt: hls.IncrInterrupt() fmt.Printf("Interrupt count: %d\n", hls.nInterrupt) go func() { hls.dbCommit() }() if hls.nInterrupt >= 2 { os.Exit(0) } return INTERRUPT case <-sig: return GOT_SIGNAL } }) } func (hls *NicoHls) IncrInterrupt() { hls.mtxInterrupt.Lock() defer hls.mtxInterrupt.Unlock() hls.nInterrupt++ } func (hls *NicoHls) interrupted() bool { hls.mtxInterrupt.Lock() defer hls.mtxInterrupt.Unlock() return hls.nInterrupt != 0 } func (hls *NicoHls) getStartDelay() int { hls.mtxRestart.Lock() defer hls.mtxRestart.Unlock() return hls.startDelay } func (hls *NicoHls) markRestartMain(delay int) { hls.mtxRestart.Lock() defer hls.mtxRestart.Unlock() if (!hls.restartMain) && (!hls.finish) { hls.startDelay = delay hls.restartMain = true } } func (hls *NicoHls) checkReturnCode(code int) { // NEVER restart goroutines here except interrupt handler switch code { case NETWORK_ERROR, MAIN_TEMPORARILY_ERROR: delay := hls.getStartDelay() if delay < 1 { hls.markRestartMain(1) } else if delay < 3 { hls.markRestartMain(3) } else if delay < 13 { // if 3,4,5..12 hls.markRestartMain(delay + 1) } else { hls.markRestartMain(60) } hls.stopPCGoroutines() case DELAY: //log.Println("delay") case PLAYLIST_403: // 番組終了時、websocketでEND_PROGRAMが来るよりも先にこうなるが、 // END_PROGRAMを受信するにはwebsocketの再接続が必要 //log.Println("403") if !hls.interrupted() { hls.markRestartMain(0) } hls.stopPGoroutines() case PLAYLIST_END: fmt.Println("playlist end.") hls.finish = true if hls.isTimeshift { if hls.commentDone { hls.stopPCGoroutines() } else if !hls.getCommentStarted() { hls.stopPCGoroutines() } else { fmt.Println("waiting comment") } } else { hls.stopPCGoroutines() } case MAIN_WS_ERROR: hls.stopPGoroutines() case MAIN_DISCONNECT: hls.stopPCGoroutines() case MAIN_END_PROGRAM: hls.finish = true hls.stopPCGoroutines() case MAIN_INVALID_STREAM_QUALITY: hls.markRestartMain(0) hls.stopPGoroutines() case PLAYLIST_ERROR: hls.stopPCGoroutines() case COMMENT_WS_ERROR: //log.Println("comment websocket error") hls.stopCGoroutines() case COMMENT_SAVE_ERROR: //log.Println("comment save error") hls.stopCGoroutines() case INTERRUPT: hls.startInterrupt() hls.stopPCGoroutines() case ERROR_SHUTDOWN: hls.stopPCGoroutines() case COMMENT_DONE: hls.commentDone = true if hls.finish { hls.stopPCGoroutines() } case OK: } } // Of playlist func (hls *NicoHls) startPGoroutine(f func(<-chan struct{}) int) { if !hls.interrupted() { hls.gmPlst.Go(f) } } // Of comment func (hls *NicoHls) startCGoroutine(f func(<-chan struct{}) int) { if !hls.interrupted() { hls.gmCmnt.Go(f) } } // Of DB func (hls *NicoHls) startDBGoroutine(f func(<-chan struct{}) int) { if !hls.interrupted() { hls.gmDB.Go(f) } } // Of main func (hls *NicoHls) startMGoroutine(f func(<-chan struct{}) int) { hls.gmMain.Go(f) } func (hls *NicoHls) waitRestartMain() bool { pc, _, _, ok := runtime.Caller(1) if ok { fn := runtime.FuncForPC(pc) if !strings.HasSuffix(fn.Name(), ".Wait") { log.Printf("[FIXME] Don't call waitRestartMain from %s\n", fn.Name()) } } hls.waitPGoroutines() hls.mtxRestart.Lock() defer hls.mtxRestart.Unlock() if hls.restartMain { hls.restartMain = false //hls.wgPlaylist = &sync.WaitGroup{} hls.startMain() return true } return false } func (hls *NicoHls) waitPGoroutines() { hls.gmPlst.Wait() } func (hls *NicoHls) waitCGoroutines() { hls.gmCmnt.Wait() } func (hls *NicoHls) waitDBGoroutines() { hls.gmDB.Wait() } func (hls *NicoHls) waitMGoroutines() { hls.gmMain.Wait() } func (hls *NicoHls) waitAllGoroutines() { hls.waitPGoroutines() hls.waitCGoroutines() hls.waitDBGoroutines() hls.waitMGoroutines() } func (hls *NicoHls) getwaybackkey(threadId string) (waybackkey string, neterr, err error) { uri := fmt.Sprintf("https://live.nicovideo.jp/api/getwaybackkey?thread=%s", url.QueryEscape(threadId)) resp, err, neterr := httpbase.Get(uri, map[string]string{"Cookie": "user_session=" + hls.NicoSession}) if err != nil { return } if neterr != nil { return } defer resp.Body.Close() dat, neterr := ioutil.ReadAll(resp.Body) if neterr != nil { return } waybackkey = strings.TrimPrefix(string(dat), "waybackkey=") return } func (hls *NicoHls) getTsCommentFromWhen() (res_from int, when float64) { return hls.dbGetFromWhen() } func (hls *NicoHls) setCommentStarted(val bool) { hls.mtxCommentStarted.Lock() defer hls.mtxCommentStarted.Unlock() hls.commentStarted = val } func (hls *NicoHls) getCommentStarted() bool { hls.mtxCommentStarted.Lock() defer hls.mtxCommentStarted.Unlock() return hls.commentStarted } func (hls *NicoHls) startComment(messageServerUri, threadId, waybackkey string) { if (!hls.getCommentStarted()) && (!hls.commentDone) { hls.setCommentStarted(true) hls.startCGoroutine(func(sig <-chan struct{}) int { defer func() { hls.setCommentStarted(false) }() var err error // here blocks several seconds conn, _, err := websocket.DefaultDialer.Dial( messageServerUri, map[string][]string{ "Origin": []string{"https://live2.nicovideo.jp"}, "Sec-WebSocket-Protocol": []string{"msg.nicovideo.jp#json"}, "User-Agent": []string{httpbase.GetUserAgent()}, }, ) if err != nil { if !hls.interrupted() { log.Println("comment connect:", err) } return COMMENT_WS_ERROR } var wsMtx sync.Mutex writeJson := func(d interface{}) error { wsMtx.Lock() defer wsMtx.Unlock() return conn.WriteJSON(d) } hls.startCGoroutine(func(sig <-chan struct{}) int { <-sig if conn != nil { conn.Close() } return OK }) hls.startCGoroutine(func(sig <-chan struct{}) int { for !hls.interrupted() { select { case <-time.After(60 * time.Second): if conn != nil { if err := writeJson(""); err != nil { if !hls.interrupted() { log.Println("comment send null:", err) } return COMMENT_WS_ERROR } } else { return OK } case <-sig: return GOT_SIGNAL } } return OK }) var mtxChatTime sync.Mutex var _chatCount int64 incChatCount := func() { mtxChatTime.Lock() defer mtxChatTime.Unlock() _chatCount++ } getChatCount := func() int64 { mtxChatTime.Lock() defer mtxChatTime.Unlock() return _chatCount } if hls.isTimeshift { hls.startCGoroutine(func(sig <-chan struct{}) int { defer func() { fmt.Println("Comment done.") }() var pre int64 var finishHint int for !hls.interrupted() { select { case <-time.After(1 * time.Second): c := getChatCount() if c == 0 || c == pre { _, when := hls.getTsCommentFromWhen() //fmt.Printf("getTsCommentFromWhen %f %d\n", when, res_from) err = writeJson([]OBJ{ OBJ{"ping": OBJ{"content": "rs:1"}}, OBJ{"ping": OBJ{"content": "ps:5"}}, OBJ{"thread": OBJ{ "fork": 0, "nicoru": 0, "res_from": -1000, "scores": 1, "thread": threadId, "user_id": hls.myUserId, "version": "20061206", "waybackkey": waybackkey, "when": when + 1, "with_global": 1, }}, OBJ{"ping": OBJ{"content": "pf:5"}}, OBJ{"ping": OBJ{"content": "rf:1"}}, }) if err != nil { return NETWORK_ERROR } } else if c < pre+100 { // 通常,1000カウント弱増えるが、少ししか増えない場合 finishHint++ if finishHint > 2 { return COMMENT_DONE } } else { finishHint = 0 } pre = c case <-sig: return GOT_SIGNAL } } return COMMENT_DONE }) } else { err = writeJson([]OBJ{ OBJ{"ping": OBJ{"content": "rs:0"}}, OBJ{"ping": OBJ{"content": "ps:0"}}, OBJ{"thread": OBJ{ "fork": 0, "nicoru": 0, "res_from": -1000, "scores": 1, "thread": threadId, "user_id": hls.myUserId, "version": "20061206", "with_global": 1, }}, OBJ{"ping": OBJ{"content": "pf:0"}}, OBJ{"ping": OBJ{"content": "rf:0"}}, }) if err != nil { if !hls.interrupted() { log.Println("comment send first:", err) } return COMMENT_WS_ERROR } } for !hls.interrupted() { select { case <-sig: return GOT_SIGNAL default: var res interface{} // Blocks here if err = conn.ReadJSON(&res); err != nil { return COMMENT_WS_ERROR } //fmt.Printf("debug %#v\n", res) if data, ok := objs.Find(res, "chat"); ok { if err := hls.commentHandler("chat", data); err != nil { return COMMENT_SAVE_ERROR } incChatCount() } else if data, ok := objs.Find(res, "thread"); ok { if err := hls.commentHandler("thread", data); err != nil { return COMMENT_SAVE_ERROR } } else if _, ok := objs.Find(res, "ping"); ok { // nop } else { fmt.Printf("[FIXME] Unknown Message: %#v\n", res) } } } return OK }) } } func urlJoin(base *url.URL, uri string) (res *url.URL, err error) { u, e := url.Parse(uri) if e != nil { err = e return } res = base.ResolveReference(u) return } func getStringBase(uri string, header map[string]string) (s string, code int, t int64, err, neterr error) { start := time.Now().UnixNano() defer func() { t = (time.Now().UnixNano() - start) / (1000 * 1000) }() resp, err, neterr := httpbase.Get(uri, header) if err != nil { return } if neterr != nil { return } defer resp.Body.Close() bs, neterr := ioutil.ReadAll(resp.Body) if neterr != nil { return } s = string(bs) code = resp.StatusCode return } func getString(uri string) (s string, code int, t int64, err, neterr error) { return getStringBase(uri, nil) } func getStringHeader(uri string, header map[string]string) (s string, code int, t int64, err, neterr error) { return getStringBase(uri, header) } func postStringHeader(uri string, header map[string]string, val url.Values) (s string, code int, t int64, err, neterr error) { start := time.Now().UnixNano() defer func() { t = (time.Now().UnixNano() - start) / (1000 * 1000) }() resp, err, neterr := httpbase.PostForm(uri, header, val) if err != nil { return } if neterr != nil { return } defer resp.Body.Close() bs, neterr := ioutil.ReadAll(resp.Body) if neterr != nil { return } s = string(bs) code = resp.StatusCode return } func getBytes(uri string) (code int, buff []byte, t int64, err, neterr error) { start := time.Now().UnixNano() defer func() { t = (time.Now().UnixNano() - start) / (1000 * 1000) }() resp, err, neterr := httpbase.Get(uri, nil) if err != nil { return } if neterr != nil { return } defer resp.Body.Close() buff, neterr = ioutil.ReadAll(resp.Body) if neterr != nil { return } code = resp.StatusCode return } func (hls *NicoHls) saveMedia(seqno int, uri string) (is403, is404, is500 bool, neterr, err error) { var timePassed []int64 if hls.nicoDebug { timePassed = append(timePassed, time.Now().UnixNano()) start := time.Now().UnixNano() defer func() { now := time.Now().UnixNano() timePassed = append(timePassed, now) t := (now - start) / (1000 * 1000) fmt.Fprintf(os.Stderr, "%s:saveMedia: seqno=%d, total %d(ms) %v\n", debug_Now(), seqno, t, timePassed) }() } code, buff, millisec, err, neterr := getBytes(uri) if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:getBytes@saveMedia: seqno=%d, code=%v, err=%v, neterr=%v, %v(ms), len=%v\n", debug_Now(), seqno, code, err, neterr, millisec, len(buff)) } if err != nil || neterr != nil { return } switch code { case 403: is403 = true return case 404: data := map[string]interface{}{ "seqno": seqno, "current": hls.playlist.seqNo, "notfound": 1, } if hls.nicoDebug { timePassed = append(timePassed, time.Now().UnixNano()) } hls.dbInsert("media", data) if hls.nicoDebug { timePassed = append(timePassed, time.Now().UnixNano()) } hls.memdbSet404(seqno) is404 = true return case 500: is500 = true return case 200: // OK } data := map[string]interface{}{ "seqno": seqno, "current": hls.playlist.seqNo, "size": len(buff), "bandwidth": hls.playlist.bandwidth, "data": buff, } if seqno == hls.playlist.seqNo { if hls.isTimeshift { data["position"] = hls.playlist.position } } if hls.nicoDebug { timePassed = append(timePassed, time.Now().UnixNano()) } hls.dbReplace("media", data) if hls.nicoDebug { timePassed = append(timePassed, time.Now().UnixNano()) } hls.memdbSet200(seqno) return } func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, is500 bool, neterr, err error) { u := argUri.String() m3u8, code, millisec, err, neterr := getString(u) if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:getPlaylist: code=%v, err=%v, neterr=%v, %v(ms) >>>%s<<<\n", debug_Now(), code, err, neterr, millisec, m3u8) } if err != nil || neterr != nil { return } switch code { case 200: case 403: is403 = true return default: if 500 <= code && code <= 599 { if strings.Contains(u, "playlist.m3u8") || !strings.Contains(u, "master.m3u8") { if hls.seqNo500 == hls.playlist.seqNo { hls.cnt500++ if hls.cnt500 >= 3 { if hls.bw500 == hls.playlist.bandwidth { err = fmt.Errorf("# playlist code=%v, hls.bw500=%v, hls.playlist.bandwidth=%v", code, hls.bw500, hls.playlist.bandwidth, ) return } else { hls.bw500 = hls.playlist.bandwidth fmt.Printf("Changing limitBw: %v -> %v\n", hls.limitBw, hls.playlist.bandwidth-1) hls.limitBw = hls.playlist.bandwidth - 1 } } } else { hls.seqNo500 = hls.playlist.seqNo hls.cnt500 = 1 } } else { // master.m3u8が500 hls.seqNo500 = -1 hls.cnt500 = 0 hls.bw500 = -1 hls.limitBw = hls.limitBwOrig } is500 = true return } fmt.Printf("#### playlist code: %d: %s\n", code, argUri.String()) err = fmt.Errorf("playlist code: %d: %s", code, argUri.String()) return } re := regexp.MustCompile(`#EXT-X-MEDIA-SEQUENCE:(\d+)`) ma := re.FindStringSubmatch(m3u8) if len(ma) > 0 { // Index m3u8 // #CURRENT-POSITION:0.0 // #DMC-CURRENT-POSITION:0.0 var currentPos float64 if ma := regexp.MustCompile(`#(?:DMC-)?CURRENT-POSITION:([\+\-]?\d+(?:\.\d+)?(?:[eE][\+\-]?\d+)?)`). FindStringSubmatch(m3u8); len(ma) > 0 { if hls.isTimeshift { n, e := strconv.ParseFloat(ma[1], 64) if e != nil { err = e return } currentPos = n hls.playlist.position = currentPos } else { // timeshiftじゃないのにCURRENT-POSITIONがあれば終了 isEnd = true return } } else { if hls.isTimeshift { currentPos = hls.timeshiftStart } } // 総時間 var streamDuration float64 if hls.isTimeshift { if ma := regexp.MustCompile(`#(?:DMC-)?STREAM-DURATION:([\+\-]?\d+(?:\.\d+)?(?:[eE][\+\-]?\d+)?)`). FindStringSubmatch(m3u8); len(ma) > 0 { n, e := strconv.ParseFloat(ma[1], 64) if e != nil { err = e return } streamDuration = n } } var seqStart int seqStart, err = strconv.Atoi(ma[1]) if err != nil { log.Fatal(err) } hls.playlist.seqNo = seqStart re := regexp.MustCompile(`#EXTINF:([\+\-]?\d+(?:\.\d+)?(?:[eE][\+\-]?\d+)?)[^\n]*\n(\S+)`) ma := re.FindAllStringSubmatch(m3u8, -1) if len(ma) == 0 { log.Println("No medias in playlist") hls.playlist.nextTime = time.Now().Add(time.Second) return } type seq_t struct { seqno int duration float64 uri string } var seqlist []seq_t var seqMax int var totalDuration float64 for i, a := range ma { var duration float64 seqno := i + hls.playlist.seqNo if seqno > seqMax { seqMax = seqno } if hls.isTimeshift || i == 0 { d, e := strconv.ParseFloat(a[1], 64) if e != nil { err = e return } if hls.isTimeshift { duration = d totalDuration += d } else { if i == 0 { if d > 3 { fmt.Printf("debug: found EXTINF=%v\n", d) d = 2.0 } else { d = d + 0.5 } t := time.Duration(float64(time.Second) * d) hls.playlist.nextTime = time.Now().Add(t) } } } uri, e := urlJoin(argUri, a[2]) if e != nil { err = e return } seqlist = append(seqlist, seq_t{ seqno: seqno, duration: duration, uri: uri.String(), }) // メディアのURLがシーケンス番号の部分だけが変わる形式かどうか if (!hls.isTimeshift) && (!hls.playlist.withoutFormat) { f := strings.Replace( strings.Replace(uri.String(), "%", "%%", -1), fmt.Sprintf("%d.ts?", seqno), "%d.ts?", 1, ) if hls.playlist.format == "" { hls.playlist.format = f } else if hls.playlist.format != f { fmt.Println(m3u8) fmt.Println("[FIXME] media format changed") hls.playlist.withoutFormat = true } } } if hls.isTimeshift { if !hls.ultrafastTimeshift { td := seqlist[0].duration * float64(time.Second) hls.playlist.nextTime = time.Now().Add(time.Duration(td)) } } // prints Current SeqNo if hls.isTimeshift { sec := int(hls.playlist.position) var pos string if sec >= 3600 { pos += fmt.Sprintf("%02d:%02d:%02d", sec/3600, (sec%3600)/60, sec%60) } else { pos += fmt.Sprintf("%02d:%02d", sec/60, sec%60) } fmt.Printf("Current SeqNo: %d, Pos: %s\n", hls.playlist.seqNo, pos) } else { fmt.Printf("Current SeqNo: %d\n", hls.playlist.seqNo) } minSeq := math.MaxInt32 maxSeq := -1 if (!hls.isTimeshift) && (!hls.playlist.withoutFormat) { // 404になるまで後ろに戻ってチャンクを取得する if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:start chunks(back)\n", debug_Now()) } for i := hls.playlist.seqNo - 1; i >= 0; i-- { if hls.memdbGetStopBack(i) { break } u := fmt.Sprintf(hls.playlist.format, i) var is404 bool is403, is404, _, neterr, err = hls.saveMedia(i, u) if neterr != nil || err != nil { return } if is403 { return } if i > maxSeq { maxSeq = i } if i < minSeq { minSeq = i } if is404 { break } } } // m3u8の通りにチャンクを取得する if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:start chunks(normal)\n", debug_Now()) } // 一時的に倍速モードを切っているかもしれないので戻す if hls.isTimeshift && (0 < hls.playlist.seqNo && hls.playlist.seqNo < 10) { hls.fastTimeshift = hls.fastTimeshiftOrig hls.ultrafastTimeshift = hls.ultrafastTimeshiftOrig } if hls.isTimeshift { hls.timeshiftStart = currentPos - 0.49 } var found404 bool for _, seq := range seqlist { if hls.isTimeshift { hls.timeshiftStart += seq.duration } if hls.memdbCheck200(seq.seqno) { if seq.seqno == hls.playlist.seqNo { if hls.isTimeshift { hls.dbSetPosition() } } continue } var is404 bool is403, is404, is500, neterr, err = hls.saveMedia(seq.seqno, seq.uri) if neterr != nil || err != nil { return } if is404 { fmt.Printf("sequence 404: %d\n", seq.seqno) found404 = true } if is403 { return } // TS時、先頭(SeqNo=0)で500となる時があるが // Seekしなければ次回に取得可能なので一時的に倍速モードを切る if is500 && hls.fastTimeshift && (seq.seqno == 0) { fmt.Println("[WARN] disabled fastTimeshift") hls.fastTimeshift = false hls.ultrafastTimeshift = false return } if seq.seqno < minSeq { minSeq = seq.seqno } if !found404 { maxSeq = seq.seqno } } if minSeq != math.MaxInt32 && maxSeq > 0 { for i := minSeq; i <= maxSeq; i++ { hls.memdbSetStopBack(i) } hls.memdbDelete(hls.playlist.seqNo) } if strings.Contains(m3u8, "#EXT-X-ENDLIST") { isEnd = true return } if hls.isTimeshift { d := streamDuration - (currentPos + totalDuration) if d < 1.0 { isEnd = true return } } } else { // Master m3u8 re := regexp.MustCompile(`#EXT-X-STREAM-INF:(?:[^\n]*[^\n\w-])?BANDWIDTH=(\d+)[^\n]*\n(\S+)`) ma := re.FindAllStringSubmatch(m3u8, -1) if len(ma) > 0 { var maxBw int var uri *url.URL for _, a := range ma { bw, err := strconv.Atoi(a[1]) if err != nil { log.Fatal(err) } set := func() { maxBw = bw uri, err = urlJoin(argUri, a[2]) if err != nil { log.Println(err) } } if maxBw == 0 { set() } else if hls.limitBw > 0 { // with limit // もし現在値が制限を超えていたら、現在値より小さければセット。 if hls.limitBw < maxBw && bw < maxBw { set() // 現在値が制限以下で、制限を超えないかつ現在値より大きければセット。 } else if maxBw <= hls.limitBw && bw <= hls.limitBw && maxBw < bw { set() } } else { // without limit if maxBw < bw { set() } } } if uri == nil { err = fmt.Errorf("playlist uri not defined") return } fmt.Printf("BANDWIDTH: %d\n", maxBw) hls.playlist.bandwidth = maxBw if hls.isTimeshift && hls.fastTimeshift { } else { hls.playlist.uriMaster = argUri hls.playlist.uri = uri } return hls.getPlaylist(uri) } else { log.Println("playlist error") } } return } func (hls *NicoHls) startPlaylist(uri string) { hls.startPGoroutine(func(sig <-chan struct{}) int { hls.playlist = playlist{} //hls.playlist.uri = uri u, e := url.Parse(uri) if e != nil { return PLAYLIST_ERROR } hls.playlist.uri = u if hls.isTimeshift { hls.playlist.uriTimeshiftMaster = u } if hls.isTimeshift { if hls.timeshiftStart == 0 { hls.timeshiftStart = hls.dbGetLastPosition() } u := hls.playlist.uriTimeshiftMaster.String() u = regexp.MustCompile(`&start=\d+(?:\.\d*)?`).ReplaceAllString(u, "") u += fmt.Sprintf("&start=%f", hls.timeshiftStart) uri, _ := url.Parse(u) hls.playlist.uri = uri } for !hls.interrupted() { var dur time.Duration if hls.playlist.nextTime.IsZero() { dur = 0 } else { now := time.Now() dur = hls.playlist.nextTime.Sub(now) } // 181002 if dur < time.Second { dur = time.Second } if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:time.After()=%v(sec)\n", debug_Now(), float64(dur)/float64(time.Second)) } select { case <-time.After(dur): var uri *url.URL if hls.isTimeshift && hls.fastTimeshift { u := hls.playlist.uriTimeshiftMaster.String() u = regexp.MustCompile(`&start=\d+(?:\.\d*)?`).ReplaceAllString(u, "") u += fmt.Sprintf("&start=%f", hls.timeshiftStart) uri, _ = url.Parse(u) } else { uri = hls.playlist.uri } //fmt.Println(uri) is403, isEnd, is500, neterr, err := hls.getPlaylist(uri) if neterr != nil { if !hls.interrupted() { log.Println("playlist:", e) } return NETWORK_ERROR } if is500 { if !hls.interrupted() { log.Println("playlist(500):", e) } return NETWORK_ERROR } if err != nil { if !hls.interrupted() { log.Println("playlist:", e) } return PLAYLIST_ERROR } if is403 { return PLAYLIST_403 } if isEnd { return PLAYLIST_END } case <-sig: return GOT_SIGNAL } } return OK }) } func (hls *NicoHls) startMain() { if hls.wsapi == 1 { hls.startMainV1() return } // エラー時はMAIN_*を返すこと hls.startPGoroutine(func(sig <-chan struct{}) int { if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:startMain: delay = %d(sec)\n", debug_Now(), hls.startDelay) } select { case <-time.After(time.Duration(hls.startDelay) * time.Second): case <-sig: return GOT_SIGNAL } if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:start dial main(%s)\n", debug_Now(), hls.webSocketUrl) } conn, _, err := websocket.DefaultDialer.Dial( hls.webSocketUrl, map[string][]string{ "User-Agent": []string{httpbase.GetUserAgent()}, }, ) if err != nil { return NETWORK_ERROR } var wsMtx sync.Mutex writeJson := func(d interface{}) error { wsMtx.Lock() defer wsMtx.Unlock() return conn.WriteJSON(d) } // debug if false { log.Printf("start ws error tsst") hls.startPGoroutine(func(sig <-chan struct{}) int { select { case <-time.After(10 * time.Second): conn.Close() return OK case <-sig: return GOT_SIGNAL } }) } hls.startPGoroutine(func(sig <-chan struct{}) int { <-sig if conn != nil { conn.Close() } return OK }) err = writeJson(OBJ{ "type": "startWatching", "data": OBJ{ "stream": OBJ{ "quality": hls.quality, //"abr", // high "protocol": "hls", "latency": "high", }, "room": OBJ{ "protocol": "webSocket", "commentable": true, }, "reconnect": true, }, }) if err != nil { if !hls.interrupted() { log.Println("websocket getpermit write:", err) } return NETWORK_ERROR } var playlistStarted bool var watchingStarted bool var watchinginterval int for !hls.interrupted() { select { case <-sig: return GOT_SIGNAL default: } var res interface{} err = conn.ReadJSON(&res) if err != nil { if (!hls.interrupted()) && (!hls.finish) { log.Println("websocket read:", err) } return NETWORK_ERROR } if hls.nicoDebug { fmt.Fprintf(os.Stderr, "%s:ReadJSON => %v\n", debug_Now(), res) } _type, ok := objs.FindString(res, "type") if !ok { fmt.Printf("type not found\n") continue } switch _type { //case "watch": //if cmd, ok := objs.FindString(res, "body", "command"); ok { //switch cmd { case "seat": if _arr, ok := objs.FindFloat64(res, "data", "keepIntervalSec"); ok { arr := []interface{}{_arr} for _, intf := range arr { if str, ok := intf.(float64); ok { num := int(str) if num > 0 { //hls.SetInterval(num) watchinginterval = num break } } } } if (!watchingStarted) && watchinginterval > 0 { watchingStarted = true hls.startPGoroutine(func(sig <-chan struct{}) int { for { select { case <-time.After(time.Duration(watchinginterval) * time.Second): err := writeJson(OBJ{ "type": "keepSeat", }) if err != nil { if !hls.interrupted() { log.Println("websocket watching:", err) } return NETWORK_ERROR } case <-sig: return GOT_SIGNAL } } }) } case "stream": if uri, ok := objs.FindString(res, "data", "uri"); ok { if (!playlistStarted) && uri != "" { playlistStarted = true hls.startPlaylist(uri) } } case "disconnect": // print params if _arr, ok := objs.FindString(res, "data", "reason"); ok { arr := []interface{}{0, _arr} fmt.Printf("%v\n", arr) if len(arr) >= 2 { if s, ok := arr[1].(string); ok { switch s { case "END_PROGRAM": return MAIN_END_PROGRAM case "SERVICE_TEMPORARILY_UNAVAILABLE", "INTERNAL_SERVERERROR": return MAIN_TEMPORARILY_ERROR case "TOO_MANY_CONNECTIONS": return MAIN_DISCONNECT case "TEMPORARILY_CROWDED": return MAIN_END_PROGRAM } } } } return MAIN_DISCONNECT case "room": // comment messageServerUri, ok := objs.FindString(res, "data", "messageServer", "uri") if !ok { break } threadId, ok := objs.FindString(res, "data", "threadId") if !ok { break } waybackkey, _ := objs.FindString(res, "data", "waybackkey") hls.startComment(messageServerUri, threadId, waybackkey) case "statistics": case "permit": case "serverTime": case "schedule": // nop //default: // fmt.Printf("%#v\n", res) // fmt.Printf("unknown command: %s\n", cmd) //} // end switch "command" //} // "watch" case "ping": err := writeJson(OBJ{ "type": "pong", }) if err != nil { if !hls.interrupted() { log.Println("websocket watching:", err) } return NETWORK_ERROR } case "error": code, ok := objs.FindString(res, "data", "code") if !ok { log.Printf("Unknown error: %#v\n", res) return ERROR_SHUTDOWN } // https://nicolive.cdn.nimg.jp/relive/front_assets/scripts/nicolib.4bb8b62b35.js switch code { case "INVALID_STREAM_QUALITY": // webSocket自体を再接続しないと、コメントサーバが取得できない switch hls.quality { case "abr": hls.quality = "high" return MAIN_INVALID_STREAM_QUALITY default: return ERROR_SHUTDOWN } //case // "INTERNAL_SERVERERROR", // "CONTENT_NOT_READY", // 終了後に出ることがある // "CONNECT_ERROR": // 終了後に出ることがある // return NETWORK_ERROR //case // "INVALID_BROADCAST_ID", // "BROADCAST_NOT_FOUND", // "NO_THREAD_AVAILABLE", // "NO_ROOM_AVAILABLE", // "NO_PERMISSION": // return ERROR_SHUTDOWN case "INVALID_MESSAGE": // 公式のTSで送られてきた。単純に無視する。 default: // log.Printf("Unknown error: %s\n%#v\n", code, res) // return ERROR_SHUTDOWN fmt.Printf("error code: %v\n", code) if hls.msgErrorSeqNo == hls.playlist.seqNo { hls.msgErrorCount++ } else { hls.msgErrorSeqNo = hls.playlist.seqNo hls.msgErrorCount = 1 } if hls.msgErrorCount >= 3 { return ERROR_SHUTDOWN } else { return NETWORK_ERROR } } default: log.Printf("Unknown type: %s\n%#v\n", _type, res) } // end switch "type" } // for ReadJSON return OK }) } func (hls *NicoHls) startMainV1() { return // old startMain } func (hls *NicoHls) serve(hlsPort int) { hls.startMGoroutine(func(sig <-chan struct{}) int { gin.SetMode(gin.ReleaseMode) gin.DefaultErrorWriter = ioutil.Discard gin.DefaultWriter = ioutil.Discard router := gin.Default() router.GET("", func(c *gin.Context) { seqno := hls.dbGetLastSeqNo() body := fmt.Sprintf( `#EXTM3U #EXT-X-VERSION:3 #EXT-X-TARGETDURATION:1 #EXT-X-MEDIA-SEQUENCE:%d #EXTINF:1.0, /ts/%d/test.ts #EXTINF:1.0, /ts/%d/test.ts #EXTINF:1.0, /ts/%d/test.ts `, seqno-2, seqno-2, seqno-1, seqno) c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) return }) router.GET("/ts/:idx/test.ts", func(c *gin.Context) { i, _ := strconv.Atoi(c.Param("idx")) b := hls.dbGetLastMedia(i) c.Data(http.StatusOK, "video/MP2T", b) return }) srv := &http.Server{ Addr: fmt.Sprintf("127.0.0.1:%d", hlsPort), Handler: router, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, MaxHeaderBytes: 1 << 20, } chLocal := make(chan struct{}) idleConnsClosed := make(chan struct{}) defer func() { close(chLocal) }() go func() { select { case <-chLocal: case <-sig: } if err := srv.Shutdown(context.Background()); err != nil { log.Printf("srv.Shutdown: %v\n", err) } close(idleConnsClosed) }() // クライアントはlocalhostでなく127.0.0.1で接続すること // localhostは遅いため if err := srv.ListenAndServe(); err != http.ErrServerClosed { log.Printf("srv.ListenAndServe: %v\n", err) } <-idleConnsClosed return OK }) } func (hls *NicoHls) Wait(testTimeout, hlsPort int) { hls.startInterrupt() defer hls.stopInterrupt() if testTimeout > 0 { hls.startMGoroutine(func(sig <-chan struct{}) int { select { case <-sig: return GOT_SIGNAL case <-time.After(time.Duration(testTimeout) * time.Second): hls.chInterrupt <- syscall.Signal(1000) return OK } }) } if hlsPort > 0 { hls.serve(hlsPort) } hls.startMain() for hls.working() { if hls.waitRestartMain() { continue } hls.stopPCGoroutines() hls.waitCGoroutines() } hls.stopAllGoroutines() hls.waitAllGoroutines() return } func postTsRsv0(opt options.Option) (err error) { if ma := regexp.MustCompile(`lv(\d+)`).FindStringSubmatch(opt.NicoLiveId); len(ma) > 0 { if err = postTsRsvBase(0, ma[1], opt.NicoSession); err != nil { return } err = postTsRsvBase(1, ma[1], opt.NicoSession) } return } func postTsRsv1(opt options.Option) (err error) { if ma := regexp.MustCompile(`lv(\d+)`).FindStringSubmatch(opt.NicoLiveId); len(ma) > 0 { err = postTsRsvBase(1, ma[1], opt.NicoSession) } return } func postTsRsvBase(num int, vid, session string) (err error) { var uri string if num == 0 { uri = fmt.Sprintf("https://live.nicovideo.jp/api/watchingreservation?mode=watch_num&vid=%s", vid) } else { uri = fmt.Sprintf("https://live.nicovideo.jp/api/watchingreservation?mode=confirm_watch_my&vid=%s", vid) } header := map[string]string{ "Cookie": "user_session=" + session, } dat0, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { if err == nil { err = neterr } return } var token string if ma := regexp.MustCompile( `TimeshiftActions\.(doRegister|confirmToWatch|moveWatch)\(['"].*?['"]\s*(?:,\s*['"](.+?)['"])`). FindStringSubmatch(dat0); len(ma) > 0 { if len(ma) > 2 { token = ma[2] } } else if strings.Contains(dat0, "視聴済み") { err = fmt.Errorf("postTsRsv: already watched") return } else { fmt.Printf("postTsRsv: token not found: >>>%s<<<\n", dat0) err = fmt.Errorf("postTsRsv: token not found") return } // "X-Requested-With": "XMLHttpRequest", // "Origin": "https://live.nicovideo.jp", // "Referer": fmt.Sprintf("https://live.nicovideo.jp/gate/%s", opt.NicoLiveId), // "X-Prototype-Version": "1.6.0.3", var vals url.Values if num == 0 { vals = url.Values{ "mode": []string{"overwrite"}, "vid": []string{vid}, "token": []string{token}, "rec_pos": []string{""}, "rec_engine": []string{""}, "rec_id": []string{""}, "_": []string{""}, } } else { vals = url.Values{ "accept": []string{"true"}, "mode": []string{"use"}, "vid": []string{vid}, "token": []string{token}, "_": []string{""}, } } dat1, _, _, err, neterr := postStringHeader("https://live.nicovideo.jp/api/watchingreservation", header, vals) if err != nil || neterr != nil { if err == nil { err = neterr } return } if (!strings.Contains(dat1, "status=\"ok\"")) && (!strings.Contains(dat1, "\"regist_finished\"")) { fmt.Printf("postTsRsv: status not ok: >>>%s<<<\n", dat1) err = fmt.Errorf("postTsRsv: status not ok") return } return } func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, tsRsv1 bool, err error) { header := map[string]string{} if opt.NicoSession != "" { header["Cookie"] = "user_session=" + opt.NicoSession } uri := fmt.Sprintf("https://live2.nicovideo.jp/watch/%s", opt.NicoLiveId) dat, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { if err == nil { err = neterr } return } // ログイン判定 if opt.NicoSession == "" { notLogin = true } else if ma := regexp.MustCompile(`login_status['"]*\s*[=:]\s*['"](.*?)['"]`).FindStringSubmatch(dat); len(ma) > 0 { switch string(ma[1]) { case "not_login": notLogin = true case "login": notLogin = false default: fmt.Printf("[FIXME] login_status = %s\n", ma[1]) } } else { notLogin = true } // 新配信 + nicocas if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &props); err != nil { return } return } else if strings.Contains(dat, "nicoliveplayer.swf") { // 旧Flashプレイヤー isFlash = true } else if regexp.MustCompile(`この番組は.{1,50}に終了`).MatchString(dat) { // タイムシフト予約ボタン if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.register`).FindStringSubmatch(dat); len(ma) > 0 { fmt.Printf("timeshift reservation required\n") tsRsv0 = true return } if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.confirm`).FindStringSubmatch(dat); len(ma) > 0 { fmt.Printf("timeshift reservation required\n") tsRsv1 = true return } } return } func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, dbName string, err error) { //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 32 //var props interface{} //var isFlash bool //var tsRsv bool props, isFlash, notLogin, tsRsv0, tsRsv1, err := getProps(opt) if err != nil { //fmt.Println(err) return } if notLogin { if opt.NicoLoginOnly { // 要ログイン return } else { // 非ログインでも録画可能なら再ログイン不要 notLogin = false } } // TS予約必要 if (tsRsv0 || tsRsv1) && opt.NicoForceResv { if tsRsv0 { err = postTsRsv0(opt) } else { err = postTsRsv1(opt) } if err == nil { reserved = true } return } if isFlash { fmt.Println("Flash page detected.") return } if false { objs.PrintAsJson(props) os.Exit(9) } proplist := map[string][]string{ // "broadcaster" // nicocas "cas-userName": []string{"broadcaster", "nickname"}, // ユーザ名 "cas-userPageUrl": []string{"broadcaster", "pageUrl"}, // "https://www.nicovideo.jp/user/\d+" // "community" "comId": []string{"community", "id"}, // "co\d+" // "program" "beginTime": []string{"program", "beginTime"}, // integer //"broadcastId": []string{"program", "broadcastId"}, // "\d+" "description": []string{"program", "description"}, // 放送説明 "endTime": []string{"program", "endTime"}, // integer "isFollowerOnly": []string{"program", "isFollowerOnly"}, // bool "isPrivate": []string{"program", "isPrivate"}, // bool "mediaServerType": []string{"program", "mediaServerType"}, // "DMC" "nicoliveProgramId": []string{"program", "nicoliveProgramId"}, // "lv\d+" "openTime": []string{"program", "openTime"}, // integer "providerType": []string{"program", "providerType"}, // "community" "status": []string{"program", "status"}, // "userName": []string{"program", "supplier", "name"}, // ユーザ名 "userPageUrl": []string{"program", "supplier", "pageUrl"}, // "https://www.nicovideo.jp/user/\d+" "title": []string{"program", "title"}, // title // "site" "nicocas": []string{"site", "nicocas"}, // "//webSocketUrl": []string{"site", "relive", "webSocketUrl"}, // "ws://..." "serverTime": []string{"site", "serverTime"}, // integer // "socialGroup" "socDescription": []string{"socialGroup", "description"}, // コミュ説明 "socId": []string{"socialGroup", "id"}, // "co\d+" or "ch\d+" "socLevel": []string{"socialGroup", "level"}, // integer "socName": []string{"socialGroup", "name"}, // community name "socType": []string{"socialGroup", "type"}, // "community" // "user" "accountType": []string{"user", "accountType"}, // "premium" "//myId": []string{"user", "id"}, // "\d+" "isLoggedIn": []string{"user", "isLoggedIn"}, // bool "//myNickname": []string{"user", "nickname"}, // string } kv := map[string]interface{}{} for k, a := range proplist { v, ok := objs.Find(props, a...) if ok { kv[k] = v if opt.NicoDebug { fmt.Println(k, v) fmt.Println("----------") } } } var nicocas bool if _, ok := kv["nicocas"]; ok { nicocas = true } if nicocas { fmt.Println("nicocas not supported.") return } else { for _, k := range []string{ "nicoliveProgramId", "//webSocketUrl", //"//myId", } { if _, ok := kv[k]; !ok { fmt.Printf("%v not found\n", k) return } } if opt.NicoFormat == "" { opt.NicoFormat = "?PID?-?UNAME?-?TITLE?" } hls, e := NewHls(opt, kv) if e != nil { err = e fmt.Println(err) return } defer hls.Close() hls.Wait(opt.NicoTestTimeout, opt.NicoHlsPort) dbName = hls.dbName playlistEnd = hls.finish done = true } /* pageUrl, _ := objs.FindString(props, "broadcaster", "pageUrl") if regexp.MustCompile(`\Ahttps?://cas\.nicovideo\.jp/.*?/.*`).MatchString(pageUrl) { // 実験放送 userId, ok := objs.FindString(props, "broadcaster", "id") if ! ok { fmt.Printf("userId not found") } nickname, ok := objs.FindString(props, "broadcaster", "nickname") if ! ok { fmt.Printf("nickname not found") } var isArchive bool switch status { case "ENDED": isArchive = true } } log4gui.Info(fmt.Sprintf("isLoggedIn: %v, user_id: %s, nickname: %s", isLoggedIn, user_id, nickname)) */ return } ================================================ FILE: src/niconico/nico_mem_db.go ================================================ package niconico import ( "fmt" "time" "os" "database/sql" ) func (hls *NicoHls) memdbOpen() (err error) { db, err := sql.Open("sqlite3", "file::memory:?mode=memory&cache=shared") if err != nil { return } hls.memdb = db err = hls.memdbCreate() if err != nil { hls.memdb.Close() } if hls.db != nil { rows, e := hls.db.Query(`SELECT * FROM (SELECT seqno, IFNULL(notfound, 0), IFNULL(size, 0) FROM media ORDER BY seqno DESC LIMIT 10) ORDER BY seqno`) if e != nil { err = e return } defer rows.Close() var found404 bool for rows.Next() { var seqno int var notfound bool var size int err = rows.Scan(&seqno, ¬found, &size) if err != nil { return } if notfound || size == 0 { hls.memdbSet404(seqno) found404 = true } else { hls.memdbSet200(seqno) } if (! found404) { hls.memdbSetStopBack(seqno) if hls.nicoDebug { fmt.Fprintf(os.Stderr, "memdbSetStopBack(%d)\n", seqno) } } } } return } func (hls *NicoHls) memdbCreate() (err error) { hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() _, err = hls.memdb.Exec(` CREATE TABLE IF NOT EXISTS media ( seqno INTEGER PRIMARY KEY NOT NULL UNIQUE, is200 INTEGER, is404 INTEGER, stopback INTEGER ) `) if err != nil { return } _, err = hls.memdb.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS media0 ON media(seqno); `) if err != nil { return } return } func (hls *NicoHls) memdbSetStopBack(seqno int) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbSetStopBack: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() _, err := hls.memdb.Exec(` INSERT OR IGNORE INTO media (seqno, stopback) VALUES (?, 1); UPDATE media SET stopback = 1 WHERE seqno=?; `, seqno, seqno) if err != nil { fmt.Println(err) } } func (hls *NicoHls) memdbGetStopBack(seqno int) (res bool) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbGetStopBack: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() hls.memdb.QueryRow("SELECT IFNULL(stopback, 0) FROM media WHERE seqno=?", seqno).Scan(&res) return } func (hls *NicoHls) memdbSet200(seqno int) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbSet200: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() hls.memdb.Exec(`INSERT OR REPLACE INTO media (seqno, is200) VALUES (?, 1)`, seqno) } func (hls *NicoHls) memdbSet404(seqno int) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbSet404: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() hls.memdb.Exec(`INSERT OR REPLACE INTO media (seqno, is404) VALUES (?, 1)`, seqno) } func (hls *NicoHls) memdbCheck200(seqno int) (res bool) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbCheck200: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() hls.memdb.QueryRow("SELECT IFNULL(is200, 0) FROM media WHERE seqno=?", seqno).Scan(&res) return } func (hls *NicoHls) memdbDelete(seqno int) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbDelete: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() min := seqno - 100 hls.memdb.Exec(`DELETE FROM media WHERE seqno < ?`, min) } func (hls *NicoHls) memdbCount() (res int) { if hls.nicoDebug { start := time.Now().UnixNano() defer func() { t := (time.Now().UnixNano() - start) / (1000 * 1000) if t > 100 { fmt.Fprintf(os.Stderr, "%s:[WARN][MEMDB]memdbCount: %d(ms)\n", debug_Now(), t) } }() } hls.memdbMtx.Lock() defer hls.memdbMtx.Unlock() hls.memdb.QueryRow("SELECT COUNT(seqno) FROM media").Scan(&res) return } ================================================ FILE: src/niconico/nico_rtmp.go ================================================ package niconico import ( "encoding/xml" "fmt" "io/ioutil" "log" "net/url" "regexp" "strings" "sync" "time" "github.com/himananiito/livedl/amf" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/options" "github.com/himananiito/livedl/rtmps" ) type Content struct { Id string `xml:"id,attr"` Text string `xml:",chardata"` } type Tickets struct { Name string `xml:"name,attr"` Text string `xml:",chardata"` } type Status struct { Title string `xml:"stream>title"` CommunityId string `xml:"stream>default_community"` Id string `xml:"stream>id"` Provider string `xml:"stream>provider_type"` IsArchive bool `xml:"stream>archive"` IsArchivePlayerServer bool `xml:"stream>is_archiveplayserver"` Ques []string `xml:"stream>quesheet>que"` Contents []Content `xml:"stream>contents_list>contents"` IsPremium bool `xml:"user>is_premium"` Url string `xml:"rtmp>url"` Ticket string `xml:"rtmp>ticket"` Tickets []Tickets `xml:"tickets>stream"` ErrorCode string `xml:"error>code"` streams []Stream chStream chan struct{} wg *sync.WaitGroup } type Stream struct { originUrl string streamName string originTicket string } func (status *Status) quesheet() { stream := make(map[string][]Stream) playType := make(map[string]string) // timeshift; tag re_pub := regexp.MustCompile(`\A/publish\s+(\S+)\s+(?:(\S+?),)?(\S+?)(?:\?(\S+))?\z`) re_play := regexp.MustCompile(`\A/play\s+(\S+)\s+(\S+)\z`) for _, q := range status.Ques { // /publish lv* /content/*/lv*_*_1_*.f4v if ma := re_pub.FindStringSubmatch(q); len(ma) >= 5 { stream[ma[1]] = append(stream[ma[1]], Stream{ originUrl: ma[2], streamName: ma[3], originTicket: ma[4], }) // /play ... } else if ma := re_play.FindStringSubmatch(q); len(ma) > 0 { // /play case:sp:rtmp:lv*_s_lv*,mobile:rtmp:lv*_s_lv*_sub1,premium:rtmp:lv*_s_lv*_sub1,default:rtmp:lv*_s_lv* main if strings.HasPrefix(ma[1], "case:") { s0 := ma[1] s0 = strings.TrimPrefix(s0, "case:") cases := strings.Split(s0, ",") // sp:rtmp:lv*_s_lv* re := regexp.MustCompile(`\A(\S+?):rtmp:(\S+?)\z`) for _, c := range cases { if ma := re.FindStringSubmatch(c); len(ma) > 0 { playType[ma[1]] = ma[2] } } // /play rtmp:lv* main } else { re := regexp.MustCompile(`\Artmp:(\S+?)\z`) if ma := re.FindStringSubmatch(ma[1]); len(ma) > 0 { playType["default"] = ma[1] } } } } pt, ok := playType["premium"] if ok && status.IsPremium { s, ok := stream[pt] if ok { status.streams = s } } else { pt, ok := playType["default"] if ok { s, ok := stream[pt] if ok { status.streams = s } } } } func (status *Status) initStreams() { if len(status.streams) > 0 { return } //if status.isOfficialLive() { status.contentsOfficialLive() //} else if status.isLive() { status.contentsNonOfficialLive() //} else { status.quesheet() //} return } func (status *Status) getFileName(index int) (name string) { if len(status.streams) == 1 { //name = fmt.Sprintf("%s.flv", status.Id) name = fmt.Sprintf("%s-%s-%s.flv", status.Id, status.CommunityId, status.Title) } else if len(status.streams) > 1 { //name = fmt.Sprintf("%s-%d.flv", status.Id, 1 + index) name = fmt.Sprintf("%s-%s-%s#%d.flv", status.Id, status.CommunityId, status.Title, 1+index) } else { log.Fatalf("No stream") } name = files.ReplaceForbidden(name) return } func (status *Status) contentsNonOfficialLive() { re := regexp.MustCompile(`\A(?:rtmp:)?(rtmp\w*://\S+?)(?:,(\S+?)(?:\?(\S+))?)?\z`) // Live (not timeshift); tag for _, c := range status.Contents { if ma := re.FindStringSubmatch(c.Text); len(ma) > 0 { status.streams = append(status.streams, Stream{ originUrl: ma[1], streamName: ma[2], originTicket: ma[3], }) } } } func (status *Status) contentsOfficialLive() { tickets := make(map[string]string) for _, t := range status.Tickets { tickets[t.Name] = t.Text } for _, c := range status.Contents { if strings.HasPrefix(c.Text, "case:") { c.Text = strings.TrimPrefix(c.Text, "case:") for _, c := range strings.Split(c.Text, ",") { c, e := url.PathUnescape(c) if e != nil { fmt.Printf("%v\n", e) } re := regexp.MustCompile(`\A(\S+?):(?:limelight:|akamai:)?(\S+),(\S+)\z`) if ma := re.FindStringSubmatch(c); len(ma) > 0 { fmt.Printf("\n%#v\n", ma) switch ma[1] { default: fmt.Printf("unknown contents case %#v\n", ma[1]) case "mobile": case "middle": case "default": status.Url = ma[2] t, ok := tickets[ma[3]] if !ok { fmt.Printf("not found %s\n", ma[3]) } fmt.Printf("%s\n", t) status.streams = append(status.streams, Stream{ streamName: ma[3], originTicket: t, }) } } } } } } func (status *Status) relayStreamName(i, offset int) (s string) { s = regexp.MustCompile(`[^/\\]+\z`).FindString(status.streams[i].streamName) if offset >= 0 { s += fmt.Sprintf("_%d", offset) } return } func (status *Status) streamName(i, offset int) (name string, err error) { if status.isOfficialLive() { if i >= len(status.streams) { err = fmt.Errorf("(status *Status) streamName(i int): Out of index: %d\n", i) return } name = status.streams[i].streamName if status.streams[i].originTicket != "" { name += "?" + status.streams[i].originTicket } return } else if status.isOfficialTs() { name = status.streams[i].streamName name = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(name, "") if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(name) { name = "mp4:" + name } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(name) { name = "raw:" + name } } else { name = status.relayStreamName(i, offset) } return } func (status *Status) tcUrl() (url string, err error) { if status.Url != "" { url = status.Url return } else { status.contentsOfficialLive() } if status.Url != "" { url = status.Url return } err = fmt.Errorf("tcUrl not found") return } func (status *Status) isTs() bool { return status.IsArchive } func (status *Status) isLive() bool { return (!status.IsArchive) } func (status *Status) isOfficialLive() bool { return (status.Provider == "official") && (!status.IsArchive) } func (status *Status) isOfficialTs() bool { if status.IsArchive { switch status.Provider { case "official": return true case "channel": return status.IsArchivePlayerServer } } return false } func (st Stream) relayStreamName(offset int) (s string) { s = regexp.MustCompile(`[^/\\]+\z`).FindString(st.streamName) if offset >= 0 { s += fmt.Sprintf("_%d", offset) } return } func (st Stream) noticeStreamName(offset int) (s string) { s = st.streamName s = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(s, "") if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(s) { s = "mp4:" + s } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(s) { s = "raw:" + s } if st.originTicket != "" { s += "?" + st.originTicket } return } func (status *Status) recStream(index int, opt options.Option) (err error) { defer func() { <-status.chStream status.wg.Done() }() stream := status.streams[index] tcUrl, err := status.tcUrl() if err != nil { return } rtmp, err := rtmps.NewRtmp( // tcUrl tcUrl, // swfUrl "http://live.nicovideo.jp/nicoliveplayer.swf?180116154229", // pageUrl "http://live.nicovideo.jp/watch/"+status.Id, // option status.Ticket, ) if err != nil { return } defer rtmp.Close() fileName, err := files.GetFileNameNext(status.getFileName(index)) if err != nil { return } rtmp.SetFlvName(fileName) tryRecord := func() (incomplete bool, err error) { if err = rtmp.Connect(); err != nil { return } // default: 2500000 //if err = rtmp.SetPeerBandwidth(100*1000*1000, 0); err != nil { if err = rtmp.SetPeerBandwidth(2500000, 0); err != nil { fmt.Printf("SetPeerBandwidth: %v\n", err) return } if err = rtmp.WindowAckSize(2500000); err != nil { fmt.Printf("WindowAckSize: %v\n", err) return } if err = rtmp.CreateStream(); err != nil { fmt.Printf("CreateStream %v\n", err) return } if err = rtmp.SetBufferLength(0, 2000); err != nil { fmt.Printf("SetBufferLength: %v\n", err) return } var offset int if status.IsArchive { offset = 0 } else { offset = -2 } if status.isOfficialTs() { for i := 0; true; i++ { if i > 30 { err = fmt.Errorf("sendFileRequest: No response") return } data, e := rtmp.Command( "sendFileRequest", []interface{}{ nil, amf.SwitchToAmf3(), []string{ stream.streamName, }, }) if e != nil { err = e return } var resCnt int switch data.(type) { case map[string]interface{}: resCnt = len(data.(map[string]interface{})) case map[int]interface{}: resCnt = len(data.(map[int]interface{})) case []interface{}: resCnt = len(data.([]interface{})) case []string: resCnt = len(data.([]string)) } if resCnt > 0 { break } time.Sleep(10 * time.Second) } } else if !status.isOfficialLive() { // /publishの第二引数 // streamName(param1:String) // 「,」で区切る // ._originUrl, streamName(playStreamName) // streamName に、「?」がついてるなら originTickt となる // streamName の.flvは削除する // streamNameが/\.(f4v|mp4)$/iなら、頭にmp4:をつける // /\.raw$/iなら、raw:をつける。 // relayStreamName: streamNameの頭からスラッシュまでを削除したもの _, err = rtmp.Command( "nlPlayNotice", []interface{}{ nil, // _connection.request.originUrl stream.originUrl, // this._connection.request.playStreamRequest // originticket あるなら // playStreamName ? this._originTicket // 無いなら playStreamName stream.noticeStreamName(offset), // var _loc1_:String = this._relayStreamName; // if(this._offset != -2) // { // _loc1_ = _loc1_ + ("_" + this.offset); // } // user nama: String 'lvxxxxxxxxx' // user kako: lvxxxxxxxxx_xxxxxxxxxxxx_1_xxxxxx.f4v_0 stream.relayStreamName(offset), // seek offset // user nama: -2, user kako: 0 offset, }) if err != nil { fmt.Printf("nlPlayNotice %v\n", err) return } } if err = rtmp.SetBufferLength(1, 3600*1000); err != nil { fmt.Printf("SetBufferLength: %v\n", err) return } // No return rtmp.SetFixAggrTimestamp(true) // user kako: lv*********_************_*_******.f4v_0 // official or channel ts: mp4:/content/********/lv*********_************_*_******.f4v //if err = rtmp.Play(status.origin.playStreamName(status.isTsOfficial(), offset)); err != nil { streamName, err := status.streamName(index, offset) if err != nil { return } if status.isOfficialTs() { ts := rtmp.GetTimestamp() if ts > 1000 { err = rtmp.PlayTime(streamName, ts-1000) } else { err = rtmp.PlayTime(streamName, -5000) } } else if status.isTs() { rtmp.SetFlush(true) err = rtmp.PlayTime(streamName, -5000) } else { err = rtmp.Play(streamName) } if err != nil { fmt.Printf("Play: %v\n", err) return } // Non-recordedなタイムシフトでseekしても、timestampが変わるだけで // 最初からの再生となってしまうのでやらないこと // 公式のタイムシフトでSeekしてもタイムスタンプがおかしい if opt.NicoTestTimeout > 0 { // test mode _, incomplete, err = rtmp.WaitTest(opt.NicoTestTimeout) } else { // normal mode _, incomplete, err = rtmp.Wait() } return } // end func //ticketTime := time.Now().Unix() //rtmp.SetNoSeek(false) for i := 0; i < 10; i++ { incomplete, e := tryRecord() if e != nil { err = e fmt.Printf("%v\n", e) return } else if incomplete && status.isOfficialTs() { fmt.Println("incomplete") time.Sleep(3 * time.Second) // update ticket if true { //if time.Now().Unix() > ticketTime + 60 { //ticketTime = time.Now().Unix() if ticket, e := getTicket(opt); e != nil { err = e return } else { rtmp.SetConnectOpt(ticket) } //} } continue } break } fmt.Printf("done\n") return } func (status *Status) recAllStreams(opt options.Option) (err error) { status.initStreams() var MaxConn int if opt.NicoRtmpMaxConn == 0 { if status.isOfficialTs() { MaxConn = 1 } else { MaxConn = 4 } } else if opt.NicoRtmpMaxConn < 0 { MaxConn = 1 } else { MaxConn = opt.NicoRtmpMaxConn } status.wg = &sync.WaitGroup{} status.chStream = make(chan struct{}, MaxConn) ticketTime := time.Now().Unix() for index, _ := range status.streams { if opt.NicoRtmpIndex != nil { if tes, ok := opt.NicoRtmpIndex[index]; !ok || !tes { continue } } // blocks here status.chStream <- struct{}{} status.wg.Add(1) go status.recStream(index, opt) now := time.Now().Unix() if now > ticketTime+60 { ticketTime = now if ticket, e := getTicket(opt); e != nil { err = e return } else { status.Ticket = ticket } } } status.wg.Wait() return } func getTicket(opt options.Option) (ticket string, err error) { status, notLogin, err := getStatus(opt) if err != nil { return } if status.Ticket != "" { ticket = status.Ticket } else { if notLogin { err = fmt.Errorf("notLogin") } else { err = fmt.Errorf("Ticket not found") } } return } func getStatus(opt options.Option) (status *Status, notLogin bool, err error) { var uri string // experimental if opt.NicoStatusHTTPS { uri = fmt.Sprintf("https://ow.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) } else { uri = fmt.Sprintf("http://watch.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) } header := make(map[string]string, 4) if opt.NicoSession != "" { header["Cookie"] = "user_session=" + opt.NicoSession } // experimental //if opt.NicoStatusHTTPS { // req.Header.Set("User-Agent", "Niconico/1.0 (Unix; U; iPhone OS 10.3.3; ja-jp; nicoiphone; iPhone5,2) Version/6.65") //} resp, err, neterr := httpbase.Get(uri, header) if err != nil { return } if neterr != nil { err = neterr return } defer resp.Body.Close() dat, _ := ioutil.ReadAll(resp.Body) status = &Status{} err = xml.Unmarshal(dat, status) if err != nil { //fmt.Println(string(dat)) fmt.Printf("error: %v", err) return } switch status.ErrorCode { case "": case "notlogin": notLogin = true default: err = fmt.Errorf("Error code: %s\n", status.ErrorCode) return } return } func NicoRecRtmp(opt options.Option) (notLogin bool, err error) { status, notLogin, err := getStatus(opt) if err != nil { return } if notLogin { return } status.recAllStreams(opt) return } ================================================ FILE: src/objs/objs.go ================================================ package objs import ( "fmt" "encoding/json" ) func PrintAsJson(data interface{}) { json, err := json.MarshalIndent(data, "", " ") if err != nil { return } fmt.Println(string(json)) } func Find(intf interface{}, keylist... string) (res interface{}, ok bool) { res = intf if len(keylist) == 0 { ok = true return } for i, k := range keylist { var test bool //var obj map[string]interface{} switch res.(type) { case map[string]interface{}: res, test = res.(map[string]interface{})[k] if (! test) { ok = false return } case []interface{}: for _, o := range res.([]interface{}) { _res, _ok := Find(o, keylist[i:]...) if _ok { res = _res ok = _ok return } } } } ok = true return } func FindFloat64(intf interface{}, keylist... string) (res float64, ok bool) { val, ok := Find(intf, keylist...) if !ok { return } res, ok = val.(float64) return } func FindString(intf interface{}, keylist... string) (res string, ok bool) { val, ok := Find(intf, keylist...) if !ok { return } res, ok = val.(string) return } func FindBool(intf interface{}, keylist... string) (res bool, ok bool) { val, ok := Find(intf, keylist...) if !ok { return } res, ok = val.(bool) return } func FindArray(intf interface{}, keylist... string) (res []interface{}, ok bool) { val, ok := Find(intf, keylist...) if !ok { return } res, ok = val.([]interface{}) return } ================================================ FILE: src/options/options.go ================================================ package options import ( "database/sql" "fmt" "io/ioutil" "log" "os" "path/filepath" "regexp" "strconv" "strings" "github.com/himananiito/livedl/buildno" "github.com/himananiito/livedl/cryptoconf" "github.com/himananiito/livedl/files" "golang.org/x/crypto/sha3" ) var DefaultTcasRetryTimeoutMinute = 5 // TcasRetryTimeoutMinute var DefaultTcasRetryInterval = 60 // TcasRetryInterval type Option struct { Command string NicoLiveId string NicoStatusHTTPS bool NicoSession string NicoLoginAlias string NicoRtmpMaxConn int NicoRtmpOnly bool NicoRtmpIndex map[int]bool NicoHlsOnly bool NicoLoginOnly bool NicoTestTimeout int TcasId string TcasRetry bool TcasRetryTimeoutMinute int // 再試行を終了する時間(初回終了または録画終了からの時間「分」) TcasRetryInterval int // 再試行を行うまでの待ち時間 YoutubeId string ConfFile string // deprecated ConfPass string // deprecated ZipFile string DBFile string NicoHlsPort int NicoLimitBw int NicoTsStart float64 NicoFormat string NicoFastTs bool NicoUltraFastTs bool NicoAutoConvert bool NicoAutoDeleteDBMode int // 0:削除しない 1:mp4が分割されなかったら削除 2:分割されても削除 NicoDebug bool // デバッグ情報の記録 ConvExt string ExtractChunks bool NicoForceResv bool // 終了番組の上書きタイムシフト予約 YtNoStreamlink bool YtNoYoutubeDl bool NicoSkipHb bool // コメント出力時に/hbコマンドを出さない HttpRootCA string HttpSkipVerify bool HttpProxy string NoChdir bool } func getCmd() (cmd string) { cmd = filepath.Base(os.Args[0]) ext := filepath.Ext(cmd) cmd = strings.TrimSuffix(cmd, ext) return } func versionStr() string { cmd := filepath.Base(os.Args[0]) ext := filepath.Ext(cmd) cmd = strings.TrimSuffix(cmd, ext) return fmt.Sprintf(`%s (%s)`, cmd, buildno.GetBuildNo()) } func version() { fmt.Println(versionStr()) os.Exit(0) } func Help(verbose ...bool) { cmd := filepath.Base(os.Args[0]) ext := filepath.Ext(cmd) cmd = strings.TrimSuffix(cmd, ext) format := `%s (%s) Usage: %s [COMMAND] options... [--] FILE COMMAND: -nico ニコニコ生放送の録画 -tcas ツイキャスの録画 -yt YouTube Liveの録画 -d2m 録画済みのdb(.sqlite3)をmp4に変換する(-db-to-mp4) オプション/option: -h ヘルプを表示 -vh 全てのオプションを表示 -v バージョンを表示 -no-chdir 起動する時chdirしない -- 後にオプションが無いことを指定 ニコニコ生放送録画用オプション: -nico-login , (+) ニコニコのIDとパスワードを指定する -nico-session Cookie[user_session]を指定する -nico-login-only=on (+) 必ずログイン状態で録画する -nico-login-only=off (+) 非ログインでも録画可能とする(デフォルト) -nico-hls-only 録画時にHLSのみを試す -nico-hls-only=on (+) 上記を有効に設定 -nico-hls-only=off (+) 上記を無効に設定(デフォルト) -nico-rtmp-only 録画時にRTMPのみを試す -nico-rtmp-only=on (+) 上記を有効に設定 -nico-rtmp-only=off (+) 上記を無効に設定(デフォルト) -nico-rtmp-max-conn RTMPの同時接続数を設定 -nico-rtmp-index [,] RTMP録画を行うメディアファイルの番号を指定 -nico-hls-port [実験的] ローカルなHLSサーバのポート番号 -nico-limit-bw (+) HLSのBANDWIDTHの上限値を指定する。0=制限なし -nico-format "FORMAT" (+) 保存時のファイル名を指定する -nico-fast-ts 倍速タイムシフト録画を行う(新配信タイムシフト) -nico-fast-ts=on (+) 上記を有効に設定 -nico-fast-ts=off (+) 上記を無効に設定(デフォルト) -nico-auto-convert=on (+) 録画終了後自動的にMP4に変換するように設定 -nico-auto-convert=off (+) 上記を無効に設定 -nico-auto-delete-mode 0 (+) 自動変換後にデータベースファイルを削除しないように設定(デフォルト) -nico-auto-delete-mode 1 (+) 自動変換でMP4が分割されなかった場合のみ削除するように設定 -nico-auto-delete-mode 2 (+) 自動変換でMP4が分割されても削除するように設定 -nico-force-reservation=on (+) 視聴にタイムシフト予約が必要な場合に自動的に上書きする -nico-force-reservation=off (+) 自動的にタイムシフト予約しない(デフォルト) -nico-skip-hb=on (+) コメント書き出し時に/hbコマンドを出さない -nico-skip-hb=off (+) コメント書き出し時に/hbコマンドも出す(デフォルト) -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する ツイキャス録画用オプション: -tcas-retry=on (+) 録画終了後に再試行を行う -tcas-retry=off (+) 録画終了後に再試行を行わない -tcas-retry-timeout (+) 再試行を開始してから終了するまでの時間（分) -1で無限ループ。デフォルト: 5分 -tcas-retry-interval (+) 再試行を行う間隔（秒）デフォルト: 60秒 Youtube live録画用オプション: -yt-api-key (+) YouTube Data API v3 keyを設定する(未使用) -yt-no-streamlink=on (+) Streamlinkを使用しない -yt-no-streamlink=off (+) Streamlinkを使用する(デフォルト) -yt-no-youtube-dl=on (+) youtube-dlを使用しない -yt-no-youtube-dl=off (+) youtube-dlを使用する(デフォルト) 変換オプション: -extract-chunks=off (+) -d2mで動画ファイルに書き出す(デフォルト) -extract-chunks=on (+) [上級者向] 各々のフラグメントを書き出す(大量のファイルが生成される) -conv-ext=mp4 (+) -d2mで出力の拡張子を.mp4とする(デフォルト) -conv-ext=ts (+) -d2mで出力の拡張子を.tsとする HTTP関連 -http-skip-verify=on (+) TLS証明書の認証をスキップする (32bit版対策) -http-skip-verify=off (+) TLS証明書の認証をスキップしない (デフォルト) (+)のついたオプションは、次回も同じ設定が使用されることを示す。 FILE: ニコニコ生放送/nicolive: http://live2.nicovideo.jp/watch/lvXXXXXXXXX lvXXXXXXXXX ツイキャス/twitcasting: https://twitcasting.tv/XXXXX ` fmt.Printf(format, cmd, buildno.GetBuildNo(), cmd) for _, b := range verbose { if b { fmt.Print(` 旧オプション: -conf-pass [廃止] 設定ファイルのパスワード -z2m 録画済みのzipをmp4に変換する(-zip-to-mp4) -nico-status-https - デバッグ用オプション: -nico-test-run ニコ生テストラン -nico-test-timeout ニコ生テストランでの各放送のタイムアウト -nico-test-format フォーマット、保存しない -nico-ufast-ts TS保存にウェイトを入れない -nico-debug デバッグ用ログ出力する HTTP関連 -http-root-ca ルート証明書ファイルを指定(pem/der) -http-skip-verify TLS証明書の認証をスキップする -http-proxy [警告] proxyを設定する [警告] 情報流出に注意。信頼できるproxy serverのみに使用すること。 `) break } } os.Exit(0) } func dbConfSet(db *sql.DB, k string, v interface{}) { query := `INSERT OR REPLACE INTO conf (k,v) VALUES (?,?)` if _, err := db.Exec(query, k, v); err != nil { log.Println(err) os.Exit(1) } } func SetNicoLogin(hash, user, pass string) (err error) { db, err := dbAccountOpen() if err != nil { if db != nil { db.Close() } return } defer db.Close() _, err = db.Exec(` INSERT OR IGNORE INTO niconico (alias, user, pass) VALUES(?, ?, ?); UPDATE niconico SET user = ?, pass = ? WHERE alias = ? `, hash, user, pass, user, pass, hash) if err != nil { fmt.Println(err) return } fmt.Printf("niconico account saved.\n") return } func SetNicoSession(hash, session string) (err error) { db, err := dbAccountOpen() if err != nil { if db != nil { db.Close() } return } defer db.Close() _, err = db.Exec(` INSERT OR IGNORE INTO niconico (alias, session) VALUES(?, ?); UPDATE niconico SET session = ? WHERE alias = ? `, hash, session, session, hash) if err != nil { fmt.Println(err) return } return } func LoadNicoAccount(alias string) (user, pass, session string, err error) { db, err := dbAccountOpen() if err != nil { if db != nil { db.Close() } return } defer db.Close() db.QueryRow(`SELECT user, pass, IFNULL(session, "") FROM niconico WHERE alias = ?`, alias).Scan(&user, &pass, &session) return } func SetYoutubeApiKey(key string) (err error) { db, err := dbAccountOpen() if err != nil { if db != nil { db.Close() } return } defer db.Close() _, err = db.Exec(` INSERT OR IGNORE INTO youtubeapikey (id, key) VALUES(1, ?); UPDATE youtubeapikey SET key = ? WHERE id = 1 `, key, key) if err != nil { fmt.Println(err) return } fmt.Printf("Youtube API KEY saved.\n") return } func LoadYoutubeApiKey() (key string, err error) { db, err := dbAccountOpen() if err != nil { if db != nil { db.Close() } return } defer db.Close() db.QueryRow(`SELECT IFNULL(key, "") FROM youtubeapikey WHERE id = 1`).Scan(&key) if key == "" { err = fmt.Errorf("apikey not found") } return } func dbAccountOpen() (db *sql.DB, err error) { base := func() string { if b := os.Getenv("LIVEDL_DIR"); b != "" { return b } if b := os.Getenv("APPDATA"); b != "" { return fmt.Sprintf("%s/livedl", b) } if b := os.Getenv("HOME"); b != "" { return fmt.Sprintf("%s/.livedl", b) } return "" }() if base == "" { log.Fatalln("basedir for account not defined") } name := fmt.Sprintf("%s/account.db", base) files.MkdirByFileName(name) db, err = sql.Open("sqlite3", name) if err != nil { log.Println(err) return } // niconico _, err = db.Exec(` CREATE TABLE IF NOT EXISTS niconico ( alias TEXT PRIMARY KEY NOT NULL UNIQUE, user TEXT NOT NULL, pass TEXT NOT NULL, session TEXT ) `) if err != nil { return } _, err = db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS niconico0 ON niconico(alias); CREATE UNIQUE INDEX IF NOT EXISTS niconico1 ON niconico(user); `) if err != nil { return } // youtube API key _, err = db.Exec(` CREATE TABLE IF NOT EXISTS youtubeapikey ( id PRIMARY KEY NOT NULL UNIQUE, key TEXT ) `) if err != nil { return } _, err = db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS youtubeapikey0 ON youtubeapikey(id); `) if err != nil { return } return } func dbOpen() (db *sql.DB, err error) { db, err = sql.Open("sqlite3", "conf.db") if err != nil { return } _, err = db.Exec(` CREATE TABLE IF NOT EXISTS conf ( k TEXT PRIMARY KEY NOT NULL UNIQUE, v BLOB ) `) if err != nil { return } _, err = db.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS conf0 ON conf(k); `) if err != nil { return } return } func ParseArgs() (opt Option) { //dbAccountOpen() db, err := dbOpen() if err != nil { log.Println(err) os.Exit(1) } defer db.Close() err = db.QueryRow(` SELECT IFNULL((SELECT v FROM conf WHERE k == "NicoFormat"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoLimitBw"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginOnly"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoHlsOnly"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoRtmpOnly"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoFastTs"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginAlias"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoAutoConvert"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoAutoDeleteDBMode"), 0), IFNULL((SELECT v FROM conf WHERE k == "TcasRetry"), 0), IFNULL((SELECT v FROM conf WHERE k == "TcasRetryTimeoutMinute"), 0), IFNULL((SELECT v FROM conf WHERE k == "TcasRetryInterval"), 0), IFNULL((SELECT v FROM conf WHERE k == "ConvExt"), ""), IFNULL((SELECT v FROM conf WHERE k == "ExtractChunks"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoForceResv"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoSkipHb"), 0), IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0); `).Scan( &opt.NicoFormat, &opt.NicoLimitBw, &opt.NicoLoginOnly, &opt.NicoHlsOnly, &opt.NicoRtmpOnly, &opt.NicoFastTs, &opt.NicoLoginAlias, &opt.NicoAutoConvert, &opt.NicoAutoDeleteDBMode, &opt.TcasRetry, &opt.TcasRetryTimeoutMinute, &opt.TcasRetryInterval, &opt.ConvExt, &opt.ExtractChunks, &opt.NicoForceResv, &opt.YtNoStreamlink, &opt.YtNoYoutubeDl, &opt.NicoSkipHb, &opt.HttpSkipVerify, ) if err != nil { log.Println(err) os.Exit(1) } args := os.Args[1:] var match []string type Parser struct { re *regexp.Regexp cb func() error } nextArg := func() (str string, err error) { if len(args) <= 0 { if len(match[0]) > 0 { err = fmt.Errorf("%v: value required", match[0]) } else { err = fmt.Errorf("value required") } } else { str = args[0] args = args[1:] } return } parseList := []Parser{ Parser{regexp.MustCompile(`\A(?i)(?:--?|/)(?:\?|h|help)\z`), func() error { Help() return nil }}, Parser{regexp.MustCompile(`\A(?i)(?:--?|/)v(?:\?|h|help)\z`), func() error { Help(true) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?(?:v|version)\z`), func() error { version() return nil }}, Parser{regexp.MustCompile(`\A(https?://(?:[^/]*@)?(?:[^/]*\.)*nicovideo\.jp(?::[^/]*)?/(?:[^/]*?/)*)?(lv\d+)(?:\?.*)?\z`), func() error { switch opt.Command { default: fmt.Printf("Use \"--\" option for FILE for %s\n", opt.Command) Help() case "", "NICOLIVE": opt.NicoLiveId = match[2] opt.Command = "NICOLIVE" case "NICOLIVE_TEST": opt.NicoLiveId = match[2] } return nil }}, Parser{regexp.MustCompile(`\A--?conf-?pass\z`), func() (err error) { str, err := nextArg() if err != nil { return } opt.ConfPass = str return }}, Parser{regexp.MustCompile(`\Ahttps?://twitcasting\.tv/([^/]+)(?:/.*)?\z`), func() error { opt.TcasId = match[1] opt.Command = "TWITCAS" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?tcas-?retry(?:=(on|off))\z`), func() error { if strings.EqualFold(match[1], "on") { opt.TcasRetry = true } else if strings.EqualFold(match[1], "off") { opt.TcasRetry = false } dbConfSet(db, "TcasRetry", opt.TcasRetry) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?tcas-?retry-?timeout(?:-?minutes?)?\z`), func() error { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--tcas-retry-timeout: Not a number: %s\n", s) } opt.TcasRetryTimeoutMinute = num dbConfSet(db, "TcasRetryTimeoutMinute", opt.TcasRetryTimeoutMinute) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?tcas-?retry-?interval\z`), func() error { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--tcas-retry-interval: Not a number: %s\n", s) } if num <= 0 { return fmt.Errorf("--tcas-retry-interval: Invalid: %d: greater than 1\n", num) } opt.TcasRetryInterval = num dbConfSet(db, "TcasRetryInterval", opt.TcasRetryInterval) return nil }}, Parser{regexp.MustCompile(`\Ahttps?://(?:[^/]*\.)*youtube\.com/(?:.*\W)?v=([\w-]+)(?:[^\w-].*)?\z`), func() error { opt.YoutubeId = match[1] opt.Command = "YOUTUBE" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico\z`), func() error { opt.Command = "NICOLIVE" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?test-?run\z`), func() error { opt.Command = "NICOLIVE_TEST" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?test-?timeout\z`), func() error { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-test-timeout: Not a number: %s\n", s) } if num <= 0 { return fmt.Errorf("--nico-test-timeout: Invalid: %d: must be greater than or equal to 1\n", num) } opt.NicoTestTimeout = num return nil }}, Parser{regexp.MustCompile(`\A(?i)--?tcas\z`), func() error { opt.Command = "TWITCAS" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?(?:yt|youtube|youtube-live)\z`), func() error { opt.Command = "YOUTUBE" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?(?:z|zip)-?(?:2|to)-?(?:m|mp4)\z`), func() error { opt.Command = "ZIP2MP4" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?(?:d|db|sqlite3?)-?(?:2|to)-?(?:m|mp4)\z`), func() error { opt.Command = "DB2MP4" return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?login-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoLoginOnly = true dbConfSet(db, "NicoLoginOnly", opt.NicoLoginOnly) } else if strings.EqualFold(match[1], "off") { opt.NicoLoginOnly = false dbConfSet(db, "NicoLoginOnly", opt.NicoLoginOnly) } else { opt.NicoLoginOnly = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?hls-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoHlsOnly = true dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) } else if strings.EqualFold(match[1], "off") { opt.NicoHlsOnly = false dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) } else { opt.NicoHlsOnly = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoRtmpOnly = true dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) } else if strings.EqualFold(match[1], "off") { opt.NicoRtmpOnly = false dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) } else { opt.NicoRtmpOnly = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?fast-?ts(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoFastTs = true dbConfSet(db, "NicoFastTs", opt.NicoFastTs) } else if strings.EqualFold(match[1], "off") { opt.NicoFastTs = false dbConfSet(db, "NicoFastTs", opt.NicoFastTs) } else { opt.NicoFastTs = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?auto-?convert(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoAutoConvert = true dbConfSet(db, "NicoAutoConvert", opt.NicoAutoConvert) } else if strings.EqualFold(match[1], "off") { opt.NicoAutoConvert = false dbConfSet(db, "NicoAutoConvert", opt.NicoAutoConvert) } else { opt.NicoAutoConvert = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?auto-?delete-?mode\z`), func() error { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-auto-delete-mode: Not a number: %s\n", s) } if num < 0 || 2 < num { return fmt.Errorf("--nico-auto-delete-mode: Invalid: %d: one of 0, 1, 2\n", num) } opt.NicoAutoDeleteDBMode = num dbConfSet(db, "NicoAutoDeleteDBMode", opt.NicoAutoDeleteDBMode) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?(?:u|ultra)fast-?ts\z`), func() error { opt.NicoUltraFastTs = true return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?index\z`), func() (err error) { str, err := nextArg() if err != nil { return } ar := strings.Split(str, ",") if len(ar) > 0 { opt.NicoRtmpIndex = make(map[int]bool) } for _, s := range ar { num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-rtmp-index: Not a number: %s\n", s) } if num <= 0 { return fmt.Errorf("--nico-rtmp-index: Invalid: %d: must be greater than or equal to 1\n", num) } opt.NicoRtmpIndex[num-1] = true } return }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?status-?https\z`), func() error { // experimental opt.NicoStatusHTTPS = true return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?hls-?port\z`), func() (err error) { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-hls-port: Not a number: %s\n", s) } if num <= 0 { return fmt.Errorf("--nico-hls-port: Invalid: %d: must be greater than or equal to 1\n", num) } opt.NicoHlsPort = num return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?limit-?bw\z`), func() (err error) { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-limit-bw: Not a number: %s\n", s) } opt.NicoLimitBw = num dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?start\z`), func() (err error) { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-ts-start: Not a number %s\n", s) } opt.NicoTsStart = float64(num) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?start-?min\z`), func() (err error) { s, err := nextArg() if err != nil { return err } num, err := strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-ts-start-min: Not a number %s\n", s) } opt.NicoTsStart = float64(num * 60) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?(?:format|fmt)\z`), func() (err error) { s, err := nextArg() if err != nil { return err } if s == "" { return fmt.Errorf("--nico-format: null string not allowed\n", s) } opt.NicoFormat = s dbConfSet(db, "NicoFormat", opt.NicoFormat) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?test-?(?:format|fmt)\z`), func() (err error) { s, err := nextArg() if err != nil { return err } if s == "" { return fmt.Errorf("--nico-test-format: null string not allowed\n", s) } opt.NicoFormat = s return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?login\z`), func() (err error) { str, err := nextArg() if err != nil { return } ar := strings.SplitN(str, ",", 2) if len(ar) >= 2 && ar[0] != "" { loginId := ar[0] loginPass := ar[1] opt.NicoLoginAlias = fmt.Sprintf("%x", sha3.Sum256([]byte(loginId))) SetNicoLogin(opt.NicoLoginAlias, loginId, loginPass) dbConfSet(db, "NicoLoginAlias", opt.NicoLoginAlias) } else { return fmt.Errorf("--nico-login: ,") } return }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?session\z`), func() (err error) { str, err := nextArg() if err != nil { return } opt.NicoSession = str return }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?load-?session\z`), func() (err error) { name, err := nextArg() if err != nil { return } b, err := ioutil.ReadFile(name) if err != nil { return } if ma := regexp.MustCompile(`(\S+)`).FindSubmatch(b); len(ma) > 0 { opt.NicoSession = string(ma[1]) } else { err = fmt.Errorf("--nico-load-session: load failured") } return }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?max-?conn\z`), func() (err error) { str, err := nextArg() if err != nil { return } num, err := strconv.Atoi(str) if err != nil { return fmt.Errorf("--nico-rtmp-max-conn %v: %v", str, err) } opt.NicoRtmpMaxConn = num return }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?debug\z`), func() error { opt.NicoDebug = true return nil }}, Parser{regexp.MustCompile(`\A(?i).+\.zip\z`), func() (err error) { switch opt.Command { case "", "ZIP2MP4": opt.Command = "ZIP2MP4" opt.ZipFile = match[0] default: return fmt.Errorf("%s: Use -- option before \"%s\"", opt.Command, match[0]) } return }}, Parser{regexp.MustCompile(`\A(?i).+\.sqlite3\z`), func() (err error) { switch opt.Command { case "", "DB2MP4": opt.Command = "DB2MP4" opt.DBFile = match[0] default: return fmt.Errorf("%s: Use -- option before \"%s\"", opt.Command, match[0]) } return }}, Parser{regexp.MustCompile(`\A(?i)--?conv-?ext(?:=(mp4|ts))\z`), func() error { if strings.EqualFold(match[1], "mp4") { opt.ConvExt = "mp4" } else if strings.EqualFold(match[1], "ts") { opt.ConvExt = "ts" } dbConfSet(db, "ConvExt", opt.ConvExt) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?extract(?:-?chunks)?(?:=(on|off))\z`), func() error { if strings.EqualFold(match[1], "on") { opt.ExtractChunks = true } else if strings.EqualFold(match[1], "off") { opt.ExtractChunks = false } dbConfSet(db, "ExtractChunks", opt.ExtractChunks) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?force-?(?:re?sv|reservation)(?:=(on|off))\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoForceResv = true } else if strings.EqualFold(match[1], "off") { opt.NicoForceResv = false } dbConfSet(db, "NicoForceResv", opt.NicoForceResv) return nil }}, Parser{regexp.MustCompile(`\A(?i)--?yt-?api-?key\z`), func() (err error) { s, err := nextArg() if err != nil { return } if s == "" { return fmt.Errorf("--yt-api-key: null string not allowed\n", s) } err = SetYoutubeApiKey(s) return }}, Parser{regexp.MustCompile(`\A(?i)--?yt-?no-?streamlink(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.YtNoStreamlink = true dbConfSet(db, "YtNoStreamlink", opt.YtNoStreamlink) } else if strings.EqualFold(match[1], "off") { opt.YtNoStreamlink = false dbConfSet(db, "YtNoStreamlink", opt.YtNoStreamlink) } else { opt.YtNoStreamlink = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?yt-?no-?youtube-?dl(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.YtNoYoutubeDl = true dbConfSet(db, "YtNoYoutubeDl", opt.YtNoYoutubeDl) } else if strings.EqualFold(match[1], "off") { opt.YtNoYoutubeDl = false dbConfSet(db, "YtNoYoutubeDl", opt.YtNoYoutubeDl) } else { opt.YtNoYoutubeDl = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?skip-?hb(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.NicoSkipHb = true dbConfSet(db, "NicoSkipHb", opt.NicoSkipHb) } else if strings.EqualFold(match[1], "off") { opt.NicoSkipHb = false dbConfSet(db, "NicoSkipHb", opt.NicoSkipHb) } else { opt.NicoSkipHb = true } return nil }}, Parser{regexp.MustCompile(`\A(?i)--?http-?root-?ca\z`), func() (err error) { str, err := nextArg() if err != nil { return } opt.HttpRootCA = str return }}, Parser{regexp.MustCompile(`\A(?i)--?http-?skip-?verify(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.HttpSkipVerify = true dbConfSet(db, "HttpSkipVerify", opt.HttpSkipVerify) } else if strings.EqualFold(match[1], "off") { opt.HttpSkipVerify = false dbConfSet(db, "HttpSkipVerify", opt.HttpSkipVerify) } else { opt.HttpSkipVerify = true } return }}, Parser{regexp.MustCompile(`\A(?i)--?http-?proxy\z`), func() (err error) { str, err := nextArg() if err != nil { return } if !strings.Contains(str, "://") { str = "http://" + str } opt.HttpProxy = str return }}, Parser{regexp.MustCompile(`\A(?i)--?no-?chdir\z`), func() (err error) { opt.NoChdir = true return }}, } checkFILE := func(arg string) bool { switch opt.Command { default: //fmt.Printf("command not specified: -- \"%s\"\n", arg) //os.Exit(1) case "YOUTUBE": if ma := regexp.MustCompile(`v=([\w-]+)`).FindStringSubmatch(arg); len(ma) > 0 { opt.YoutubeId = ma[1] return true } else if ma := regexp.MustCompile(`\A([\w-]+)\z`).FindStringSubmatch(arg); len(ma) > 0 { opt.YoutubeId = ma[1] return true } else { fmt.Printf("Not YouTube id: %s\n", arg) os.Exit(1) } case "NICOLIVE": if ma := regexp.MustCompile(`(lv\d+)`).FindStringSubmatch(arg); len(ma) > 0 { opt.NicoLiveId = ma[1] return true } case "TWITCAS": if opt.TcasId != "" { fmt.Printf("Unknown option: %s\n", arg) Help() } if ma := regexp.MustCompile(`(?:.*/)?([^/]+)\z`).FindStringSubmatch(arg); len(ma) > 0 { opt.TcasId = ma[1] return true } case "ZIP2MP4": if ma := regexp.MustCompile(`(?i)\.zip`).FindStringSubmatch(arg); len(ma) > 0 { opt.ZipFile = arg return true } case "DB2MP4": if ma := regexp.MustCompile(`(?i)\.sqlite3`).FindStringSubmatch(arg); len(ma) > 0 { opt.DBFile = arg return true } return false } // end switch return false } LB_ARG: for len(args) > 0 { arg, _ := nextArg() if arg == "--" { switch len(args) { case 0: fmt.Printf("argument not specified after \"--\"\n") os.Exit(1) default: fmt.Printf("too many arguments after \"--\": %v\n", args) os.Exit(1) case 1: arg, _ := nextArg() checkFILE(arg) } } else { for _, p := range parseList { if match = p.re.FindStringSubmatch(arg); len(match) > 0 { if e := p.cb(); e != nil { fmt.Println(e) os.Exit(1) } continue LB_ARG } } if ok := checkFILE(arg); !ok { fmt.Printf("Unknown option: %v\n", arg) Help() } } } if opt.ConfFile == "" { opt.ConfFile = fmt.Sprintf("%s.conf", getCmd()) } // [deprecated] // load session info if data, e := cryptoconf.Load(opt.ConfFile, opt.ConfPass); e != nil { err = e return } else { loginId, _ := data["NicoLoginId"].(string) if loginId != "" { loginPass, _ := data["NicoLoginPass"].(string) hash := fmt.Sprintf("%x", sha3.Sum256([]byte(loginId))) SetNicoLogin(hash, loginId, loginPass) if opt.NicoLoginAlias == "" { opt.NicoLoginAlias = hash dbConfSet(db, "NicoLoginAlias", opt.NicoLoginAlias) } os.Remove(opt.ConfFile) } } // prints switch opt.Command { case "NICOLIVE": fmt.Printf("Conf(NicoLoginOnly): %#v\n", opt.NicoLoginOnly) fmt.Printf("Conf(NicoFormat): %#v\n", opt.NicoFormat) fmt.Printf("Conf(NicoLimitBw): %#v\n", opt.NicoLimitBw) fmt.Printf("Conf(NicoHlsOnly): %#v\n", opt.NicoHlsOnly) fmt.Printf("Conf(NicoRtmpOnly): %#v\n", opt.NicoRtmpOnly) fmt.Printf("Conf(NicoFastTs): %#v\n", opt.NicoFastTs) fmt.Printf("Conf(NicoAutoConvert): %#v\n", opt.NicoAutoConvert) if opt.NicoAutoConvert { fmt.Printf("Conf(NicoAutoDeleteDBMode): %#v\n", opt.NicoAutoDeleteDBMode) fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) } fmt.Printf("Conf(NicoForceResv): %#v\n", opt.NicoForceResv) fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) case "YOUTUBE": fmt.Printf("Conf(YtNoStreamlink): %#v\n", opt.YtNoStreamlink) fmt.Printf("Conf(YtNoYoutubeDl): %#v\n", opt.YtNoYoutubeDl) case "TWITCAS": fmt.Printf("Conf(TcasRetry): %#v\n", opt.TcasRetry) fmt.Printf("Conf(TcasRetryTimeoutMinute): %#v\n", opt.TcasRetryTimeoutMinute) fmt.Printf("Conf(TcasRetryInterval): %#v\n", opt.TcasRetryInterval) case "DB2MP4": fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) } fmt.Printf("Conf(HttpSkipVerify): %#v\n", opt.HttpSkipVerify) if opt.NicoDebug { fmt.Printf("Conf(NicoDebug): %#v\n", opt.NicoDebug) } // check switch opt.Command { case "": fmt.Printf("Command not specified\n") Help() case "YOUTUBE": if opt.YoutubeId == "" { Help() } case "NICOLIVE": if opt.NicoLiveId == "" { Help() } case "NICOLIVE_TEST": case "TWITCAS": if opt.TcasId == "" { Help() } case "ZIP2MP4": if opt.ZipFile == "" { Help() } case "DB2MP4": if opt.DBFile == "" { Help() } default: fmt.Printf("[FIXME] options.go/argcheck for %s\n", opt.Command) os.Exit(1) } return } ================================================ FILE: src/procs/base/base.go ================================================ package base import ( "io" "os" "os/exec" ) func Open(cmdList *[]string, stdinEn, stdoutEn, stdErrEn, consoleEn bool, args []string) (cmd *exec.Cmd, stdin io.WriteCloser, stdout, stderr io.ReadCloser, err error) { for i, cmdName := range *cmdList { cmd = exec.Command(cmdName, args...) if stdinEn { stdin, err = cmd.StdinPipe() if err != nil { return } } if stdoutEn { stdout, err = cmd.StdoutPipe() if err != nil { return } } else { if consoleEn { cmd.Stdout = os.Stdout } } if stdErrEn { stderr, err = cmd.StderrPipe() if err != nil { return } } else { if consoleEn { cmd.Stderr = os.Stderr } } if err = cmd.Start(); err != nil { continue } else { if i != 0 { *cmdList = []string{cmdName} } //fmt.Printf("CMD: %#v\n", cmd.Args) return } } // prog not found cmd = nil return } ================================================ FILE: src/procs/ffmpeg/ffmpeg.go ================================================ package ffmpeg import ( "fmt" "io" "os/exec" "github.com/himananiito/livedl/procs/base" ) var cmdList = []string{ "./bin/ffmpeg/bin/ffmpeg", "./bin/ffmpeg/ffmpeg", "./bin/ffmpeg", "./ffmpeg/bin/ffmpeg", "./ffmpeg/ffmpeg", "./ffmpeg", "ffmpeg", } func Open(opt ...string) (cmd *exec.Cmd, stdin io.WriteCloser, err error) { cmd, stdin, _, _, err = base.Open(&cmdList, true, false, false, true, opt) if cmd == nil { err = fmt.Errorf("ffmpeg not found") return } return } ================================================ FILE: src/procs/kill.go ================================================ package procs import ( "fmt" "log" "runtime" "github.com/himananiito/livedl/procs/base" ) func Kill(pid int) { if runtime.GOOS == "windows" { options := []string{ "/PID", fmt.Sprintf("%v", pid), "/T", "/F", } list := []string{"taskkill"} if taskkill, _, _, _, err := base.Open(&list, false, false, false, false, options); err == nil { taskkill.Wait() } } else { log.Fatalf("[FIXME] Kill for %v not supported", runtime.GOOS) } } ================================================ FILE: src/procs/streamlink/streamlink.go ================================================ package streamlink import ( "fmt" "io" "os/exec" "github.com/himananiito/livedl/procs/base" ) var cmdList = []string{ "./bin/streamlink/streamlink", "./bin/Streamlink/Streamlink", "./bin/streamlink", "./bin/Streamlink", "./streamlink/streamlink", "./Streamlink/Streamlink", "./Streamlink", "streamlink", "Streamlink", } func Open(opt ...string) (cmd *exec.Cmd, stdout, stderr io.ReadCloser, err error) { cmd, _, stdout, stderr, err = base.Open(&cmdList, false, true, true, false, opt) if cmd == nil { err = fmt.Errorf("streamlink not found") return } return } ================================================ FILE: src/procs/youtube_dl/youtube-dl.go ================================================ package youtube_dl import ( "fmt" "io" "os/exec" "github.com/himananiito/livedl/procs/base" ) var cmdList = []string{ "./bin/youtube-dl/youtube-dl", "./bin/youtube-dl", "./youtube-dl/youtube-dl", "./youtube-dl", "youtube-dl", } func Open(opt ...string) (cmd *exec.Cmd, stdout, stderr io.ReadCloser, err error) { cmd, _, stdout, stderr, err = base.Open(&cmdList, false, true, true, false, opt) if cmd == nil { err = fmt.Errorf("youtube-dl not found") return } return } ================================================ FILE: src/rtmps/message.go ================================================ package rtmps import ( "bytes" "encoding/binary" "fmt" "io" "log" "time" "github.com/himananiito/livedl/amf" ) const ( TID_SETCHUNKSIZE = 1 TID_ABORT = 2 TID_ACKNOWLEDGEMENT = 3 TID_USERCONTROL = 4 TID_WINDOW_ACK_SIZE = 5 TID_SETPEERBANDWIDTH = 6 TID_AUDIO = 8 TID_VIDEO = 9 TID_AMF3COMMAND = 17 TID_AMF0COMMAND = 20 TID_AMF0DATA = 18 TID_AMF3DATA = 15 TID_AGGREGATE = 22 ) const ( UC_STREAMBEGIN = 0 UC_STREAMEOF = 1 UC_STREAMDRY = 2 UC_SETBUFFERLENGTH = 3 UC_STREAMISRECORDED = 4 UC_PINGREQUEST = 6 UC_PINGRESPONSE = 7 UC_BUFFEREMPTY = 31 UC_BUFFERREADY = 32 ) func intToBE16(num int) (data []byte) { tmp := make([]byte, 2) binary.BigEndian.PutUint16(tmp, uint16(num)) data = append(data, tmp[:]...) return } func intToBE24(num int) (data []byte) { tmp := make([]byte, 4) binary.BigEndian.PutUint32(tmp, uint32(num)) data = append(data, tmp[1:]...) return } func intToBE32(num int) (data []byte) { tmp := make([]byte, 4) binary.BigEndian.PutUint32(tmp, uint32(num)) data = append(data, tmp[:]...) return } func intToLE32(num int) (data []byte) { tmp := make([]byte, 4) binary.LittleEndian.PutUint32(tmp, uint32(num)) data = append(data, tmp[:]...) return } func chunkBasicHeader(fmt, csid int) (data []byte) { if 2 <= csid && csid <= 63 { b := byte(((fmt & 3) << 6) | (csid & 0x3F)) data = append(data, b) } else if 64 <= csid && csid <= 319 { b0 := byte((fmt & 3) << 6) b1 := byte(csid - 64) data = append(data, b0, b1) } else if 320 <= csid && csid <= 65599 { b0 := byte(((fmt & 3) << 6) | 1) b1 := byte((csid & 0xFF) - 64) b2 := byte(csid >> 8) data = append(data, b0, b1, b2) } else { log.Printf("[FIXME] Chunk basic header: csid out of range: %d", csid) } return } var start = millisec() func millisec() int64 { return time.Now().UnixNano() / int64(time.Millisecond) } func getTime() int { delta := millisec() - start return int(delta) } func type0(buff *bytes.Buffer, csId int, typeId byte, streamId int, length int) { buff.Write(chunkBasicHeader(0, csId)) // timestamp //buff.Write(intToBE24(getTime())) buff.Write(intToBE24(0)) // message length buff.Write(intToBE24(length)) // message type id buff.WriteByte(typeId) // Stream ID buff.Write(intToLE32(streamId)) // body //buff.Write(body) return } func type3(buff *bytes.Buffer, csId int) { buff.Write(chunkBasicHeader(3, csId)) } func encodeAcknowledgement(asz int) (buff *bytes.Buffer, err error) { buff = bytes.NewBuffer(nil) bsz := intToBE32(asz) type0(buff, 2, TID_ACKNOWLEDGEMENT, 0, len(bsz)) if _, err = buff.Write(bsz); err != nil { return } return } func encodeWindowAckSize(asz int) (buff *bytes.Buffer, err error) { buff = bytes.NewBuffer(nil) bsz := intToBE32(asz) type0(buff, 2, TID_WINDOW_ACK_SIZE, 0, len(bsz)) if _, err = buff.Write(bsz); err != nil { return } return } func encodeSetPeerBandwidth(wsz, lim int) (buff *bytes.Buffer, err error) { buff = bytes.NewBuffer(nil) b := intToBE32(wsz) b = append(b, byte(lim)) type0(buff, 2, TID_SETPEERBANDWIDTH, 0, len(b)) if _, err = buff.Write(b); err != nil { return } return } func encodePingResponse(timestamp int) (buff *bytes.Buffer, err error) { buff = bytes.NewBuffer(nil) var body []byte body = append(body, intToBE16(UC_PINGRESPONSE)...) body = append(body, intToBE32(timestamp)...) type0(buff, 2, TID_USERCONTROL, 0, len(body)) if _, err = buff.Write(body); err != nil { return } return } func encodeSetBufferLength(streamId, length int) (buff *bytes.Buffer, err error) { buff = bytes.NewBuffer(nil) var body []byte body = append(body, intToBE16(UC_SETBUFFERLENGTH)...) body = append(body, intToBE32(streamId)...) body = append(body, intToBE32(int(length))...) type0(buff, 2, TID_USERCONTROL, 0, len(body)) if _, err = buff.Write(body); err != nil { return } return } func amf0Command(chunkSize, csId, streamId int, body []byte) (wbuff *bytes.Buffer, err error) { wbuff = bytes.NewBuffer(nil) rbuff := bytes.NewBuffer(body) type0(wbuff, csId, TID_AMF0COMMAND, streamId, rbuff.Len()) if chunkSize < rbuff.Len() { if _, err = io.CopyN(wbuff, rbuff, int64(chunkSize)); err != nil { return } } else { if _, err = io.CopyN(wbuff, rbuff, int64(rbuff.Len())); err != nil { return } } for rbuff.Len() > 0 { type3(wbuff, csId) if chunkSize < rbuff.Len() { if _, err = io.CopyN(wbuff, rbuff, int64(chunkSize)); err != nil { return } } else { if _, err = io.CopyN(wbuff, rbuff, int64(rbuff.Len())); err != nil { return } } } //log.Fatalf("amf0Command %#v", wbuff) return } func decodeFmtCsId(rdr io.Reader, msg *rtmpMsg) (err error) { b0 := make([]byte, 1) msg.hdrLength++ _, err = io.ReadFull(rdr, b0) if err != nil { return } format := (int(b0[0]) >> 6) & 3 csId := int(b0[0]) & 0x3F switch csId { case 0: b1 := make([]byte, 1) msg.hdrLength++ if _, err = io.ReadFull(rdr, b1); err != nil { return } csId = int(b1[0]) + 64 case 1: b1 := make([]byte, 2) msg.hdrLength += 2 if _, err = io.ReadFull(rdr, b1); err != nil { return } csId = (int(b1[1]) << 8) | (int(b1[0]) + 64) } msg.format = format msg.csId = csId if !msg.readingBody { msg.formatOrigin = format msg.csIdOrigin = csId } // fmt.Printf("debug format type %v csid %v\n", format, csId) return } func decodeInt8(rdr io.Reader) (num int, err error) { buf := make([]byte, 1) if _, err = io.ReadFull(rdr, buf); err != nil { return } num = int(buf[0]) return } func decodeBEInt16(rdr io.Reader) (num int, err error) { buf := make([]byte, 2) if _, err = io.ReadFull(rdr, buf); err != nil { return } num = (int(buf[0]) << 8) | int(buf[1]) return } func decodeBEInt24(rdr io.Reader) (num int, err error) { buf := make([]byte, 3) if _, err = io.ReadFull(rdr, buf); err != nil { return } num = (int(buf[0]) << 16) | (int(buf[1]) << 8) | int(buf[2]) return } func decodeBEInt32(rdr io.Reader) (num int, err error) { buf := make([]byte, 4) if _, err = io.ReadFull(rdr, buf); err != nil { return } num = (int(buf[0]) << 24) | (int(buf[1]) << 16) | (int(buf[2]) << 8) | int(buf[3]) return } func decodeLEInt32(rdr io.Reader) (num int, err error) { buf := make([]byte, 4) if _, err = io.ReadFull(rdr, buf); err != nil { return } num = (int(buf[3]) << 24) | (int(buf[2]) << 16) | (int(buf[1]) << 8) | int(buf[0]) return } func decodeTimestamp(rdr io.Reader, msg *rtmpMsg) (err error) { msg.hdrLength += 3 timestamp, err := decodeBEInt24(rdr) if err != nil { return } msg.timestampField = timestamp return } func decodeTimestampEX(rdr io.Reader, msg *rtmpMsg) (err error) { msg.hdrLength += 4 timestamp, err := decodeBEInt32(rdr) if err != nil { return } //fmt.Printf("decodeTimestampEX %v\n", timestamp) if !msg.readingBody { msg.timestampEx = timestamp } return } func decodeMsgLength(rdr io.Reader, msg *rtmpMsg) (err error) { msg.hdrLength += 3 length, err := decodeBEInt24(rdr) msg.msgLength = length return } func decodeMsgType(rdr io.Reader, msg *rtmpMsg) (err error) { msg.hdrLength += 1 msg_t, err := decodeInt8(rdr) msg.msgTypeId = msg_t return } func decodeStreamId(rdr io.Reader, msg *rtmpMsg) (err error) { msg.hdrLength += 4 sid, err := decodeLEInt32(rdr) msg.msgStreamId = sid return } func decodeType0(rdr io.Reader, msg *rtmpMsg) (err error) { if err = decodeTimestamp(rdr, msg); err != nil { return } if err = decodeMsgLength(rdr, msg); err != nil { return } if err = decodeMsgType(rdr, msg); err != nil { return } err = decodeStreamId(rdr, msg) return } func decodeType1(rdr io.Reader, msg *rtmpMsg) (err error) { if err = decodeTimestamp(rdr, msg); err != nil { return } if err = decodeMsgLength(rdr, msg); err != nil { return } err = decodeMsgType(rdr, msg) return } func decodeType2(rdr io.Reader, msg *rtmpMsg) (err error) { err = decodeTimestamp(rdr, msg) return } type rtmpMsg struct { format int formatOrigin int csId int csIdOrigin int timestampField int timestampDelta int timestampEx int timestampActual int msgLength int msgTypeId int msgStreamId int bodyBuff *bytes.Buffer readingBody bool hdrLength int splitCount int } func readChunkBody(rdr io.Reader, msg *rtmpMsg, csz int) (err error) { if msg.bodyBuff == nil { msg.bodyBuff = bytes.NewBuffer(nil) } rem := msg.msgLength - msg.bodyBuff.Len() //fmt.Printf("readChunkBody: %v %v\n", msg.msgLength, msg.bodyBuff.Len()) if rem > csz { _, err = io.CopyN(msg.bodyBuff, rdr, int64(csz)) } else { _, err = io.CopyN(msg.bodyBuff, rdr, int64(rem)) } if err != nil { return } return } func decodeHeader(rdr io.Reader, msg *rtmpMsg) (err error) { if err = decodeFmtCsId(rdr, msg); err != nil { return } switch msg.format { case 0: if err = decodeType0(rdr, msg); err != nil { return } case 1: if err = decodeType1(rdr, msg); err != nil { return } case 2: if err = decodeType2(rdr, msg); err != nil { return } case 3: if msg.readingBody { msg.splitCount++ if msg.csId != msg.csIdOrigin { err = &DecodeError{ Fun: "decodeHeader", Msg: fmt.Sprintf("msg.csId(%d) != msg.csIdOrigin(%d)", msg.csId, msg.csIdOrigin), } return } } default: err = &DecodeError{ Fun: "decodeHeader", Msg: fmt.Sprintf("Unknown fmt: %v", msg.format), } return } return } func decodeSetChunkSize(rbuff *bytes.Buffer) (csz int, err error) { num, e := decodeBEInt32(rbuff) if e != nil { err = e return } csz = num & 0x7fffffff return } func decodeWindowAckSize(rbuff *bytes.Buffer) (asz int, err error) { asz, e := decodeBEInt32(rbuff) if e != nil { err = e return } return } func decodeSetPeerBandwidth(rbuff *bytes.Buffer) (res []int, err error) { wsz, err := decodeBEInt32(rbuff) if err != nil { return } lim, err := decodeInt8(rbuff) if err != nil { return } res = append(res, wsz, lim) return } func decodeUserControl(rbuff *bytes.Buffer) (res []int, err error) { evt, err := decodeBEInt16(rbuff) if err != nil { return } res = append(res, evt) switch evt { case UC_BUFFEREMPTY, UC_BUFFERREADY: // Buffer Empty, Buffer Ready // http://repo.or.cz/w/rtmpdump.git/blob/8880d1456b282ee79979adbe7b6a6eb8ad371081:/librtmp/rtmp.c#l2787 case UC_STREAMBEGIN, UC_STREAMEOF, UC_STREAMDRY, UC_STREAMISRECORDED, UC_PINGREQUEST, UC_PINGRESPONSE: // 4-byte stream id num, e := decodeBEInt32(rbuff) if e != nil { err = e return } res = append(res, num) case UC_SETBUFFERLENGTH: // 4-byte stream id sid, e := decodeBEInt32(rbuff) if e != nil { err = e return } res = append(res, sid) // 4-byte buffer length bsz, e := decodeBEInt32(rbuff) if e != nil { err = e return } res = append(res, bsz) default: err = &DecodeError{ Fun: "decodeUserControl", Msg: fmt.Sprintf("Unknown User control: %v", evt), } return } return } type message struct { msg_t int timestamp int data *bytes.Buffer } func decodeMessage(rbuff *bytes.Buffer) (res message, err error) { msg_t, err := decodeInt8(rbuff) if err != nil { return } plen, err := decodeBEInt24(rbuff) if err != nil { return } ts_0, err := decodeBEInt24(rbuff) if err != nil { return } ts_1, err := decodeInt8(rbuff) if err != nil { return } ts := (ts_1 << 24) | ts_0 // stream id _, err = decodeBEInt24(rbuff) if err != nil { return } //fmt.Printf("debug decodeMessage: type(%v) len(%v) ts(%v)\n", msg_t, plen, ts_0) buff := bytes.NewBuffer(nil) if _, err = io.CopyN(buff, rbuff, int64(plen)); err != nil { return } // backPointer _, err = decodeBEInt32(rbuff) if err != nil { return } res = message{ msg_t: msg_t, timestamp: ts, data: buff, } return } func decodeAggregate(rbuff *bytes.Buffer) (res []message, err error) { for rbuff.Len() > 0 { msg, e := decodeMessage(rbuff) if e != nil { err = e return } res = append(res, msg) } return } func decodeOne(rdr io.Reader, csz int, info map[int]chunkInfo) (ts int, msg_t int, res interface{}, rsz int, err error) { msg := rtmpMsg{} // rtmp header if err = decodeHeader(rdr, &msg); err != nil { return } // restore fields from previous chunk header var prevChunk chunkInfo if msg.formatOrigin != 0 { var ok bool if prevChunk, ok = info[msg.csIdOrigin]; !ok { err = &DecodeError{ Fun: "decodeOne", Msg: fmt.Sprintf("Not exists previous chunk(csId = %v)", msg.csIdOrigin), } return } } //fmt.Printf("debug decodeOne msg.timestampField %d\n", msg.timestampField) if (msg.timestampField == 0xffffff) || ((msg.formatOrigin == 3) && (prevChunk.timestampField == 0xffffff)) { if err = decodeTimestampEX(rdr, &msg); err != nil { return } //fmt.Printf("%#v\n", msg) switch msg.formatOrigin { case 0: msg.timestampActual = msg.timestampEx msg.timestampDelta = msg.timestampEx case 1, 2: msg.timestampActual = prevChunk.timestampActual + msg.timestampEx msg.timestampDelta = msg.timestampEx case 3: msg.timestampActual = msg.timestampEx msg.timestampDelta = msg.timestampEx msg.timestampField = 0xffffff } } else { switch msg.formatOrigin { case 0: msg.timestampActual = msg.timestampField msg.timestampDelta = msg.timestampField case 1, 2: msg.timestampActual = prevChunk.timestampActual + msg.timestampField msg.timestampDelta = msg.timestampField case 3: msg.timestampActual = prevChunk.timestampActual + prevChunk.timestampDelta msg.timestampDelta = prevChunk.timestampDelta } } switch msg.formatOrigin { case 1: msg.msgStreamId = prevChunk.msgStreamId case 2, 3: msg.msgLength = prevChunk.msgLength msg.msgTypeId = prevChunk.msgTypeId msg.msgStreamId = prevChunk.msgStreamId } info[msg.csId] = chunkInfo{ timestampField: msg.timestampField, timestampDelta: msg.timestampDelta, timestampActual: msg.timestampActual, msgLength: msg.msgLength, msgTypeId: msg.msgTypeId, msgStreamId: msg.msgStreamId, } ts = msg.timestampActual msg.readingBody = true // rtmp payload for { if err = readChunkBody(rdr, &msg, csz); err != nil { return } if msg.msgLength <= msg.bodyBuff.Len() { break } //if err = decodeFmtCsId(rdr, &msg); err != nil { if err = decodeHeader(rdr, &msg); err != nil { return } // timestamp extended if msg.timestampField == 0xffffff { if err = decodeTimestampEX(rdr, &msg); err != nil { return } } } //fmt.Printf("debug rtmp decodeOne: %#v\n", msg) // read byte count rsz = msg.hdrLength + msg.msgLength msg_t = msg.msgTypeId switch msg.msgTypeId { case TID_AGGREGATE: if res, err = decodeAggregate(msg.bodyBuff); err != nil { return } case TID_AUDIO, TID_VIDEO: res = msg.bodyBuff case TID_WINDOW_ACK_SIZE: if res, err = decodeWindowAckSize(msg.bodyBuff); err != nil { return } case TID_SETPEERBANDWIDTH: if res, err = decodeSetPeerBandwidth(msg.bodyBuff); err != nil { return } case TID_AMF0COMMAND: if res, err = amf.DecodeAmf0(msg.bodyBuff.Bytes()); err != nil { return } case TID_AMF3COMMAND: if res, err = amf.DecodeAmf0(msg.bodyBuff.Bytes(), true); err != nil { return } case TID_AMF0DATA: if res, err = amf.DecodeAmf0(msg.bodyBuff.Bytes()); err != nil { return } case TID_SETCHUNKSIZE: if res, err = decodeSetChunkSize(msg.bodyBuff); err != nil { return } case TID_USERCONTROL: if res, err = decodeUserControl(msg.bodyBuff); err != nil { return } default: err = &DecodeError{ Fun: "decodeOne", Msg: fmt.Sprintf("msgTypeId: not implement: %v\n%#v", msg.msgTypeId, msg.bodyBuff.Bytes()), } return } return } ================================================ FILE: src/rtmps/rtmp.go ================================================ package rtmps import ( "bytes" "fmt" "io" "io/ioutil" "math/rand" "net" "regexp" "time" "github.com/himananiito/livedl/amf" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/flvs" "github.com/himananiito/livedl/objs" ) type DecodeError struct { Fun string Msg string } func (e *DecodeError) Error() string { return fmt.Sprintf("%s: %s", e.Fun, e.Msg) } type chunkInfo struct { timestampField int timestampDelta int timestampActual int msgLength int msgTypeId int msgStreamId int } type Rtmp struct { proto string // No reset address string // No reset app string // No reset tcUrl string // No reset swfUrl string // No reset pageUrl string // No reset connectOpt []interface{} conn *net.TCPConn // RESET_ON_CONNECT chunkSizeSend int // RESET_ON_CONNECT chunkSizeRecv int // RESET_ON_CONNECT transactionId int // RESET_ON_CONNECT windowSize int // RESET_ON_CONNECT chunkInfo map[int]chunkInfo // RESET_ON_CONNECT readCount int // RESET_ON_CONNECT totalReadBytes int // RESET_ON_CONNECT isRecorded bool timestamp int // NO_RESET duration int flvName string flv *flvs.Flv fixAggrTimestamp bool streamId int nextLogTs int VideoExists bool noSeek bool flush bool startTime int } func NewRtmp(tc, swf, page string, opt ...interface{}) (rtmp *Rtmp, err error) { re := regexp.MustCompile(`\A(\w+)://([^/\s]+)/(\S+)\z`) mstr := re.FindStringSubmatch(tc) if mstr == nil { err = fmt.Errorf("tcUrl incorrect: %v", tc) return } rtmp = &Rtmp{ proto: mstr[1], address: mstr[2], app: mstr[3], tcUrl: tc, swfUrl: swf, pageUrl: page, connectOpt: opt, } return } func (rtmp *Rtmp) Connect() (err error) { if rtmp.conn != nil { rtmp.conn.Close() rtmp.conn = nil time.Sleep(3) } rtmp.windowSize = 2500000 rtmp.chunkInfo = make(map[int]chunkInfo) rtmp.chunkSizeSend = 128 rtmp.chunkSizeRecv = 128 rtmp.transactionId = 1 rtmp.readCount = 0 rtmp.totalReadBytes = 0 err = rtmp.connect( rtmp.app, rtmp.tcUrl, rtmp.swfUrl, rtmp.pageUrl, rtmp.connectOpt..., ) return } func (rtmp *Rtmp) SetFlush(b bool) { rtmp.flush = b } func (rtmp *Rtmp) SetNoSeek(b bool) { rtmp.noSeek = b } func (rtmp *Rtmp) SetConnectOpt(opt ...interface{}) { rtmp.connectOpt = opt } func (rtmp *Rtmp) connect(app, tc, swf, page string, opt ...interface{}) (err error) { raddr, err := net.ResolveTCPAddr("tcp", rtmp.address) if err != nil { fmt.Printf("%v\n", err) return } switch rtmp.proto { case "rtmp": conn, e := net.DialTCP("tcp", nil, raddr) if e != nil { err = e return } rtmp.conn = conn default: err = fmt.Errorf("Unknown protocol: %v", rtmp.proto) return } err = handshake(rtmp.conn) if err != nil { rtmp.conn.Close() return } var data []interface{} data = append(data, map[string]interface{}{ "app": app, "flashVer": "WIN 29,0,0,113", "swfUrl": swf, "tcUrl": tc, "fpad": false, "capabilities": 239, "audioCodecs": 0xFFF, //3575, "videoCodecs": 0xFF, //252, "videoFunction": 1, "pageUrl": page, "objectEncoding": 3, }) for _, o := range opt { data = append(data, o) } _, err = rtmp.Command("connect", data) return } const ( NORMAL = iota COMMAND PAUSE TEST ) func (rtmp *Rtmp) wait(findTrId int, pause bool, testTimeout int) (done, incomplete bool, trData interface{}, err error) { var mode int var endUnix int64 var endTime time.Time if findTrId >= 0 { mode = COMMAND } else if pause { mode = PAUSE } else if testTimeout > 0 { mode = TEST endUnix = time.Now().Unix() + int64(testTimeout) endTime = time.Unix(endUnix, 0) } if mode != COMMAND { findTrId = -1 } for { if mode == TEST { rtmp.conn.SetReadDeadline(endTime) } else { rtmp.conn.SetReadDeadline(time.Now().Add(300 * time.Second)) } __done, __incomplete, trFound, pause, __trData, e := rtmp.recvChunk(findTrId, pause) if e != nil { err = e return } if __done || __incomplete { done = __done incomplete = __incomplete return } switch mode { case COMMAND: if trFound { trData = __trData return } case PAUSE: if pause { return } case TEST: if time.Now().Unix() >= endUnix { return } } } } func (rtmp *Rtmp) WaitPause() (done, incomplete bool, err error) { done, incomplete, _, err = rtmp.wait(-1, true, -1) return } func (rtmp *Rtmp) WaitTest(testTimeout int) (done, incomplete bool, err error) { done, incomplete, _, err = rtmp.wait(-1, false, testTimeout) return } func (rtmp *Rtmp) Wait() (done, incomplete bool, err error) { done, incomplete, _, err = rtmp.wait(-1, false, -1) return } func (rtmp *Rtmp) waitCommand(findTrId int) (done, incomplete bool, trData interface{}, err error) { done, incomplete, trData, err = rtmp.wait(findTrId, false, -1) return } func (rtmp *Rtmp) SetFlvName(name string) { rtmp.flvName = name } func (rtmp *Rtmp) openFlv(incr bool) (err error) { if rtmp.flvName == "" { err = fmt.Errorf("FLV file name not set: call SetFlvName(string)") return } var fileName string if incr { if fileName, err = files.GetFileNameNext(rtmp.flvName); err != nil { return } } else { fileName = rtmp.flvName } flv, err := flvs.Open(fileName) if err != nil { return } rtmp.flv = flv return } func (rtmp *Rtmp) GetTimestamp() int { return rtmp.timestamp } func (rtmp *Rtmp) SetTimestamp(t int) { rtmp.timestamp = t } func (rtmp *Rtmp) writeMetaData(body map[string]interface{}, ts int) (err error) { if rtmp.flv == nil { if err = rtmp.openFlv(false); err != nil { return } } //buf := new(bytes.Buffer) data := []interface{}{} data = append(data, "onMetaData") data = append(data, body) dat, err := amf.EncodeAmf0(data, true) //fmt.Printf("writeMetaData %v %#v\n", ts, dat) rdr := bytes.NewBuffer(dat) err = rtmp.flv.WriteMetaData(rdr, ts) return } func (rtmp *Rtmp) writeAudio(rdr *bytes.Buffer, ts int) (err error) { if rtmp.flv == nil { if err = rtmp.openFlv(false); err != nil { return } } err = rtmp.flv.WriteAudio(rdr, ts) return } func (rtmp *Rtmp) writeVideo(rdr *bytes.Buffer, ts int) (err error) { if rtmp.flv == nil { if err = rtmp.openFlv(false); err != nil { return } } /*else if (!rtmp.flv.VideoExists() && rtmp.flv.AudioExists()) && ts > 1000 { if err = rtmp.openFlv(true); err != nil { return } }*/ err = rtmp.flv.WriteVideo(rdr, ts) return } func (rtmp *Rtmp) SetFixAggrTimestamp(sw bool) { rtmp.fixAggrTimestamp = sw } func (rtmp *Rtmp) CheckStatus(label string, ts int, data interface{}, waitPause bool) (done, incomplete, pauseFound bool, err error) { code, ok := objs.FindString(data, "code") if !ok { err = fmt.Errorf("%s: code Not found", label) return } switch code { case "NetStream.Pause.Notify": if waitPause { pauseFound = true } case "NetStream.Unpause.Notify": case "NetStream.Play.Stop": case "NetStream.Play.Complete": fmt.Printf("NetStream.Play.Complete: last timestamp: %d(flv)\n", rtmp.flv.GetLastTimestamp()) if (ts + 1000) > rtmp.duration { done = true } else { incomplete = true } case "NetStream.Play.Start": case "NetStream.Play.Reset": case "NetStream.Seek.Notify": case "NetStream.Play.Failed": done = true default: fmt.Printf("[FIXME] Unknown Code: %s\n", code) } return } // trId: transaction id to find func (rtmp *Rtmp) recvChunk(findTrId int, waitPause bool) (done, incomplete, trFound, pauseFound bool, trData interface{}, err error) { ts, msg_t, res, rdbytes, err := decodeOne(rtmp.conn, rtmp.chunkSizeRecv, rtmp.chunkInfo) if err != nil { switch err.(type) { case *net.OpError: return case *DecodeError: // データを受信したが、パースエラーとなった場合はやり直したい fmt.Printf("Please retry: RTMP: %v\n", err.Error()) incomplete = true err = nil return } return } ts = ts + rtmp.startTime // byte counter for acknowledgement rtmp.totalReadBytes += rdbytes rtmp.readCount += rdbytes if rtmp.readCount >= (rtmp.windowSize / 2) { rtmp.readCount = 0 if err = rtmp.acknowledgement(); err != nil { return } } // print play timestamp if true { if rtmp.duration > 0 { switch msg_t { case TID_AUDIO, TID_VIDEO, TID_AGGREGATE: if ts >= rtmp.nextLogTs { fmt.Printf("#%8d/%d(%4.1f%%) : %s\n", ts, rtmp.duration, float64(ts)/float64(rtmp.duration)*100, rtmp.flvName) rtmp.nextLogTs = ts + 10000 } } } else { switch msg_t { case TID_AUDIO, TID_VIDEO, TID_AGGREGATE: if ts >= rtmp.nextLogTs { fmt.Printf("#%8d : %s\n", ts, rtmp.flvName) rtmp.nextLogTs = ts + 10000 } } } } switch msg_t { case TID_AUDIO: if ts > rtmp.timestamp { rtmp.timestamp = ts } if err = rtmp.writeAudio(res.(*bytes.Buffer), ts); err != nil { return } case TID_VIDEO: if ts > rtmp.timestamp { rtmp.timestamp = ts } if err = rtmp.writeVideo(res.(*bytes.Buffer), ts); err != nil { return } case TID_AGGREGATE: if ts > rtmp.timestamp { rtmp.timestamp = ts } var fstTs int for i, v := range res.([]message) { var tsAggr int if rtmp.fixAggrTimestamp { var delta int if i == 0 { fstTs = v.timestamp } delta = v.timestamp - fstTs tsAggr = ts + delta //fmt.Printf("FixAggrTs: fixed(%d), delta(%d), ts(%d), mts(%d)\n", tsAggr, delta, ts, v.timestamp) } else { if i == 0 { if ts != v.timestamp { err = fmt.Errorf("aggregate timestamp incorrect: ts:(%v) vs aggr[0].ts(%v)", ts, v.timestamp) return } } tsAggr = v.timestamp } if /*rtmp.isRecorded &&*/ rtmp.duration > 0 { switch v.msg_t { case TID_AUDIO, TID_VIDEO: // fmt.Printf(" %8d/%d(%4.1f%%) : %2d\n", tsAggr, rtmp.duration, float64(tsAggr)/float64(rtmp.duration)*100, v.msg_t) } } switch v.msg_t { case TID_AUDIO: // audio if err = rtmp.writeAudio(v.data, tsAggr); err != nil { return } case TID_VIDEO: // video if err = rtmp.writeVideo(v.data, tsAggr); err != nil { return } } } case TID_AMF0DATA, TID_AMF3DATA: objs.PrintAsJson(res) list, ok := res.([]interface{}) if !ok { err = fmt.Errorf("result AMF Data is not array") return } if len(list) >= 2 { name, ok := list[0].(string) if !ok { err = fmt.Errorf("result AMF Data[0] is not string") return } switch name { case "onPlayStatus": done, incomplete, pauseFound, err = rtmp.CheckStatus("onPlayStatus", ts, list[1], waitPause) case "onMetaData": dur, ok := objs.FindFloat64(list[1], "duration") if ok { rtmp.duration = int(dur * 1000) } else { if rtmp.isRecorded { fmt.Println("[WARN] onMetaData: duration not found") } } if meta, ok := list[1].(map[string]interface{}); ok { rtmp.writeMetaData(meta, ts) } _, ok = objs.Find(list[1], "videoframerate") if ok { rtmp.VideoExists = true } } } case TID_AMF0COMMAND, TID_AMF3COMMAND: objs.PrintAsJson(res) list, ok := res.([]interface{}) if !ok { err = fmt.Errorf("result AMF Command is not array") return } if len(list) >= 3 { name, ok := list[0].(string) if !ok { err = fmt.Errorf("result AMF Command name is not string") return } trIdFloat, ok := list[1].(float64) if !ok { err = fmt.Errorf("result AMF Command transaction id is not number") return } trId := int(trIdFloat) if (trId > 0) && (trId == findTrId) { trFound = true if len(list) >= 4 { trData = list[3] } } switch name { case "_error", "close": err = fmt.Errorf("AMF command not success: transaction id(%d) -> %s", trId, name) return case "onStatus": done, incomplete, pauseFound, err = rtmp.CheckStatus("onStatus", ts, list[3], waitPause) } } case TID_SETCHUNKSIZE: rtmp.chunkSizeRecv = res.(int) case TID_WINDOW_ACK_SIZE: rtmp.windowSize = res.(int) case TID_USERCONTROL: switch res.([]int)[0] { case UC_PINGREQUEST: //fmt.Printf("ping request %d\n", res.([]int)[1]) if err = rtmp.pingResponse(res.([]int)[1]); err != nil { return } case UC_STREAMBEGIN: rtmp.streamId = res.([]int)[1] case UC_STREAMISRECORDED: fmt.Printf("stream is recorded\n") rtmp.isRecorded = true case UC_BUFFEREMPTY: if rtmp.isRecorded { fmt.Printf("required Seek: %d\n", rtmp.timestamp) // <-- test rtmp.PauseRaw() incomplete = true return // test --> if rtmp.noSeek { incomplete = true return } ts := rtmp.timestamp - 10000 if ts < 0 { ts = 0 } done, incomplete, err = rtmp.PauseUnpause(ts) if done || incomplete || err != nil { return } //rtmp.Seek(ts) } } default: //fmt.Printf("got: %8d %d %#v\n", ts, msg_t, res) } return } func (rtmp *Rtmp) Close() (err error) { if rtmp.conn != nil { err = rtmp.conn.Close() } if rtmp.flv != nil { rtmp.flv.Close() } return } func (rtmp *Rtmp) SetPeerBandwidth(wsz, lim int) (err error) { buff, err := encodeSetPeerBandwidth(wsz, lim) if err != nil { return } if _, err = buff.WriteTo(rtmp.conn); err != nil { return } return } func (rtmp *Rtmp) pingResponse(timestamp int) (err error) { buff, err := encodePingResponse(timestamp) if _, err = buff.WriteTo(rtmp.conn); err != nil { return } return } func (rtmp *Rtmp) acknowledgement() (err error) { buff, err := encodeAcknowledgement(rtmp.totalReadBytes) if _, err = buff.WriteTo(rtmp.conn); err != nil { return } return } func (rtmp *Rtmp) WindowAckSize(asz int) (err error) { buff, err := encodeWindowAckSize(asz) if _, err = buff.WriteTo(rtmp.conn); err != nil { return } return } func (rtmp *Rtmp) SetBufferLength(streamId, len int) (err error) { buff, err := encodeSetBufferLength(streamId, len) if _, err = buff.WriteTo(rtmp.conn); err != nil { return } return } // command name, transaction ID, and command object func (rtmp *Rtmp) Command(name string, args []interface{}) (trData interface{}, err error) { var trId int var csId int var streamId int switch name { case "connect": rtmp.transactionId = 1 trId = rtmp.transactionId csId = 3 streamId = 0 case "play", "seek", "pause", "pauseRaw": trId = 0 csId = 8 streamId = 1 default: // createStream, call, close, ... rtmp.transactionId++ trId = rtmp.transactionId csId = 3 streamId = 0 } cmd := []interface{}{name, trId} cmd = append(cmd, args...) objs.PrintAsJson(cmd) body, err := amf.EncodeAmf0(cmd, false) wbuff, err := amf0Command(rtmp.chunkSizeSend, csId, streamId, body) if _, err = wbuff.WriteTo(rtmp.conn); err != nil { return } if trId > 0 { if _, _, trData, err = rtmp.waitCommand(trId); err != nil { return } } return } func (rtmp *Rtmp) Unpause(timestamp int) (err error) { var data []interface{} data = append(data, nil) data = append(data, false) data = append(data, timestamp) _, err = rtmp.Command("pause", data) return } func (rtmp *Rtmp) Pause(timestamp int) (err error) { var data []interface{} data = append(data, nil) data = append(data, true) data = append(data, timestamp) _, err = rtmp.Command("pause", data) return } func (rtmp *Rtmp) PauseRaw() (err error) { _, err = rtmp.Command("pauseRaw", []interface{}{ nil, true, 0, }) return } func (rtmp *Rtmp) PauseUnpause(timestamp int) (done, incomplete bool, err error) { if err = rtmp.Pause(timestamp); err != nil { return } fmt.Println("paused") done, incomplete, err = rtmp.WaitPause() if done || incomplete || err != nil { return } fmt.Println("wait pause") if err = rtmp.Unpause(timestamp); err != nil { return } fmt.Println("Unpaused") return } func (rtmp *Rtmp) PlayTime(stream string, timestamp int) (err error) { rtmp.startTime = timestamp if rtmp.startTime < 0 { rtmp.startTime = 0 } //fmt.Printf("debug rtmp.startTime: %d\n", rtmp.startTime) var data []interface{} data = append(data, nil) data = append(data, stream) data = append(data, timestamp) // Start // NicoOfficialTs, Never append Duration and flush if rtmp.flush { data = append(data, -1) // Duration data = append(data, true) // flush } _, err = rtmp.Command("play", data) return } func (rtmp *Rtmp) Play(stream string) error { return rtmp.PlayTime(stream, -5000) } func (rtmp *Rtmp) Seek(timestamp int) (err error) { //fmt.Printf("debug Seek to %d\n", timestamp) var data []interface{} data = append(data, nil) data = append(data, timestamp) _, err = rtmp.Command("seek", data) //fmt.Printf("debug Seek done\n") return } func (rtmp *Rtmp) CreateStream() (err error) { var data []interface{} data = append(data, nil) _, err = rtmp.Command("createStream", data) return } func handshake(conn *net.TCPConn) (err error) { wbuff := bytes.NewBuffer(nil) // C0 wbuff.WriteByte(3) // C1 rnd := rand.New(rand.NewSource(time.Now().UnixNano())) io.CopyN(wbuff, rnd, 1536) // Send C0+C1 if _, err = wbuff.WriteTo(conn); err != nil { return } // Recv S0 if _, err = io.CopyN(ioutil.Discard, conn, 1); err != nil { return } // Recv S1 if _, err = io.CopyN(wbuff, conn, 1536); err != nil { return } // Send C2(=S1) if _, err = wbuff.WriteTo(conn); err != nil { return } // Recv S2 if _, err = io.CopyN(ioutil.Discard, conn, 1536); err != nil { return } return } ================================================ FILE: src/twitcas/twicas.go ================================================ package twitcas import ( "database/sql" "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "net/url" "os" "os/exec" "time" "github.com/gorilla/websocket" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/procs/ffmpeg" _ "github.com/mattn/go-sqlite3" ) type Twitcas struct { Conn *websocket.Conn } func connectStream(proto, host, mode string, id uint64, proxy string) (conn *websocket.Conn, err error) { streamUrl := fmt.Sprintf( //case A.InnerFrame:return"i"; //case A.Pframe:return"p"; //case A.DisposableProfile:return"bd"; //case A.Bframe:return"b"; //case A.Any:return"any"; //case A.KeyFrame:default:return"k"} //"%s://%s/ws.app/stream/%d/fmp4/k/0/1?mode=%s", "%s://%s/ws.app/stream/%d/fmp4/bd/1/1500?mode=%s", proto, host, id, mode, ) // fmt.Println(streamUrl) var origin string if proto == "wss" { origin = fmt.Sprintf("https://%s", host) } else { origin = fmt.Sprintf("http://%s", host) } header := http.Header{} header.Set("Origin", origin) //header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36") header.Set("User-Agent", httpbase.GetUserAgent()) timeout, _ := time.ParseDuration("10s") dialer := websocket.Dialer{ HandshakeTimeout: timeout, } if proxy != "" { dialer.Proxy = func(req *http.Request) (u *url.URL, err error) { var proxyUrl string if proto == "wss" { proxyUrl = fmt.Sprintf("https://%s", proxy) } else { proxyUrl = fmt.Sprintf("http://%s", proxy) } return url.ParseRequestURI(proxyUrl) } } conn, _, err = dialer.Dial(streamUrl, header) return } func getStream(user, proxy string) (conn *websocket.Conn, movieId uint64, err error) { url := fmt.Sprintf( "https://twitcasting.tv/streamserver.php?target=%s&mode=client", user, ) type StreamServer struct { Movie struct { Id uint64 `json:"id"` Live bool `json:"live"` } `json:"movie"` Fmp4 struct { Host string `json:"host"` Proto string `json:"proto"` Source bool `json:"source"` MobileSource bool `json:"mobilesource"` } `json:"fmp4"` } req, err := http.NewRequest("GET", url, nil) if err != nil { return } client := new(http.Client) client.Timeout, _ = time.ParseDuration("10s") resp, err := client.Do(req) if err != nil { return } defer resp.Body.Close() respBytes, err := ioutil.ReadAll(resp.Body) if err != nil { return } //fmt.Printf("debug %s\n", string(respBytes)) data := new(StreamServer) err = json.Unmarshal(respBytes, data) if err != nil { return } if !data.Movie.Live { // movie not active err = errors.New(user + " --> " + "Offline or User Not Found") return } else { var mode string if data.Fmp4.Source { // StreamQuality.High mode = "main" } else if data.Fmp4.MobileSource { // StreamQuality.Middle mode = "mobilesource" } else { // StreamQuality.Low mode = "base" } if data.Fmp4.Proto != "" && data.Fmp4.Host != "" && data.Movie.Id != 0 { conn, err = connectStream(data.Fmp4.Proto, data.Fmp4.Host, mode, data.Movie.Id, proxy) if err != nil { return } movieId = data.Movie.Id } else { err = errors.New(user + " --> " + "No Stream Defined") return } } return } func createFileUser(user string, movieId uint64) (f *os.File, filename string, err error) { user = files.ReplaceForbidden(user) filename = fmt.Sprintf("%s_%d.mp4", user, movieId) for i := 2; i < 1000; i++ { _, err := os.Stat(filename) if err != nil { break } filename = fmt.Sprintf("%s_%d_%d.mp4", user, movieId, i) } f, err = os.Create(filename) return } // FIXME: return codeの整理 func TwitcasRecord(user, proxy string) (done, dbLocked bool) { conn, movieId, err := getStream(user, proxy) if err != nil { fmt.Printf("@err getStream: %v\n", err) return } if conn == nil { fmt.Println("[FIXME] conn is nil") return } defer conn.Close() dbName := fmt.Sprintf("tmp/tcas-%v-lock.db", movieId) files.MkdirByFileName(dbName) db, err := sql.Open("sqlite3", dbName) if err != nil { fmt.Println(err) return } defer db.Close() _, err = db.Exec(`BEGIN EXCLUSIVE`) if err != nil { dbLocked = true return } defer os.Remove(dbName) //func Open(opt... string) (cmd *exec.Cmd, stdin io.WriteCloser, err error) { var cmd *exec.Cmd var stdin io.WriteCloser var fileOpened bool filenameBase := fmt.Sprintf("%s_%d.ts", user, movieId) filenameBase = files.ReplaceForbidden(filenameBase) // fixed #8 closeFF := func() { if stdin != nil { stdin.Close() } if cmd != nil { cmd.Wait() } stdin = nil cmd = nil } openFF := func() (err error) { closeFF() filename, err := files.GetFileNameNext(filenameBase) if err != nil { fmt.Println(err) return } c, in, err := ffmpeg.Open("-i", "-", "-c", "copy", "-y", filename) if err != nil { return } cmd = c stdin = in fileOpened = true return } for { conn.SetReadDeadline(time.Now().Add(60 * time.Second)) messageType, data, err := conn.ReadMessage() if err != nil { fmt.Printf("@err ReadMessage: %v\n\n", err) return } if messageType == 2 { if cmd == nil || stdin == nil { if err = openFF(); err != nil { fmt.Println(err) return } defer closeFF() } if _, err := stdin.Write(data); err != nil { fmt.Println(err) return } } else if messageType == 1 { type TextMessage struct { Code int `json:"code"` } msg := new(TextMessage) err = json.Unmarshal(data, msg) if err != nil { // json decode error fmt.Printf("@err %v\n", err) return } if (msg.Code == 100) || (msg.Code == 101) || (msg.Code == 110) { // ignore } else if msg.Code == 400 { // invalid_parameter return } else if msg.Code == 401 { // passcode_required return } else if msg.Code == 403 { //access_forbidden return } else if msg.Code == 500 { // offline return } else if msg.Code == 503 { // server_error return } else if msg.Code == 504 { // live_ended break } else { fmt.Printf("@FIXME %v\n\n", string(data)) return } } } closeFF() done = fileOpened return } ================================================ FILE: src/youtube/comment.go ================================================ package youtube import ( "context" "database/sql" "encoding/json" "fmt" "log" "os" "path/filepath" "strconv" "strings" "sync" "time" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/objs" _ "github.com/mattn/go-sqlite3" ) func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, continuation, name string) (done bool) { dbName := files.ChangeExtention(name, "yt.sqlite3") db, err := dbOpen(ctx, dbName) if err != nil { fmt.Println(err) return } defer db.Close() mtx := &sync.Mutex{} testContinuation, count, _ := dbGetContinuation(ctx, db, mtx) if testContinuation != "" { continuation = testContinuation } var printTime int64 MAINLOOP: for { select { case <-ctx.Done(): break MAINLOOP case <-sig: break MAINLOOP default: } timeoutMs, _done, err, neterr := func() (timeoutMs int, _done bool, err, neterr error) { var uri string if isReplay { uri = fmt.Sprintf("https://www.youtube.com/live_chat_replay?continuation=%s&pbj=1", continuation) } else { uri = fmt.Sprintf("https://www.youtube.com/live_chat/get_live_chat?continuation=%s&pbj=1", continuation) } code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ "Cookie": Cookie, "User-Agent": UserAgent, }) if err != nil { return } if neterr != nil { return } if code != 200 { neterr = fmt.Errorf("Status code: %v\n", code) return } var data interface{} err = json.Unmarshal(buff, &data) if err != nil { err = fmt.Errorf("json decode error") return } liveChatContinuation, ok := objs.Find(data, "response", "continuationContents", "liveChatContinuation") if !ok { err = fmt.Errorf("(response liveChatContinuation) not found") return } if actions, ok := objs.FindArray(liveChatContinuation, "actions"); ok { var videoOffsetTimeMsec string for _, a := range actions { var item interface{} var ok bool item, ok = objs.Find(a, "addChatItemAction", "item") if !ok { item, ok = objs.Find(a, "addLiveChatTickerItemAction", "item") if !ok { item, ok = objs.Find(a, "replayChatItemAction", "actions", "addChatItemAction", "item") if ok { videoOffsetTimeMsec, _ = objs.FindString(a, "replayChatItemAction", "videoOffsetTimeMsec") } } } if !ok { //objs.PrintAsJson(a) //fmt.Println("(actions item) not found") continue } var liveChatMessageRenderer interface{} liveChatMessageRenderer, ok = objs.Find(item, "liveChatTextMessageRenderer") if !ok { liveChatMessageRenderer, ok = objs.Find(item, "liveChatPaidMessageRenderer") } if !ok { continue } authorExternalChannelId, _ := objs.FindString(liveChatMessageRenderer, "authorExternalChannelId") authorName, _ := objs.FindString(liveChatMessageRenderer, "authorName", "simpleText") id, ok := objs.FindString(liveChatMessageRenderer, "id") if !ok { continue } message, _ := objs.FindString(liveChatMessageRenderer, "message", "simpleText") timestampUsec, ok := objs.FindString(liveChatMessageRenderer, "timestampUsec") if !ok { continue } if false { fmt.Printf("%v ", videoOffsetTimeMsec) fmt.Printf("%v %v %v %v %v\n", timestampUsec, authorName, authorExternalChannelId, message, id) } dbInsert(ctx, gm, db, mtx, id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation, count, ) count++ } // アーカイブ時、20秒毎に進捗を表示 if videoOffsetTimeMsec != "" { now := time.Now().Unix() if now-printTime > 20 { printTime = now if msec, e := strconv.ParseInt(videoOffsetTimeMsec, 10, 64); e == nil { total := msec / 1000 hour := total / 3600 min := (total % 3600) / 60 sec := (total % 3600) % 60 fmt.Printf("comment pos: %02d:%02d:%02d\n", hour, min, sec) } } } //fmt.Println("------------") } if continuations, ok := objs.Find(liveChatContinuation, "continuations"); ok { //objs.PrintAsJson(continuations) if c, ok := objs.FindString(continuations, "timedContinuationData", "continuation"); ok { continuation = c } else if c, ok := objs.FindString(continuations, "liveChatReplayContinuationData", "continuation"); ok { continuation = c } else if c, ok := objs.FindString(continuations, "invalidationContinuationData", "continuation"); ok { continuation = c } else if c, ok := objs.FindString(continuations, "playerSeekContinuationData", "continuation"); ok { if isReplay { _done = true return } continuation = c } else { objs.PrintAsJson(continuations) err = fmt.Errorf("(liveChatContinuation continuation) not found") return } if t, ok := objs.FindString(continuations, "timedContinuationData", "timeoutMs"); ok { timeout, err := strconv.Atoi(t) if err != nil { timeoutMs = timeout } } else if t, ok := objs.FindString(continuations, "invalidationContinuationData", "continuation"); ok { timeout, err := strconv.Atoi(t) if err != nil { timeoutMs = timeout } } } else { objs.PrintAsJson(liveChatContinuation) err = fmt.Errorf("(liveChatContinuation>continuations) not found") return } return }() if err != nil { fmt.Println(err) break } if neterr != nil { fmt.Println(neterr) break } if _done { done = true break MAINLOOP } if timeoutMs < 1000 { if isReplay { timeoutMs = 1000 } else { timeoutMs = 6000 } } time.Sleep(time.Duration(timeoutMs) * time.Millisecond) } return } func dbOpen(ctx context.Context, name string) (db *sql.DB, err error) { db, err = sql.Open("sqlite3", name) if err != nil { return } _, err = db.ExecContext(ctx, ` PRAGMA synchronous = OFF; PRAGMA journal_mode = WAL; `) if err != nil { db.Close() return } err = dbCreate(ctx, db) if err != nil { db.Close() } return } func dbCreate(ctx context.Context, db *sql.DB) (err error) { // table media _, err = db.ExecContext(ctx, ` CREATE TABLE IF NOT EXISTS comment ( id TEXT PRIMARY KEY NOT NULL UNIQUE, timestampUsec INTEGER NOT NULL, videoOffsetTimeMsec INTEGER, authorName TEXT, channelId TEXT, message TEXT, continuation TEXT, count INTEGER NOT NULL ) `) if err != nil { return } _, err = db.ExecContext(ctx, ` CREATE UNIQUE INDEX IF NOT EXISTS comment0 ON comment(id); CREATE UNIQUE INDEX IF NOT EXISTS comment1 ON comment(timestampUsec); CREATE UNIQUE INDEX IF NOT EXISTS comment2 ON comment(videoOffsetTimeMsec); CREATE UNIQUE INDEX IF NOT EXISTS comment3 ON comment(count); `) if err != nil { return } return } func dbInsert(ctx context.Context, gm *gorman.GoroutineManager, db *sql.DB, mtx *sync.Mutex, id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation string, count int) { usec, err := strconv.ParseInt(timestampUsec, 10, 64) if err != nil { fmt.Printf("ParseInt error: %s\n", timestampUsec) return } var offset interface{} if videoOffsetTimeMsec == "" { offset = nil } else { n, err := strconv.ParseInt(videoOffsetTimeMsec, 10, 64) if err != nil { offset = nil } else { offset = n } } query := `INSERT OR IGNORE INTO comment (id, timestampUsec, videoOffsetTimeMsec, authorName, channelId, message, continuation, count) VALUES (?,?,?,?,?,?,?,?)` gm.Go(func(<-chan struct{}) int { mtx.Lock() defer mtx.Unlock() if _, err := db.ExecContext(ctx, query, id, usec, offset, authorName, authorExternalChannelId, message, continuation, count, ); err != nil { if err.Error() != "context canceled" { fmt.Println(err) } return 1 } return 0 }) return } func dbGetContinuation(ctx context.Context, db *sql.DB, mtx *sync.Mutex) (res string, cnt int, err error) { mtx.Lock() defer mtx.Unlock() err = db.QueryRowContext(ctx, "SELECT continuation, count FROM comment ORDER BY count DESC LIMIT 1").Scan(&res, &cnt) return } var SelComment = `SELECT timestampUsec, IFNULL(videoOffsetTimeMsec, -1), authorName, channelId, message FROM comment ORDER BY timestampUsec ` func WriteComment(db *sql.DB, fileName string) { rows, err := db.Query(SelComment) if err != nil { log.Println(err) return } defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") dir := filepath.Dir(fileName) base := filepath.Base(fileName) base, err = files.GetFileNameNext(base) if err != nil { fmt.Println(err) os.Exit(1) } fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) } defer f.Close() fmt.Fprintf(f, "%s\r\n", ``) fmt.Fprintf(f, "%s\r\n", ``) firstOffsetUsec := int64(-1) for rows.Next() { var timestampUsec int64 var videoOffsetTimeMsec int64 var authorName string var channelId string var message string err = rows.Scan( ×tampUsec, &videoOffsetTimeMsec, &authorName, &channelId, &message, ) if err != nil { log.Println(err) return } var vpos int64 if videoOffsetTimeMsec >= 0 { vpos = videoOffsetTimeMsec / 10 } else { if firstOffsetUsec < 0 { firstOffsetUsec = timestampUsec } diff := timestampUsec - firstOffsetUsec vpos = diff / (10 * 1000) } line := fmt.Sprintf( ``) } ================================================ FILE: src/youtube/youtube.go ================================================ package youtube import ( "fmt" // "net/http" // "io/ioutil" "bufio" "context" "encoding/json" "html" "os" "os/signal" "regexp" "strings" "sync" "syscall" "time" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" "github.com/himananiito/livedl/httpbase" "github.com/himananiito/livedl/objs" "github.com/himananiito/livedl/procs" "github.com/himananiito/livedl/procs/streamlink" "github.com/himananiito/livedl/procs/youtube_dl" ) var Cookie = "PREF=f1=50000000&f4=4000000&hl=en" var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" var split = func(data []byte, atEOF bool) (advance int, token []byte, err error) { for i := 0; i < len(data); i++ { if data[i] == '\n' { return i + 1, data[:i+1], nil } if data[i] == '\r' { if (i + 1) == len(data) { return 0, nil, nil } if data[i+1] == '\n' { return i + 2, data[:i+2], nil } return i + 1, data[:i+1], nil } } if atEOF && len(data) > 0 { return len(data), data, nil } return 0, nil, nil } func getChatContinuation(buff []byte) (isReplay bool, continuation string, err error) { if ma := regexp.MustCompile(`(?s)\Wwindow\["ytInitialData"\]\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`).FindSubmatch(buff); len(ma) > 1 { var data interface{} err = json.Unmarshal(ma[1], &data) if err != nil { err = fmt.Errorf("ytInitialData parse error") return } //objs.PrintAsJson(data); liveChatRenderer, ok := objs.Find(data, "contents", "twoColumnWatchNextResults", "conversationBar", "liveChatRenderer", ) if !ok { err = fmt.Errorf("liveChatRenderer not found") return } isReplay, _ = objs.FindBool(liveChatRenderer, "isReplay") subMenuItems, ok := objs.FindArray(liveChatRenderer, "header", "liveChatHeaderRenderer", "viewSelector", "sortFilterSubMenuRenderer", "subMenuItems", ) if !ok { err = fmt.Errorf("subMenuItems not found") return } for _, item := range subMenuItems { title, _ := objs.FindString(item, "title") //selected, _ := objs.FindBool(item, "selected") c, _ := objs.FindString(item, "continuation", "reloadContinuationData", "continuation") if (title != "") && (!strings.Contains(title, "Top")) { continuation = c return } continuation = c } } else { err = fmt.Errorf("ytInitialData not found") return } if continuation == "" { err = fmt.Errorf("continuation not found") } return } func getInfo(buff []byte) (title, ucid, author string, err error) { var data interface{} re := regexp.MustCompile(`(?s)\Wytplayer\.config\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) if ma := re.FindSubmatch(buff); len(ma) > 1 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &data); err != nil { err = fmt.Errorf("ytplayer parse error") return } } else { err = fmt.Errorf("ytplayer.config not found") return } //objs.PrintAsJson(data); return title, ok := objs.FindString(data, "args", "title") if !ok { err = fmt.Errorf("title not found") return } ucid, _ = objs.FindString(data, "args", "ucid") author, _ = objs.FindString(data, "args", "author") return } func execStreamlink(gm *gorman.GoroutineManager, uri, name string) (notSupport bool, err error) { cmd, stdout, stderr, err := streamlink.Open(uri, "best", "--retry-max", "10", "-o", name) if err != nil { return } defer stdout.Close() defer stderr.Close() chStdout := make(chan string, 10) chStderr := make(chan string, 10) chEof := make(chan struct{}, 2) // stdout gm.Go(func(c <-chan struct{}) int { defer func() { chEof <- struct{}{} }() scanner := bufio.NewScanner(stdout) scanner.Split(split) for scanner.Scan() { chStdout <- scanner.Text() } return 0 }) // stderr gm.Go(func(c <-chan struct{}) int { defer func() { chEof <- struct{}{} }() scanner := bufio.NewScanner(stderr) scanner.Split(split) for scanner.Scan() { chStderr <- scanner.Text() } return 0 }) // outputs gm.Go(func(c <-chan struct{}) int { for { var s string select { case s = <-chStdout: case s = <-chStderr: case <-chEof: return 0 } if strings.HasPrefix(s, "[cli][error]") { fmt.Print(s) notSupport = true procs.Kill(cmd.Process.Pid) break } else if strings.HasPrefix(s, "Traceback (most recent call last):") { fmt.Print(s) notSupport = true //procs.Kill(cmd.Process.Pid) //break } else { fmt.Print(s) } } return 0 }) cmd.Wait() return } func execYoutube_dl(gm *gorman.GoroutineManager, uri, name string) (err error) { defer func() { part := name + ".part" if _, test := os.Stat(part); test == nil { if _, test := os.Stat(name); test != nil { os.Rename(part, name) } } }() cmd, stdout, stderr, err := youtube_dl.Open("--no-mtime", "--no-color", "-o", name, uri) if err != nil { return } defer stdout.Close() defer stderr.Close() chStdout := make(chan string, 10) chStderr := make(chan string, 10) chEof := make(chan struct{}, 2) // stdout gm.Go(func(c <-chan struct{}) int { defer func() { chEof <- struct{}{} }() scanner := bufio.NewScanner(stdout) scanner.Split(split) for scanner.Scan() { chStdout <- scanner.Text() } return 0 }) // stderr gm.Go(func(c <-chan struct{}) int { defer func() { chEof <- struct{}{} }() scanner := bufio.NewScanner(stderr) scanner.Split(split) for scanner.Scan() { chStderr <- scanner.Text() } return 0 }) // outputs gm.Go(func(c <-chan struct{}) int { var old int64 for { var s string select { case s = <-chStdout: case s = <-chStderr: case <-chEof: return 0 } if strings.HasPrefix(s, "[https @ ") { // ffmpeg unwanted logs } else { if strings.HasPrefix(s, "[download]") { var now = time.Now().UnixNano() if now-old > 2*1000*1000*1000 { old = now } else { continue } } fmt.Print(s) } } return 0 }) cmd.Wait() return } var COMMENT_DONE = 1000 func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { uri := fmt.Sprintf("https://www.youtube.com/watch?v=%s", id) code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ "Cookie": Cookie, "User-Agent": UserAgent, }) if err != nil { return } if neterr != nil { return } if code != 200 { neterr = fmt.Errorf("Status code: %v\n", code) return } title, ucid, author, err := getInfo(buff) if err != nil { return } if false { fmt.Println(ucid) } isReplay, continuation, err := getChatContinuation(buff) origName := fmt.Sprintf("%s-%s_%s.mp4", author, title, id) origName = files.ReplaceForbidden(origName) name, err := files.GetFileNameNext(origName) if err != nil { fmt.Println(err) return } fmt.Println(name) mtxComDone := &sync.Mutex{} var commentDone bool var gm *gorman.GoroutineManager var gmCom *gorman.GoroutineManager gm = gorman.WithChecker(func(c int) { switch c { case 0: default: gm.Cancel() if gmCom != nil { gmCom.Cancel() } } }) gmCom = gorman.WithChecker(func(c int) { switch c { case 0: case COMMENT_DONE: func() { mtxComDone.Lock() defer mtxComDone.Unlock() commentDone = true }() default: gmCom.Cancel() } }) chInterrupt := make(chan os.Signal, 10) signal.Notify(chInterrupt, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) defer signal.Stop(chInterrupt) ctx, cancel := context.WithCancel(context.Background()) var interrupt bool gm.Go(func(c <-chan struct{}) int { select { case <-chInterrupt: interrupt = true case <-c: } cancel() gm.Cancel() return 1 }) if continuation != "" { gmCom.Go(func(c <-chan struct{}) int { getComment(gmCom, ctx, c, isReplay, continuation, origName) fmt.Printf("\ncomment done\n") return COMMENT_DONE }) } gm.Go(func(c <-chan struct{}) int { select { case <-c: cancel() } return 0 }) var retry bool if !ytNoStreamlink { retry, err = execStreamlink(gm, uri, name) } if !interrupt { if err != nil || retry || (ytNoStreamlink && (!ytNoYoutube_dl)) { execYoutube_dl(gm, uri, name) } } if continuation != "" { if isReplay { if !commentDone { fmt.Printf("\nwaiting comment\n") gmCom.Wait() } else { gmCom.Wait() } } else { gmCom.Cancel() gmCom.Wait() } } gm.Cancel() gm.Wait() return } ================================================ FILE: src/youtube/youtube.gox ================================================ package youtube import ( "fmt" "net/http" "io/ioutil" "regexp" "encoding/json" "html" "strings" "net/url" "os" "strconv" "bytes" "os/exec" "os/signal" "archive/zip" "sync" "../obj" "io" "../files" "../httpsub" "../zip2mp4" "log" ) type YtDash struct { SeqNo int SeqNoFound bool SeqNoBack int VAddr string VQuery url.Values AAddr string AQuery url.Values TsFile *os.File FFCmd *exec.Cmd FFBuffer *bytes.Buffer TryBack bool StartBack bool ChEnd chan bool ChEndBack chan bool zipFile *os.File zipWriter *zip.Writer mZip sync.Mutex fileName string Title string Id string } func (yt *YtDash) SetFileName(fileName string) { yt.fileName = files.ReplaceForbidden(fileName) } func (yt *YtDash) fetch(isVideo, isBack bool) (fileName string, err error) { var addr string var query url.Values var sn int if isVideo { addr = yt.VAddr query = yt.VQuery } else { addr = yt.AAddr query = yt.AQuery } if isBack && (! yt.SeqNoFound) { err = fmt.Errorf("isBack && (! SeqNoFound)") return } if yt.SeqNoFound { //fmt.Printf("SQ set to %d\n", yt.SeqNo) if isBack { sn = yt.SeqNoBack } else { sn = yt.SeqNo } query.Set("sq", fmt.Sprintf("%d", sn)) //fmt.Printf("%v\n", query) } uri := fmt.Sprintf("%s?%s", addr, query.Encode()) req, _ := http.NewRequest("GET", uri, nil) client := new(http.Client) resp, err := client.Do(req) if err != nil { return } defer resp.Body.Close() switch resp.StatusCode { case 200: default: err = fmt.Errorf("StatusCode is %v\n%v\n%v", resp.StatusCode, uri, query) return } switch query.Get("source") { case "yt_live_broadcast": bs, e := ioutil.ReadAll(resp.Body) if e != nil { err = e return } if (! yt.SeqNoFound) && (! isBack) { if ma := regexp.MustCompile(`Sequence-Number\s*:\s*(\d+)`).FindSubmatch(bs); len(ma) > 0 { sn, err = strconv.Atoi(string(ma[1])) if err != nil { err = fmt.Errorf("Sequence-Number Not a Number: %v", ma) return } yt.SeqNo = sn yt.SeqNoBack = sn - 1 yt.SeqNoFound = true fmt.Printf("start SeqNo: %d\n", sn) } else { err = fmt.Errorf("Sequence-Number Not found") return } yt.RecordBack() } if isVideo { fileName = fmt.Sprintf("video-%d.mp4", sn) } else { fileName = fmt.Sprintf("audio-%d.mp4", sn) } buff := bytes.NewBuffer(bs) if err = yt.WriteZip(fileName, buff); err != nil { return } } return } func (yt *YtDash) fetchVideo() (string, error) { return yt.fetch(true, false) } func (yt *YtDash) fetchAudio() (string, error) { return yt.fetch(false, false) } func (yt *YtDash) IncrSeqNo() { yt.SeqNo++ } func (yt *YtDash) fetchVideoBack() (string, error) { return yt.fetch(true, true) } func (yt *YtDash) fetchAudioBack() (string, error) { return yt.fetch(false, true) } func (yt *YtDash) DecrSeqNoBack() { yt.SeqNoBack-- } func (yt *YtDash) RecordYoutube() { var vname string var aname string func() { uri := fmt.Sprintf("%s?%s", yt.VAddr, yt.VQuery.Encode()) vname = fmt.Sprintf("%s(%s)-v.mp4", yt.Title, yt.Id) fmt.Println(uri) sub := httpsub.Get(uri, vname) sub.Concurrent(4) sub.Wait() }() func() { uri := fmt.Sprintf("%s?%s", yt.AAddr, yt.AQuery.Encode()) aname = fmt.Sprintf("%s(%s)-a.mp4", yt.Title, yt.Id) sub := httpsub.Get(uri, aname) sub.Concurrent(4) sub.Wait() }() if zip2mp4.FFmpegExists() { exts := []string{"mp4", "mkv"} for _, ext := range exts { oname := fmt.Sprintf("%s(%s).%s", yt.Title, yt.Id, ext) if zip2mp4.MergeVA(vname, aname, oname) { os.Remove(vname) os.Remove(aname) return } else { os.Remove(oname) } } } // ffmpeg Not exists OR merge NG fv, e := os.Open(vname) if e != nil { log.Fatalln(e) } yt.WriteZip("video.mp4", fv) fv.Close() fa, e := os.Open(aname) if e != nil { log.Fatalln(e) } yt.WriteZip("audio.mp4", fa) fa.Close() os.Remove(vname) os.Remove(aname) } func (yt *YtDash) Wait() { yt.ChEnd = make(chan bool) yt.ChEndBack = make(chan bool) switch yt.VQuery.Get("source") { case "youtube": yt.RecordYoutube() case "yt_live_broadcast": yt.RecordForward() <-yt.ChEnd if yt.StartBack { <-yt.ChEndBack } } } func (yt *YtDash) Close() { if yt.zipWriter != nil { yt.zipWriter.Close() } if yt.zipFile != nil { yt.zipFile.Close() } } func (yt *YtDash) OpenFile() (err error) { fileName, err := files.GetFileNameNext(yt.fileName) if err != nil { return } file, err := os.Create(fileName) if err != nil { log.Fatalln(err) } yt.zipFile = file yt.zipWriter = zip.NewWriter(file) chSig := make(chan os.Signal, 1) signal.Notify(chSig, os.Interrupt) go func() { <-chSig yt.mZip.Lock() defer yt.mZip.Unlock() if yt.zipWriter != nil { yt.zipWriter.Close() } os.Exit(0) }() return } func (yt *YtDash) WriteZip(name string, rdr io.Reader) (err error) { yt.mZip.Lock() defer yt.mZip.Unlock() if yt.zipFile == nil || yt.zipWriter == nil { yt.OpenFile() } wr, err := yt.zipWriter.Create(name) if err != nil { return } if _, err = io.Copy(wr, rdr); err != nil { return } return } func (yt *YtDash) RecordForward() { go func() { defer func() { close(yt.ChEnd) }() for { vfile, err := yt.fetchVideo() if err != nil { fmt.Printf("RecordForward: %v\n", err) return } afile, err := yt.fetchAudio() if err != nil { fmt.Printf("RecordForward: %v\n", err) return } if true { fmt.Printf("%s, %s\n", vfile, afile) } yt.IncrSeqNo() } }() } func (yt *YtDash) RecordBack() { if yt.TryBack && (! yt.StartBack) { yt.StartBack = true go func() { defer func() { close(yt.ChEndBack) }() for yt.SeqNoBack >= 0 { vfile, err := yt.fetchVideoBack() if err != nil { fmt.Printf("RecordBack: %v\n", err) return } afile, err := yt.fetchAudioBack() if err != nil { fmt.Printf("RecordBack: %v\n", err) return } if true { fmt.Printf("%s, %s\n", vfile, afile) } yt.DecrSeqNoBack() } }() } } func Record(id string) (err error) { uri := fmt.Sprintf("https://www.youtube.com/watch?v=%s", id) req, _ := http.NewRequest("GET", uri, nil) client := new(http.Client) resp, err := client.Do(req) if err != nil { fmt.Println(err) return } defer resp.Body.Close() dat, _ := ioutil.ReadAll(resp.Body) var a interface{} re := regexp.MustCompile(`\Wytplayer\.config\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) if ma := re.FindSubmatch(dat); len(ma) > 0 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &a); err != nil { fmt.Println(str) fmt.Println(err) return } } else { fmt.Println("ytplayer.config not found") return } // debug print //obj.PrintAsJson(a) title, ok := obj.FindString(a, "args", "title") if (! ok) { fmt.Println("title not found") return } res, ok := obj.FindString(a, "args", "adaptive_fmts") if (! ok) { if res, ok := obj.FindString(a, "args", "hlsvp"); ok { fmt.Printf("hls: %s\n", res) return } obj.PrintAsJson(a) return } var maxVideoBr int var maxAudioBr int var videoUrl string var audioUrl string var qualityLabel string for _, s := range strings.Split(res, ",") { //fmt.Println(s) f, e := url.ParseQuery(s) //obj.PrintAsJson(f) //fmt.Println(f) if e != nil { fmt.Println(e) return } // type // bitrate t := f.Get("type") br, err := strconv.Atoi(f.Get("bitrate")) if err != nil { continue } if strings.HasPrefix(t, "video") { if br > maxVideoBr { maxVideoBr = br videoUrl = f.Get("url") qualityLabel = f.Get("quality_label") } } else if strings.HasPrefix(t, "audio") { if br > maxAudioBr { maxAudioBr = br audioUrl = f.Get("url") } } } fmt.Printf("Quality: %s\n", qualityLabel) varr := strings.SplitN(videoUrl, "?", 2) if len(varr) != 2 { return } aarr := strings.SplitN(audioUrl, "?", 2) if len(aarr) != 2 { return } yt := new(YtDash) defer yt.Close() yt.Id = id yt.Title = files.ReplaceForbidden(title) yt.SetFileName(fmt.Sprintf("%s(%s).zip", title, id)) yt.VAddr = varr[0] vQuery, e := url.ParseQuery(varr[1]) if e != nil { return } yt.VQuery = vQuery //obj.PrintAsJson(vQuery) //fmt.Println(yt.VAddr + "?" + vQuery.Encode()) yt.AAddr = aarr[0] aQuery, e := url.ParseQuery(aarr[1]) if e != nil { return } yt.AQuery = aQuery yt.TryBack = true yt.Wait() return } ================================================ FILE: src/zip2mp4/zip2mp4.go ================================================ package zip2mp4 import ( "archive/zip" "bytes" "database/sql" "fmt" "io" "io/ioutil" "log" "os" "os/exec" "regexp" "sort" "strconv" "time" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/log4gui" "github.com/himananiito/livedl/niconico" "github.com/himananiito/livedl/procs/ffmpeg" "github.com/himananiito/livedl/youtube" _ "github.com/mattn/go-sqlite3" ) type ZipMp4 struct { ZipName string Mp4NameOpened string mp4List []string FFMpeg *exec.Cmd FFStdin io.WriteCloser } var cmdListFF = []string{ "./bin/ffmpeg/ffmpeg", "./bin/ffmpeg", "./ffmpeg/ffmpeg", "./ffmpeg", "ffmpeg", } var cmdListMP42TS = []string{ "./bin/bento4/bin/mp42ts", "./bento4/bin/mp42ts", "./bento4/mp42ts", "./bin/bento4/mp42ts", "./bin/mp42ts", "./mp42ts", "mp42ts", } // return cmd = nil if cmd not exists func openProg(cmdList *[]string, stdinEn, stdoutEn, stdErrEn, consoleEn bool, args []string) (cmd *exec.Cmd, stdin io.WriteCloser, stdout, stderr io.ReadCloser) { for i, cmdName := range *cmdList { cmd = exec.Command(cmdName, args...) var err error if stdinEn { stdin, err = cmd.StdinPipe() if err != nil { log.Fatalln(err) } } if stdoutEn { stdout, err = cmd.StdoutPipe() if err != nil { log.Fatalln(err) } } else { if consoleEn { cmd.Stdout = os.Stdout } } if stdErrEn { stderr, err = cmd.StderrPipe() if err != nil { log.Fatalln(err) } } else { if consoleEn { cmd.Stderr = os.Stderr } } if err = cmd.Start(); err != nil { continue } else { if i != 0 { *cmdList = []string{cmdName} } //fmt.Printf("CMD: %#v\n", cmd.Args) return } } cmd = nil return } func MergeVA(vFileName, aFileName, oFileName string) bool { cmd, _, _, _ := openProg(&cmdListFF, false, false, false, true, []string{ "-i", vFileName, "-i", aFileName, "-c", "copy", "-y", oFileName, }) if cmd == nil { return false } if err := cmd.Wait(); err != nil { fmt.Println(err) return false } return true } func FFmpegExists() bool { cmd, _, _, _ := openProg(&cmdListFF, false, false, false, false, []string{"-version"}) if cmd == nil { return false } cmd.Wait() return true } func GetFormat(fileName string) (vFormat, aFormat string) { cmd, _, stdout, stderr := openProg(&cmdListFF, false, true, true, false, []string{"-i", fileName}) if cmd == nil { return } b1, _ := ioutil.ReadAll(stdout) b2, _ := ioutil.ReadAll(stderr) cmd.Wait() s := string(b1) + string(b2) if ma := regexp.MustCompile(`(?i)Stream\s+#.+?:\s+Video:\s+(.*?),`).FindStringSubmatch(s); len(ma) > 0 { vFormat = ma[1] } if ma := regexp.MustCompile(`(?i)Stream\s+#.+?:\s+Audio:\s+(.*?),`).FindStringSubmatch(s); len(ma) > 0 { aFormat = ma[1] } return } func openFFMpeg(stdinEn, stdoutEn, stdErrEn, consoleEn bool, args []string) (cmd *exec.Cmd, stdin io.WriteCloser, stdout, stderr io.ReadCloser) { return openProg(&cmdListFF, stdinEn, stdoutEn, stdErrEn, consoleEn, args) } func openMP42TS(consoleEn bool, args []string) (cmd *exec.Cmd) { cmd, _, _, _ = openProg(&cmdListMP42TS, false, false, false, consoleEn, args) return } func (z *ZipMp4) Wait() { if z.FFStdin != nil { z.FFStdin.Close() } if z.FFMpeg != nil { if err := z.FFMpeg.Wait(); err != nil { log.Fatalln(err) } z.FFMpeg = nil } } func (z *ZipMp4) CloseFFInput() { z.FFStdin.Close() } func (z *ZipMp4) OpenFFMpeg(ext string) { // z.Wait() if ext == "" { ext = "mp4" } name := files.ChangeExtention(z.ZipName, ext) name, err := files.GetFileNameNext(name) if err != nil { fmt.Println(err) os.Exit(1) } z.Mp4NameOpened = name z.mp4List = append(z.mp4List, name) cmd, stdin, err := ffmpeg.Open( "-i", "-", "-c", "copy", //"-movflags", "faststart", // test "-y", name, ) if err != nil { log.Fatalln(err) } z.FFMpeg = cmd z.FFStdin = stdin } func (z *ZipMp4) FFInputCombFromFile(videoFile, audioFile string) { vTs := fmt.Sprintf("%s.ts", videoFile) cmdV := openMP42TS(false, []string{ videoFile, vTs, }) if cmdV == nil { fmt.Println("mp42ts not found OR command failed") os.Exit(1) } defer os.Remove(vTs) aTs := fmt.Sprintf("%s.ts", audioFile) cmdA := openMP42TS(false, []string{ audioFile, aTs, }) if cmdA == nil { fmt.Println("mp42ts not found OR command failed") os.Exit(1) } defer os.Remove(aTs) if err := cmdV.Wait(); err != nil { log.Fatalln(err) } if err := cmdA.Wait(); err != nil { log.Fatalln(err) } cmd, _, stdout, _ := openFFMpeg(false, true, false, false, []string{ "-i", vTs, "-i", aTs, "-c", "copy", "-f", "mpegts", "-", }) if cmd == nil { log.Fatalln("ffmpeg not installed") } z.FFInput(stdout) if err := cmd.Wait(); err != nil { log.Fatalln(err) } } func (z *ZipMp4) FFInput(rdr io.Reader) { if _, err := io.Copy(z.FFStdin, rdr); err != nil { log.Fatalln(err) } } type Index struct { int } type Chunk struct { VideoIndex *Index AudioIndex *Index VAIndex *Index } func Convert(fileName string) (err error) { zr, err := zip.OpenReader(fileName) if err != nil { return } chunks := make(map[int64]Chunk) for i, r := range zr.File { //fmt.Printf("X %v %v\n", i, r.Name) if ma := regexp.MustCompile(`\Avideo-(\d+)\.\w+\z`).FindStringSubmatch(r.Name); len(ma) > 0 { num, err := strconv.ParseInt(string(ma[1]), 10, 64) if err != nil { log.Fatal(err) } if v, ok := chunks[num]; ok { v.VideoIndex = &Index{i} chunks[num] = v } else { chunks[num] = Chunk{VideoIndex: &Index{i}} } //fmt.Printf("V %v %v\n", i, r.Name) } else if ma := regexp.MustCompile(`\Aaudio-(\d+)\.\w+\z`).FindStringSubmatch(r.Name); len(ma) > 0 { num, err := strconv.ParseInt(string(ma[1]), 10, 64) if err != nil { log.Fatal(err) } if v, ok := chunks[num]; ok { v.AudioIndex = &Index{i} chunks[num] = v } else { chunks[num] = Chunk{AudioIndex: &Index{i}} } //fmt.Printf("A %v %v\n", num, r.Name) } else if ma := regexp.MustCompile(`\A(\d+)\.\w+\z`).FindStringSubmatch(r.Name); len(ma) > 0 { num, err := strconv.ParseInt(string(ma[1]), 10, 64) if err != nil { log.Fatal(err) } if v, ok := chunks[num]; ok { v.VAIndex = &Index{i} chunks[num] = v } else { chunks[num] = Chunk{VAIndex: &Index{i}} } //fmt.Printf("V+A %v %v\n", num, r.Name) } else { fmt.Printf("%v %v\n", i, r.Name) log4gui.Info(fmt.Sprintf("Unsupported zip: %s", fileName)) os.Exit(1) } } keys := make([]int64, 0, len(chunks)) for k := range chunks { keys = append(keys, k) } sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) var tmpVideoName string var tmpAudioName string var zm *ZipMp4 defer func() { if zm != nil { zm.CloseFFInput() zm.Wait() } }() zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg("mp4") prevIndex := int64(-1) for _, key := range keys { if prevIndex >= 0 { if key != prevIndex+1 { // [FIXME] reopen new mp4file? //return fmt.Errorf("\n\nError: seq skipped: %d --> %d\n\n", prevIndex, key) fmt.Printf("\nSeqNo. skipped: %d --> %d\n", prevIndex, key) if zm != nil { zm.CloseFFInput() zm.Wait() } zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg("mp4") } } prevIndex = key if chunks[key].VAIndex != nil { r, e := zr.File[chunks[key].VAIndex.int].Open() if e != nil { log.Fatalln(e) } zm.FFInput(r) r.Close() } else if chunks[key].VideoIndex != nil && chunks[key].AudioIndex != nil { if tmpVideoName == "" { f, e := ioutil.TempFile(".", "__temp-") if e != nil { log.Fatalln(e) } f.Close() tmpVideoName = f.Name() } if tmpAudioName == "" { f, e := ioutil.TempFile(".", "__temp-") if e != nil { log.Fatalln(e) } f.Close() tmpAudioName = f.Name() } // open temporary file tmpVideo, err := os.Create(tmpVideoName) if err != nil { log.Fatalln(err) } tmpAudio, err := os.Create(tmpAudioName) if err != nil { log.Fatalln(err) } // copy Video to file rv, e := zr.File[chunks[key].VideoIndex.int].Open() if e != nil { log.Fatalln(e) } if _, e := io.Copy(tmpVideo, rv); e != nil { log.Fatalln(e) } rv.Close() tmpVideo.Close() // copy Audio to file ra, e := zr.File[chunks[key].AudioIndex.int].Open() if e != nil { log.Fatalln(e) } if _, e := io.Copy(tmpAudio, ra); e != nil { log.Fatalln(e) } ra.Close() tmpAudio.Close() // combine video + audio using ffmpeg(+mp42ts) zm.FFInputCombFromFile(tmpVideoName, tmpAudioName) os.Remove(tmpVideoName) os.Remove(tmpAudioName) } else { if (chunks[key].VideoIndex == nil && chunks[key].AudioIndex != nil) || (chunks[key].VideoIndex != nil && chunks[key].AudioIndex == nil) { fmt.Printf("\nIncomplete sequence. skipped: %d\n", key) if zm != nil { zm.CloseFFInput() zm.Wait() } zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg("mp4") } } } zm.CloseFFInput() zm.Wait() fmt.Printf("\nfinish: %s\n", zm.Mp4NameOpened) return } func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() niconico.WriteComment(db, fileName, skipHb) rows, err := db.Query(niconico.SelMedia) if err != nil { return } defer rows.Close() dir := files.RemoveExtention(fileName) if err = files.MkdirByFileName(dir + "/"); err != nil { return } var printTime int64 for rows.Next() { var seqno int64 var bw int var size int var data []byte err = rows.Scan(&seqno, &bw, &size, &data) if err != nil { return } name := fmt.Sprintf("%s/%d.ts", dir, seqno) // print now := time.Now().Unix() if now != printTime { printTime = now fmt.Println(name) } err = func() (e error) { f, e := os.Create(name) if e != nil { return } defer f.Close() _, e = f.Write(data) return }() if err != nil { return } } done = true return } func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() niconico.WriteComment(db, fileName, skipHb) var zm *ZipMp4 defer func() { if zm != nil { //zm.CloseFFInput() zm.Wait() } }() zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg(ext) rows, err := db.Query(niconico.SelMedia) if err != nil { return } defer rows.Close() prevBw := -1 prevIndex := int64(-1) for rows.Next() { var seqno int64 var bw int var size int var data []byte err = rows.Scan(&seqno, &bw, &size, &data) if err != nil { return } // チャンクが飛んでいる場合はファイルを分ける // BANDWIDTHが変わる場合はファイルを分ける if (prevIndex >= 0 && seqno != prevIndex+1) || (prevBw >= 0 && bw != prevBw) { if bw != prevBw { fmt.Printf("\nBandwitdh changed: %d --> %d\n\n", prevBw, bw) } else { fmt.Printf("\nSeqNo. skipped: %d --> %d\n\n", prevIndex, seqno) } //if zm != nil { // zm.CloseFFInput() // zm.Wait() //} zm.OpenFFMpeg(ext) } prevBw = bw prevIndex = seqno zm.FFInput(bytes.NewBuffer(data)) } //zm.CloseFFInput() zm.Wait() fmt.Printf("\nfinish:\n") for _, s := range zm.mp4List { fmt.Println(s) } done = true nMp4s = len(zm.mp4List) return } func YtComment(fileName string) (done bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() youtube.WriteComment(db, fileName) return } ================================================ FILE: updatebuildno.go ================================================ package main import ( "log" "os" "io/ioutil" "time" "strconv" "regexp" "fmt" ) func main() { f, err := os.OpenFile("src/buildno/buildno.go", os.O_RDWR, 0755) if err != nil { log.Fatal(err) } defer f.Close() if _, err := f.Seek(0, 0); err != nil { log.Fatal(err) } data, err := ioutil.ReadAll(f) if err != nil { log.Fatal(err) } var buildNo int64 if ma := regexp.MustCompile(`BuildNo\s*=\s*"(\d+)"`).FindSubmatch(data); len(ma) > 0 { buildNo, err = strconv.ParseInt(string(ma[1]), 10, 64) if err != nil { log.Fatal(err) } } else { log.Fatal("BuildNo not match") } buildNo++ var now = time.Now() buildDate := fmt.Sprintf("%04d%02d%02d", now.Year(), now.Month(), now.Day(), ) fmt.Printf("%v.%v\n", buildDate, buildNo) if _, err := f.Seek(0, 0); err != nil { log.Fatal(err) } if err := f.Truncate(0); err != nil { log.Fatal(err) } f.WriteString(fmt.Sprintf(` package buildno var BuildDate = "%s" var BuildNo = "%d" `, buildDate, buildNo)) }