[ { "path": ".gitattributes", "content": "*.go text eol=lf\n*.md text eol=lf\n" }, { "path": ".github/ISSUE_TEMPLATE/can-not-download-video.md", "content": "---\nname: Can not download video\nabout: If you find that a website download is not working\ntitle: \"[download fail]: the website name\"\nlabels: bug\nassignees: ''\n\n---\n\n\n\n\n**Website name**: name\n\n\n\n\n**OS:**: Windows/Linux/macOS\n\n\n\n\n**Video URL:**: url\n\n\n\n\n**Stack overflow**\n\n```\nerror message here\n```\n\n\n\n\n**Screenshots**\n\nnone\n\n\n\n\n**Additional context**\n\nnone\n" }, { "path": ".github/ISSUE_TEMPLATE/support-a-new-website.md", "content": "---\nname: Support a new website\nabout: If you want to request support for a new website\ntitle: \"[new website require]: the website name\"\nlabels: enhancement\nassignees: ''\n\n---\n\n\n\n\n- **Website name**: name\n\n\n\n\n- **Stream link**: url\n" }, { "path": ".github/workflows/builder.yml", "content": "name: Builder\n\non:\n push:\n branches: \"*\"\n paths:\n - \"**/*.go\"\n - \"go.mod\"\n - \"go.sum\"\n - \".github/workflows/builder.yml\"\n pull_request:\n branches: \"*\"\n paths:\n - \"**/*.go\"\n - \"go.mod\"\n - \"go.sum\"\n - \".github/workflows/builder.yml\"\n workflow_dispatch:\n\nenv:\n PRODUCT: lux\n CGO_ENABLED: 0\n GO111MODULE: on\n\njobs:\n build:\n name: Build\n strategy:\n matrix:\n os: [ linux, freebsd, openbsd, dragonfly, windows, darwin ]\n arch: [ amd64, 386 ]\n include:\n - os: linux\n arch: arm\n arm: 5\n - os: linux\n arch: arm\n arm: 6\n - os: linux\n arch: arm\n arm: 7\n - os: linux\n arch: arm64\n - os: linux\n arch: mips\n mips: softfloat\n - os: linux\n arch: mips\n mips: hardfloat\n - os: linux\n arch: mipsle\n mipsle: softfloat\n - os: linux\n arch: mipsle\n mipsle: hardfloat\n - os: linux\n arch: mips64\n - os: linux\n arch: mips64le\n - os: linux\n arch: ppc64\n - os: linux\n arch: ppc64le\n - os: linux\n arch: s390x\n - os: windows\n arch: arm\n - os: android\n arch: arm64\n - os: darwin\n arch: arm64\n - os: freebsd\n arch: arm64\n exclude:\n - os: darwin\n arch: 386\n - os: dragonfly\n arch: 386\n fail-fast: false\n runs-on: ubuntu-latest\n continue-on-error: true\n env:\n GOOS: ${{ matrix.os }}\n GOARCH: ${{ matrix.arch }}\n GOARM: ${{ matrix.arm }}\n GOMIPS: ${{ matrix.mips }}\n GOMIPS64: ${{ matrix.mips64 }}\n GOMIPSLE: ${{ matrix.mipsle }}\n steps:\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: 1.24\n\n - name: Check out code base\n if: github.event_name == 'push'\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n\n - name: Check out code base\n if: github.event_name == 'pull_request'\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n ref: ${{ github.event.pull_request.head.sha }}\n\n - name: Cache go module\n uses: actions/cache@v4\n with:\n path: ~/go/pkg/mod\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: ${{ runner.os }}-go-\n\n - name: Get dependencies\n run: |\n go get -v -t -d ./...\n\n - name: Build binary\n id: builder\n run: |\n ARGS=\"${GOOS}-${GOARCH}\"\n if [[ -n \"${GOARM}\" ]]; then\n ARGS=\"${ARGS}v${GOARM}\"\n elif [[ -n \"${GOMIPS}\" ]]; then\n ARGS=\"${ARGS}-${GOMIPS}\"\n elif [[ -n \"${GOMIPS64}\" ]]; then\n ARGS=\"${ARGS}-${GOMIPS64}\"\n elif [[ -n \"${GOMIPSLE}\" ]]; then\n ARGS=\"${ARGS}-${GOMIPSLE}\"\n fi\n go build -trimpath --ldflags \"-s -w -buildid=\" -v -o ./bin/${{ env.PRODUCT }}-${ARGS}\n echo \"::set-output name=filename::${{ env.PRODUCT }}-${ARGS}\"\n\n - name: Upload binary artifacts\n uses: actions/upload-artifact@v4\n with:\n name: ${{ steps.builder.outputs.filename }}\n path: ./bin/${{ env.PRODUCT }}*\n if-no-files-found: error\n" }, { "path": ".github/workflows/ci.yml", "content": "name: ci\non:\n push:\n pull_request:\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\njobs:\n ci:\n runs-on: ${{ matrix.os }}\n timeout-minutes: 30\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest, macOS-latest]\n name: Go ${{ matrix.go }} in ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n - name: Environment\n run: |\n go version\n go env\n - name: Lint\n uses: golangci/golangci-lint-action@v5\n with:\n version: v1.64.7\n only-new-issues: true\n - name: Test\n env:\n GOFLAGS: -mod=mod\n run: go test -race -coverpkg=./... -coverprofile=coverage.txt ./...\n - name: Send coverage\n run: bash <(curl -s https://codecov.io/bash)\n" }, { "path": ".github/workflows/goreleaser.yml", "content": "name: goreleaser\non:\n push:\n tags:\n - '*'\npermissions:\n contents: write\njobs:\n goreleaser:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: 1.24\n - name: Run GoReleaser\n uses: goreleaser/goreleaser-action@v5\n with:\n version: latest\n args: release --clean\n env:\n GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}\n" }, { "path": ".github/workflows/stream_acfun.yml", "content": "name: acfun\n\non:\n push:\n paths:\n - \"extractors/acfun/*.go\"\n - \".github/workflows/stream_acfun.yml\"\n pull_request:\n paths:\n - \"extractors/acfun/*.go\"\n - \".github/workflows/stream_acfun.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/acfun\n" }, { "path": ".github/workflows/stream_bcy.yml", "content": "name: bcy\n\non:\n push:\n paths:\n - \"extractors/bcy/*.go\"\n - \".github/workflows/stream_bcy.yml\"\n pull_request:\n paths:\n - \"extractors/bcy/*.go\"\n - \".github/workflows/stream_bcy.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bcy\n" }, { "path": ".github/workflows/stream_bilibili.yml", "content": "name: bilibili\n\non:\n push:\n paths:\n - \"extractors/bilibili/*.go\"\n - \".github/workflows/stream_bilibili.yml\"\n pull_request:\n paths:\n - \"extractors/bilibili/*.go\"\n - \".github/workflows/stream_bilibili.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bilibili\n" }, { "path": ".github/workflows/stream_bitchute.yml", "content": "name: bitchute\n\non:\n push:\n paths:\n - \"extractors/bitchute/*.go\"\n - \".github/workflows/stream_bitchute.yml\"\n pull_request:\n paths:\n - \"extractors/bitchute/*.go\"\n - \".github/workflows/stream_bitchute.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bitchute\n" }, { "path": ".github/workflows/stream_douyin.yml", "content": "name: douyin\n\non:\n push:\n paths:\n - \"extractors/douyin/*.go\"\n - \".github/workflows/stream_douyin.yml\"\n pull_request:\n paths:\n - \"extractors/douyin/*.go\"\n - \".github/workflows/stream_douyin.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/douyin\n" }, { "path": ".github/workflows/stream_douyu.yml", "content": "name: douyu\n\non:\n push:\n paths:\n - \"extractors/douyu/*.go\"\n - \".github/workflows/stream_douyu.yml\"\n pull_request:\n paths:\n - \"extractors/douyu/*.go\"\n - \".github/workflows/stream_douyu.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/douyu\n" }, { "path": ".github/workflows/stream_eporner.yml", "content": "name: eporner\n\non:\n push:\n paths:\n - \"extractors/eporner/*.go\"\n - \".github/workflows/stream_eporner.yml\"\n pull_request:\n paths:\n - \"extractors/eporner/*.go\"\n - \".github/workflows/stream_eporner.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/eporner\n" }, { "path": ".github/workflows/stream_facebook.yml", "content": "name: facebook\n\non:\n push:\n paths:\n - \"extractors/facebook/*.go\"\n - \".github/workflows/stream_facebook.yml\"\n pull_request:\n paths:\n - \"extractors/facebook/*.go\"\n - \".github/workflows/stream_facebook.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/facebook\n" }, { "path": ".github/workflows/stream_geekbang.yml", "content": "name: geekbang\n\non:\n push:\n paths:\n - \"extractors/geekbang/*.go\"\n - \".github/workflows/stream_geekbang.yml\"\n pull_request:\n paths:\n - \"extractors/geekbang/*.go\"\n - \".github/workflows/stream_geekbang.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/geekbang\n" }, { "path": ".github/workflows/stream_haokan.yml", "content": "name: haokan\n\non:\n push:\n paths:\n - \"extractors/haokan/*.go\"\n - \".github/workflows/stream_haokan.yml\"\n pull_request:\n paths:\n - \"extractors/haokan/*.go\"\n - \".github/workflows/stream_haokan.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/haokan\n" }, { "path": ".github/workflows/stream_hupu.yml", "content": "name: hupu\n\non:\n push:\n paths:\n - \"extractors/hupu/*.go\"\n - \".github/workflows/stream_hupu.yml\"\n pull_request:\n paths:\n - \"extractors/hupu/*.go\"\n - \".github/workflows/stream_hupu.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/hupu\n" }, { "path": ".github/workflows/stream_huya.yml", "content": "name: huya\n\non:\n push:\n paths:\n - \"extractors/huya/*.go\"\n - \".github/workflows/stream_huya.yml\"\n pull_request:\n paths:\n - \"extractors/huya/*.go\"\n - \".github/workflows/stream_huya.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/huya\n" }, { "path": ".github/workflows/stream_instagram.yml", "content": "name: instagram\n\non:\n push:\n paths:\n - \"extractors/instagram/*.go\"\n - \".github/workflows/stream_instagram.yml\"\n pull_request:\n paths:\n - \"extractors/instagram/*.go\"\n - \".github/workflows/stream_instagram.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/instagram\n" }, { "path": ".github/workflows/stream_iqiyi.yml", "content": "name: iqiyi\n\non:\n push:\n paths:\n - \"extractors/iqiyi/*.go\"\n - \".github/workflows/stream_iqiyi.yml\"\n pull_request:\n paths:\n - \"extractors/iqiyi/*.go\"\n - \".github/workflows/stream_iqiyi.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/iqiyi\n" }, { "path": ".github/workflows/stream_ixigua.yml", "content": "name: ixigua\n\non:\n push:\n paths:\n - \"extractors/ixigua/*.go\"\n - \".github/workflows/stream_ixigua.yml\"\n pull_request:\n paths:\n - \"extractors/ixigua/*.go\"\n - \".github/workflows/stream_ixigua.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/ixigua\n" }, { "path": ".github/workflows/stream_kuaishou.yml", "content": "name: kuaishou\n\non:\n push:\n paths:\n - \"extractors/kuaishou/*.go\"\n - \".github/workflows/stream_kuaishou.yml\"\n pull_request:\n paths:\n - \"extractors/kuaishou/*.go\"\n - \".github/workflows/stream_kuaishou.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/kuaishou\n" }, { "path": ".github/workflows/stream_mgtv.yml", "content": "name: mgtv\n\non:\n push:\n paths:\n - \"extractors/mgtv/*.go\"\n - \".github/workflows/stream_mgtv.yml\"\n pull_request:\n paths:\n - \"extractors/mgtv/*.go\"\n - \".github/workflows/stream_mgtv.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/mgtv\n" }, { "path": ".github/workflows/stream_miaopai.yml", "content": "name: miaopai\n\non:\n push:\n paths:\n - \"extractors/miaopai/*.go\"\n - \".github/workflows/stream_miaopai.yml\"\n pull_request:\n paths:\n - \"extractors/miaopai/*.go\"\n - \".github/workflows/stream_miaopai.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/miaopai\n" }, { "path": ".github/workflows/stream_netease.yml", "content": "name: netease\n\non:\n push:\n paths:\n - \"extractors/netease/*.go\"\n - \".github/workflows/stream_netease.yml\"\n pull_request:\n paths:\n - \"extractors/netease/*.go\"\n - \".github/workflows/stream_netease.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/netease\n" }, { "path": ".github/workflows/stream_odysee.yml", "content": "name: odysee\n\non:\n push:\n paths:\n - \"extractors/odysee/*.go\"\n - \".github/workflows/stream_odysee.yml\"\n pull_request:\n paths:\n - \"extractors/odysee/*.go\"\n - \".github/workflows/stream_odysee.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/odysee\n" }, { "path": ".github/workflows/stream_pinterest.yml", "content": "name: pinterest\n\non:\n push:\n paths:\n - \"extractors/pinterest/*.go\"\n - \".github/workflows/stream_pinterest.yml\"\n pull_request:\n paths:\n - \"extractors/pinterest/*.go\"\n - \".github/workflows/stream_pinterest.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pinterest\n" }, { "path": ".github/workflows/stream_pixivision.yml", "content": "name: pixivision\n\non:\n push:\n paths:\n - \"extractors/pixivision/*.go\"\n - \".github/workflows/stream_pixivision.yml\"\n pull_request:\n paths:\n - \"extractors/pixivision/*.go\"\n - \".github/workflows/stream_pixivision.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pixivision\n" }, { "path": ".github/workflows/stream_pornhub.yml", "content": "name: pornhub\n\non:\n push:\n paths:\n - \"extractors/pornhub/*.go\"\n - \".github/workflows/stream_pornhub.yml\"\n pull_request:\n paths:\n - \"extractors/pornhub/*.go\"\n - \".github/workflows/stream_pornhub.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pornhub\n" }, { "path": ".github/workflows/stream_qq.yml", "content": "name: qq\n\non:\n push:\n paths:\n - \"extractors/qq/*.go\"\n - \".github/workflows/stream_qq.yml\"\n pull_request:\n paths:\n - \"extractors/qq/*.go\"\n - \".github/workflows/stream_qq.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/qq\n" }, { "path": ".github/workflows/stream_reddit.yml", "content": "name: reddit\n\non:\n push:\n paths:\n - \"extractors/reddit/*.go\"\n - \".github/workflows/stream_reddit.yml\"\n pull_request:\n paths:\n - \"extractors/reddit/*.go\"\n - \".github/workflows/stream_reddit.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/reddit\n" }, { "path": ".github/workflows/stream_rumble.yml", "content": "name: rumble\n\non:\n push:\n paths:\n - \"extractors/rumble/*.go\"\n - \".github/workflows/stream_rumble.yml\"\n pull_request:\n paths:\n - \"extractors/rumble/*.go\"\n - \".github/workflows/stream_rumble.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/rumble\n" }, { "path": ".github/workflows/stream_streamtape.yml", "content": "name: streamtape\n\non:\n push:\n paths:\n - \"extractors/streamtape/*.go\"\n - \".github/workflows/stream_streamtape.yml\"\n pull_request:\n paths:\n - \"extractors/streamtape/*.go\"\n - \".github/workflows/stream_streamtape.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/streamtape\n" }, { "path": ".github/workflows/stream_tangdou.yml", "content": "name: tangdou\n\non:\n push:\n paths:\n - \"extractors/tangdou/*.go\"\n - \".github/workflows/stream_tangdou.yml\"\n pull_request:\n paths:\n - \"extractors/tangdou/*.go\"\n - \".github/workflows/stream_tangdou.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tangdou\n" }, { "path": ".github/workflows/stream_threads.yml", "content": "name: instagram\n\non:\n push:\n paths:\n - \"extractors/threads/*.go\"\n - \".github/workflows/stream_threads.yml\"\n pull_request:\n paths:\n - \"extractors/threads/*.go\"\n - \".github/workflows/stream_threads.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/threads\n" }, { "path": ".github/workflows/stream_tiktok.yml", "content": "name: tiktok\n\non:\n push:\n paths:\n - \"extractors/tiktok/*.go\"\n - \".github/workflows/stream_tiktok.yml\"\n pull_request:\n paths:\n - \"extractors/tiktok/*.go\"\n - \".github/workflows/stream_tiktok.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tiktok\n" }, { "path": ".github/workflows/stream_tumblr.yml", "content": "name: tumblr\n\non:\n push:\n paths:\n - \"extractors/tumblr/*.go\"\n - \".github/workflows/stream_tumblr.yml\"\n pull_request:\n paths:\n - \"extractors/tumblr/*.go\"\n - \".github/workflows/stream_tumblr.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tumblr\n" }, { "path": ".github/workflows/stream_twitter.yml", "content": "name: twitter\n\non:\n push:\n paths:\n - \"extractors/twitter/*.go\"\n - \".github/workflows/stream_twitter.yml\"\n pull_request:\n paths:\n - \"extractors/twitter/*.go\"\n - \".github/workflows/stream_twitter.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/twitter\n" }, { "path": ".github/workflows/stream_udn.yml", "content": "name: udn\n\non:\n push:\n paths:\n - \"extractors/udn/*.go\"\n - \".github/workflows/stream_udn.yml\"\n pull_request:\n paths:\n - \"extractors/udn/*.go\"\n - \".github/workflows/stream_udn.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/udn\n" }, { "path": ".github/workflows/stream_vimeo.yml", "content": "name: vimeo\n\non:\n push:\n paths:\n - \"extractors/vimeo/*.go\"\n - \".github/workflows/stream_vimeo.yml\"\n pull_request:\n paths:\n - \"extractors/vimeo/*.go\"\n - \".github/workflows/stream_vimeo.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/vimeo\n" }, { "path": ".github/workflows/stream_vk.yml", "content": "name: vk\n\non:\n push:\n paths:\n - \"extractors/vk/*.go\"\n - \".github/workflows/stream_vk.yml\"\n pull_request:\n paths:\n - \"extractors/vk/*.go\"\n - \".github/workflows/stream_vk.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/vk\n" }, { "path": ".github/workflows/stream_weibo.yml", "content": "name: weibo\n\non:\n push:\n paths:\n - \"extractors/weibo/*.go\"\n - \".github/workflows/stream_weibo.yml\"\n pull_request:\n paths:\n - \"extractors/weibo/*.go\"\n - \".github/workflows/stream_weibo.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/weibo\n" }, { "path": ".github/workflows/stream_xiaohongshu.yml", "content": "name: xiaohongshu\n\non:\n push:\n paths:\n - \"extractors/xiaohongshu/*.go\"\n - \".github/workflows/stream_xiaohongshu.yml\"\n pull_request:\n paths:\n - \"extractors/xiaohongshu/*.go\"\n - \".github/workflows/stream_xiaohongshu.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xiaohongshu\n" }, { "path": ".github/workflows/stream_ximalaya.yml", "content": "name: ximalaya\n\non:\n push:\n paths:\n - \"extractors/ximalaya/*.go\"\n - \".github/workflows/stream_ximalaya.yml\"\n pull_request:\n paths:\n - \"extractors/ximalaya/*.go\"\n - \".github/workflows/stream_ximalaya.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/ximalaya\n" }, { "path": ".github/workflows/stream_xinpianchang.yml", "content": "name: xinpianchang\n\non:\n push:\n paths:\n - \"extractors/xinpianchang/*.go\"\n - \".github/workflows/stream_xinpianchang.yml\"\n pull_request:\n paths:\n - \"extractors/xinpianchang/*.go\"\n - \".github/workflows/stream_xinpianchang.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xinpianchang\n" }, { "path": ".github/workflows/stream_xvideos.yml", "content": "name: xvideos\n\non:\n push:\n paths:\n - \"extractors/xvideos/*.go\"\n - \".github/workflows/stream_xvideos.yml\"\n pull_request:\n paths:\n - \"extractors/xvideos/*.go\"\n - \".github/workflows/stream_xvideos.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xvideos\n" }, { "path": ".github/workflows/stream_yinyuetai.yml", "content": "name: yinyuetai\n\non:\n push:\n paths:\n - \"extractors/yinyuetai/*.go\"\n - \".github/workflows/stream_yinyuetai.yml\"\n pull_request:\n paths:\n - \"extractors/yinyuetai/*.go\"\n - \".github/workflows/stream_yinyuetai.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/yinyuetai\n" }, { "path": ".github/workflows/stream_youku.yml", "content": "name: youku\n\non:\n push:\n paths:\n - \"extractors/youku/*.go\"\n - \".github/workflows/stream_youku.yml\"\n pull_request:\n paths:\n - \"extractors/youku/*.go\"\n - \".github/workflows/stream_youku.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/youku\n" }, { "path": ".github/workflows/stream_youtube.yml", "content": "name: youtube\n\non:\n push:\n paths:\n - \"extractors/youtube/*.go\"\n - \".github/workflows/stream_youtube.yml\"\n pull_request:\n paths:\n - \"extractors/youtube/*.go\"\n - \".github/workflows/stream_youtube.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/youtube\n" }, { "path": ".github/workflows/stream_zhihu.yml", "content": "name: zhihu\n\non:\n push:\n paths:\n - \"extractors/zhihu/*.go\"\n - \".github/workflows/stream_zhihu.yml\"\n pull_request:\n paths:\n - \"extractors/zhihu/*.go\"\n - \".github/workflows/stream_zhihu.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/zhihu\n" }, { "path": ".github/workflows/stream_zingmp3.yml", "content": "name: zingmp3\n\non:\n push:\n paths:\n - \"extractors/zingmp3/*.go\"\n - \".github/workflows/stream_zingmp3.yml\"\n pull_request:\n paths:\n - \"extractors/zingmp3/*.go\"\n - \".github/workflows/stream_zingmp3.yml\"\n schedule:\n # run ci weekly\n - cron: \"0 0 * * 0\"\n\njobs:\n test:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n go: [\"1.24\"]\n os: [ubuntu-latest]\n name: ${{ matrix.os }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Test\n run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/zingmp3\n" }, { "path": ".gitignore", "content": "# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n\n# Test binary, build with `go test -c`\n*.test\n\n# Output of the go coverage tool, specifically when used with LiteIDE\n*.out\ncoverage.txt\n\n# Python\n*.pyc\n\n.vscode\n.idea\ndist/\n*_token\ndownloader/*.jpg\n\n# Ignore compiled binaries\n# - native\nannie\n# - gox builds\nannie_*\n\n# macOS\n.DS_Store\n\n*.mp4\n*.mkv\n*.webm\n\nlux\n" }, { "path": ".golangci.yml", "content": "run:\n concurrency: 2\n timeout: 5m\n go: 1.24\n\nlinter-settings:\n goconst:\n min-len: 2\n min-occurrences: 2\n\nlinters:\n enable:\n - bodyclose\n - errcheck\n - goconst\n - gofmt\n - goimports\n - gosimple\n - govet\n - ineffassign\n - misspell\n - nilerr\n - staticcheck\n - typecheck\n - unconvert\n - unparam\n - unused\n - whitespace\n\nissues:\n exclude-use-default: false\n exclude-rules:\n - path: _test.go\n linters:\n - errcheck\n" }, { "path": ".goreleaser.yml", "content": "project_name: lux\nenv:\n - GO111MODULE=on\n - CGO_ENABLED=0\nbefore:\n hooks:\n - go mod download\nbuilds:\n- binary: lux\n ldflags: -s -w -X github.com/iawia002/lux/app.version={{ .RawVersion }}\n goos:\n - windows\n - darwin\n - linux\n - freebsd\n - openbsd\n - netbsd\n goarch:\n - \"386\"\n - amd64\n - arm\n - arm64\n ignore:\n - goos: freebsd\n goarch: arm\n goarm: 6\n - goos: freebsd\n goarch: arm64\n - goos: openbsd\n goarch: arm\n goarm: 6\narchives:\n- name_template: >-\n {{ .ProjectName }}_\n {{- .Version }}_\n {{- title .Os }}_\n {{- if eq .Arch \"amd64\" }}x86_64\n {{- else if eq .Arch \"386\" }}i386\n {{- else }}{{ .Arch }}{{ end }}\n {{- if .Arm }}v{{ .Arm }}{{ end }}\n format: tar.gz\n format_overrides:\n - goos: windows\n format: zip\n files:\n - none*\n wrap_in_directory: false\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing Guide\n\n* [Style Guide](#style-guide)\n* [Build](#build)\n* [Features Requested](#features-requested)\n\n## Style Guide\n### Code format\nLux uses [gofmt](https://golang.org/cmd/gofmt) to format the code, you must use [gofmt](https://golang.org/cmd/gofmt) to format your code before submitting.\n\n### linter\nWe recommend using [golint](https://github.com/golang/lint) or [gometalinter](https://github.com/alecthomas/gometalinter) to check your code format.\n\n## Build\n\nMake sure that this folder is in `GOPATH`, then:\n\n```bash\n$ go build\n```\n\n## Features Requested\nThere are several [features](https://github.com/iawia002/lux/issues?q=is%3Aissue+is%3Aopen+label%3Afeature-request) requested by the community. If you have any idea, feel free to fork the repo, follow the style guide above, push and merge it after passing the test. Besides, you are welcomed to propose new features through the issue.\n" }, { "path": "Cask.toml", "content": "[package]\nname = \"github.com/iawia002/lux\"\nbin = \"lux\"\nauthors = [\"Xinzhao Xu \"]\nkeywords = [\"go\", \"golang\", \"crawler\", \"scraper\", \"downloader\", \"youtube\", \"video\", \"download\", \"tumblr\", \"bilibili\", \"qq\", \"hacktoberfest\", \"youku\", \"iqiyi\"]\nrepository = \"https://github.com/iawia002/lux\"\ndescription = \"\"\"\n👾 Fast and simple video download library and CLI tool written in Go\n\"\"\"\n\n[darwin]\nx86_64 = { url = \"https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_x86_64.tar.gz\" }\naarch64 = { url = \"https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_arm64.tar.gz\" }\n\n[windows]\nx86_64 = { url = \"https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Windows_x86_64.zip\" }\n\n[linux]\nx86_64 = { url = \"https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_x86_64.tar.gz\" }\naarch64 = { url = \"https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_arm64.tar.gz\" }\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright 2018-present, iawia002\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "

Lux

\n\n

Let there be Lux!

\n\n
\n \n \"Codecov\"\n \n \n \"GitHub\n \n \n \"Go\n \n \n \"GitHub\n \n \n \"Homebrew\"\n \n
\n\n👾 Lux is a fast and simple video downloader built with Go.\n\n- [Installation](#installation)\n - [Prerequisites](#prerequisites)\n - [Install via `go install`](#install-via-go-install)\n - [Homebrew (macOS only)](#homebrew-macos-only)\n - [Arch Linux](#arch-linux)\n - [Void Linux](#void-linux)\n - [Scoop on Windows](#scoop-on-windows)\n - [Chocolatey on Windows](#chocolatey-on-windows)\n - [Cask on Windows/macOS/Linux](#cask-on-windowsmacoslinux)\n- [Getting Started](#getting-started)\n - [Download a video](#download-a-video)\n - [Download anything else](#download-anything-else)\n - [Download playlist](#download-playlist)\n - [Multiple inputs](#multiple-inputs)\n - [Resume a download](#resume-a-download)\n - [Auto retry](#auto-retry)\n - [Cookies](#cookies)\n - [Proxy](#proxy)\n - [Multi-Thread](#multi-thread)\n - [Short link](#short-link)\n - [bilibili](#bilibili)\n - [Use specified Referrer](#use-specified-referrer)\n - [Specify the output path and name](#specify-the-output-path-and-name)\n - [Debug Mode](#debug-mode)\n - [Reuse extracted data](#reuse-extracted-data)\n - [Options](#options)\n - [Download:](#download)\n - [Network:](#network)\n - [Playlist:](#playlist)\n - [Filesystem:](#filesystem)\n - [Subtitle:](#subtitle)\n - [Youku:](#youku)\n - [aria2:](#aria2)\n- [Supported Sites](#supported-sites)\n- [Known issues](#known-issues)\n - [优酷](#优酷)\n - [西瓜/头条视频](#西瓜头条视频)\n- [Contributing](#contributing)\n- [Authors](#authors)\n- [Similar projects](#similar-projects)\n- [License](#license)\n\n## Installation\n\n### Prerequisites\n\nThe following dependencies are required and must be installed separately.\n\n- **[FFmpeg](https://www.ffmpeg.org)**\n\n> **Note**: FFmpeg does not affect the download, only affects the final file merge.\n\n### Install via `go install`\n\nTo install Lux, use `go install`, or download the binary file from [Releases](https://github.com/iawia002/lux/releases) page.\n\n```bash\n$ go install github.com/iawia002/lux@latest\n```\n\n### Homebrew (macOS only)\n\nFor macOS users, you can install `lux` via:\n\n```bash\n$ brew install lux\n```\n\n### Arch Linux\n\nFor Arch Users [AUR](https://aur.archlinux.org/packages/lux-dl/) package is available.\n\n### Void Linux\n\nFor Void linux users, you can install `lux` via:\n\n```\n$ xbps-install -S lux\n```\n\n### [Scoop](https://scoop.sh/) on Windows\n\n```sh\n$ scoop install lux\n```\n\n### [Chocolatey](https://chocolatey.org/) on Windows\n\n```\n$ choco install lux\n```\n\n### [Cask](https://github.com/axetroy/cask.rs) on Windows/macOS/Linux\n\n```sh\n$ cask install github.com/iawia002/lux\n```\n\n## Getting Started\n\nUsage:\n\n```\nlux [OPTIONS] URL [URL...]\n```\n\n### Download a video\n\n```console\n$ lux \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\"\n\n Site: YouTube youtube.com\n Title: Rick Astley - Never Gonna Give You Up (Video)\n Type: video\n Stream:\n [248] -------------------\n Quality: 1080p video/webm; codecs=\"vp9\"\n Size: 63.93 MiB (67038963 Bytes)\n # download with: lux -f 248 ...\n\n 41.88 MiB / 63.93 MiB [=================>-------------] 65.51% 4.22 MiB/s 00m05s\n```\n\nThe `-i` option displays all available quality of video without downloading.\n\n```console\n$ lux -i \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\"\n\n Site: YouTube youtube.com\n Title: Rick Astley - Never Gonna Give You Up (Video)\n Type: video\n Streams: # All available quality\n [248] -------------------\n Quality: 1080p video/webm; codecs=\"vp9\"\n Size: 49.29 MiB (51687554 Bytes)\n # download with: lux -f 248 ...\n\n [137] -------------------\n Quality: 1080p video/mp4; codecs=\"avc1.640028\"\n Size: 43.45 MiB (45564306 Bytes)\n # download with: lux -f 137 ...\n\n [398] -------------------\n Quality: 720p video/mp4; codecs=\"av01.0.05M.08\"\n Size: 37.12 MiB (38926432 Bytes)\n # download with: lux -f 398 ...\n\n [136] -------------------\n Quality: 720p video/mp4; codecs=\"avc1.4d401f\"\n Size: 31.34 MiB (32867324 Bytes)\n # download with: lux -f 136 ...\n\n [247] -------------------\n Quality: 720p video/webm; codecs=\"vp9\"\n Size: 31.03 MiB (32536181 Bytes)\n # download with: lux -f 247 ...\n```\n\nUse `lux -f stream \"URL\"` to download a specific stream listed in the output of `-i` option.\n\n### Download anything else\n\nIf Lux is provided the URL of a specific resource, then it will be downloaded directly:\n\n```console\n$ lux \"https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg\"\n\nlux doesn't support this URL right now, but it will try to download it directly\n\n Site: Universal\n Title: 1f5a87801a0711e898b12b640777720f\n Type: image/jpeg\n Stream:\n [default] -------------------\n Size: 1.00 MiB (1051042 Bytes)\n # download with: lux -f default \"URL\"\n\n 1.00 MiB / 1.00 MiB [===================================] 100.00% 1.21 MiB/s 0s\n```\n\n### Download playlist\n\nThe `-p` option downloads an entire playlist instead of a single video.\n\n```console\n$ lux -i -p \"https://www.bilibili.com/bangumi/play/ep198061\"\n\n Site: 哔哩哔哩 bilibili.com\n Title: Doctor X 第四季：第一集\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 845.66 MiB (886738354 Bytes)\n # download with: lux -f default \"URL\"\n\n\n Site: 哔哩哔哩 bilibili.com\n Title: Doctor X 第四季：第二集\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 930.71 MiB (975919195 Bytes)\n # download with: lux -f default \"URL\"\n\n......\n```\n\nYou can use the `-start`, `-end` or `-items` option to specify the download range of the list:\n\n```\n-start\n \tPlaylist video to start at (default 1)\n-end\n \tPlaylist video to end at\n-items\n \tPlaylist video items to download. Separated by commas like: 1,5,6,8-10\n```\n\nFor bilibili playlists only:\n\n```\n-eto\n File name of each bilibili episode doesn't include the playlist title\n```\n\n### Multiple inputs\n\nYou can also download multiple URLs at once:\n\n```console\n$ lux -i \"https://www.bilibili.com/video/av21877586\" \"https://www.bilibili.com/video/av21990740\"\n\n Site: 哔哩哔哩 bilibili.com\n Title: 【莓机会了】甜到虐哭的13集单集MAD「我现在什么都不想干,更不想看14集」\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 51.88 MiB (54403767 Bytes)\n # download with: lux -f default \"URL\"\n\n\n Site: 哔哩哔哩 bilibili.com\n Title: 【莓救了】甜到虐哭！！！国家队单集MAD-当熟悉的bgm响起，眼泪从脸颊滑下\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 77.63 MiB (81404093 Bytes)\n # download with: lux -f default \"URL\"\n```\n\nThese URLs will be downloaded one by one.\n\nYou can also use the `-F` option to read URLs from file:\n\n```console\n$ lux -F ~/Desktop/u.txt\n\n Site: 微博 weibo.com\n Title: 在Google，我们设计什么？ via@阑夕\n Type: video\n Stream:\n [default] -------------------\n Size: 19.19 MiB (20118196 Bytes)\n # download with: lux -f default \"URL\"\n\n 19.19 MiB / 19.19 MiB [=================================] 100.00% 9.69 MiB/s 1s\n\n......\n```\n\nYou can use the `-start`, `-end` or `-items` option to specify the download range of the list:\n\n```\n-start\n \tFile line to start at (default 1)\n-end\n \tFile line to end at\n-items\n \tFile lines to download. Separated by commas like: 1,5,6,8-10\n```\n\n### Resume a download\n\nCtrl+C interrupts a download.\n\nA temporary `.download` file is kept in the output directory. If `lux` is ran with the same arguments, then the download progress will resume from the last session.\n\n### Auto retry\n\nlux will auto retry when the download failed, you can specify the retry times by `-retry` option (default is 100).\n\n### Cookies\n\nCookies can be provided to `lux` with the `-c` option if they are required for accessing the video.\n\nCookies can be the following format or [Netscape Cookie](https://curl.haxx.se/rfc/cookie_spec.html) format:\n\n```console\nname=value; name2=value2; ...\n```\n\nCookies can be a string or a text file, supply cookies in one of the two following ways.\n\nAs a string:\n\n```console\n$ lux -c \"name=value; name2=value2\" \"https://www.bilibili.com/video/av20203945\"\n```\n\nAs a text file:\n\n```console\n$ lux -c cookies.txt \"https://www.bilibili.com/video/av20203945\"\n```\n\n### Proxy\n\nYou can set the HTTP/SOCKS5 proxy using environment variables:\n\n```console\n$ HTTP_PROXY=\"http://127.0.0.1:1087/\" lux -i \"https://www.youtube.com/watch?v=Gnbch2osEeo\"\n```\n\n```console\n$ HTTP_PROXY=\"socks5://127.0.0.1:1080/\" lux -i \"https://www.youtube.com/watch?v=Gnbch2osEeo\"\n```\n\n### Multi-Thread\n\nUse `--multi-thread` or `-m` multiple threads to download single video.\n\nUse `--thread` or `-n` option to set the number of download threads(default is 10).\n\n> Note: If the video has multi fragment, the number of actual download threads will increase.\n>\n> For example:\n> * If `-n` is set to 10, and the video has 2 fragments, then 20 threads will actually be used.\n> * If the video has 20 fragments, only 10 fragments are downloaded in the same time, the actual threads count is 100.\n\n> **Special Tips:** Use too many threads in **mgtv** download will cause HTTP 403 error, we recommend setting the number of threads to **1**.\n\n### Short link\n\n#### bilibili\n\nYou can just use `av` or `ep` number to download bilibili's video:\n\n```console\n$ lux -i ep198381 av21877586\n\n Site: 哔哩哔哩 bilibili.com\n Title: 狐妖小红娘：第79话 南国公主的吃货本色\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 485.23 MiB (508798478 Bytes)\n # download with: lux -f default \"URL\"\n\n\n Site: 哔哩哔哩 bilibili.com\n Title: 【莓机会了】甜到虐哭的13集单集MAD「我现在什么都不想干,更不想看14集」\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 51.88 MiB (54403767 Bytes)\n # download with: lux -f default \"URL\"\n```\n\n### Use specified Referrer\n\nA Referrer can be used for the request with the `-r` option:\n\n```console\n$ lux -r \"https://www.bilibili.com/video/av20383055/\" \"http://cn-scnc1-dx.acgvideo.com/\"\n```\n\n### Specify the output path and name\n\nThe `-o` option sets the path, and `-O` option sets the name of the downloaded file:\n\n```console\n$ lux -o ../ -O \"hello\" \"https://example.com\"\n```\n\n### Debug Mode\n\nThe `-d` option outputs network request messages:\n\n```console\n$ lux -i -d \"http://www.bilibili.com/video/av20088587\"\n\nURL: http://www.bilibili.com/video/av20088587\nMethod: GET\nHeaders: http.Header{\n \"Referer\": {\"http://www.bilibili.com/video/av20088587\"},\n \"Accept\": {\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\"},\n \"Accept-Charset\": {\"UTF-8,*;q=0.5\"},\n \"Accept-Encoding\": {\"gzip,deflate,sdch\"},\n \"Accept-Language\": {\"en-US,en;q=0.8\"},\n \"User-Agent\": {\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36\"},\n}\nStatus Code: 200\n\nURL: https://interface.bilibili.com/v2/playurl?appkey=84956560bc028eb7&cid=32782944&otype=json&qn=116&quality=116&type=&sign=fb2e3f261fec398652f96d358517e535\nMethod: GET\nHeaders: http.Header{\n \"Accept-Charset\": {\"UTF-8,*;q=0.5\"},\n \"Accept-Encoding\": {\"gzip,deflate,sdch\"},\n \"Accept-Language\": {\"en-US,en;q=0.8\"},\n \"User-Agent\": {\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36\"},\n \"Referer\": {\"https://interface.bilibili.com/v2/playurl?appkey=84956560bc028eb7&cid=32782944&otype=json&qn=116&quality=116&type=&sign=fb2e3f261fec398652f96d358517e535\"},\n \"Accept\": {\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\"},\n}\nStatus Code: 200\n\n Site: 哔哩哔哩 bilibili.com\n Title: 燃油动力的遥控奥迪R8跑赛道\n Type: video\n Streams: # All available quality\n [default] -------------------\n Quality: 高清 1080P\n Size: 64.38 MiB (67504795 Bytes)\n # download with: lux -f default \"URL\"\n```\n\n### Reuse extracted data\n\nThe `-j` option will print the extracted data in JSON format.\n\n```console\n$ lux -j \"https://www.bilibili.com/video/av20203945\"\n\n{\n \"site\": \"哔哩哔哩 bilibili.com\",\n \"title\": \"【2018拜年祭单品】相遇day by day\",\n \"type\": \"video\",\n \"streams\": {\n \"15\": {\n \"urls\": [\n {\n \"url\": \"...\",\n \"size\": 18355205,\n \"ext\": \"flv\"\n }\n ],\n \"quality\": \"流畅 360P\",\n \"size\": 18355205\n },\n \"32\": {\n \"urls\": [\n {\n \"url\": \"...\",\n \"size\": 40058632,\n \"ext\": \"flv\"\n }\n ],\n \"quality\": \"清晰 480P\",\n \"size\": 40058632\n },\n \"64\": {\n \"urls\": [\n {\n \"url\": \"...\",\n \"size\": 82691087,\n \"ext\": \"flv\"\n }\n ],\n \"quality\": \"高清 720P\",\n \"size\": 82691087\n },\n \"80\": {\n \"urls\": [\n {\n \"url\": \"...\",\n \"size\": 121735559,\n \"ext\": \"flv\"\n }\n ],\n \"quality\": \"高清 1080P\",\n \"size\": 121735559\n }\n }\n}\n```\n\n### Options\n\n```\n -i\tInformation only\n -F string\n \tURLs file path\n -d\tDebug mode\n -j\tPrint extracted data\n -s\tMinimum outputs\n -v\tShow version\n```\n\n#### Download:\n\n```\n -f string\n \tSelect specific stream to download\n -p\tDownload playlist\n -n int\n \tThe number of download thread (only works for multiple-parts video) (default 10)\n -c string\n \tCookie\n -r string\n \tUse specified Referrer\n -cs int\n \tHTTP chunk size for downloading (in MB) (default 1)\n```\n\n#### Network:\n\n```\n -retry int\n \tHow many times to retry when the download failed (default 10)\n```\n\n#### Playlist:\n\n```\n -start int\n \tPlaylist video to start at (default 1)\n -end int\n \tPlaylist video to end at\n -items string\n \tPlaylist video items to download. Separated by commas like: 1,5,6,8-10\n```\n\n#### Filesystem:\n\n```\n -o string\n \tSpecify the output path\n -O string\n \tSpecify the output file name\n```\n\n#### Subtitle:\n\n```\n -C\tDownload subtitles\n -C -items en,zh\n \tDownload specific languages (YouTube only)\n -C -items en,zh -embed \n \tEmbed subtitles into the video (YouTube only)\n```\n\n#### Youku:\n\n```\n -ccode string\n \tYouku ccode (default \"0502\")\n -ckey string\n \tYouku ckey (default \"7B19C0AB12633B22E7FE81271162026020570708D6CC189E4924503C49D243A0DE6CD84A766832C2C99898FC5ED31F3709BB3CDD82C96492E721BDD381735026\")\n -password string\n \tYouku password\n```\n\n#### aria2:\n\n> Note: If you use aria2 to download, you need to merge the multi-part videos yourself.\n\n```\n -aria2\n \tUse Aria2 RPC to download\n -aria2addr string\n \tAria2 Address (default \"localhost:6800\")\n -aria2method string\n \tAria2 Method (default \"http\")\n -aria2token string\n \tAria2 RPC Token\n```\n\n## Supported Sites\n\n| Site | URL | 🎬 Videos | 🌁 Images | 🔊 Audio | 📚 Playlist | 🍪 VIP adaptation | Build Status |\n| ---------------- | ------------------------------------------------------------------------- | -------- | -------- | ------- | ---------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| 抖音 | | ✓ | ✓ | | | | [![douyin](https://github.com/iawia002/lux/actions/workflows/stream_douyin.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_douyin.yml) |\n| 哔哩哔哩 | | ✓ | | | ✓ | ✓ | [![bilibili](https://github.com/iawia002/lux/actions/workflows/stream_bilibili.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bilibili.yml) |\n| 半次元 | | | ✓ | | | | [![bcy](https://github.com/iawia002/lux/actions/workflows/stream_bcy.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bcy.yml) |\n| pixivision | | | ✓ | | | | [![pixivision](https://github.com/iawia002/lux/actions/workflows/stream_pixivision.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pixivision.yml) |\n| 优酷 | | ✓ | | | | ✓ | [![youku](https://github.com/iawia002/lux/actions/workflows/stream_youku.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_youku.yml) |\n| YouTube | | ✓ | | | ✓ | | [![youtube](https://github.com/iawia002/lux/actions/workflows/stream_youtube.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_youtube.yml) |\n| 西瓜视频（头条） | , , | ✓ | | | | | [![ixigua](https://github.com/iawia002/lux/actions/workflows/stream_ixigua.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_ixigua.yml) |\n| 爱奇艺 | | ✓ | | | | | [![iqiyi](https://github.com/iawia002/lux/actions/workflows/stream_iqiyi.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_iqiyi.yml) |\n| 新片场 | | ✓ | | | | | [![xinpianchang](https://github.com/iawia002/lux/actions/workflows/stream_xinpianchang.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xinpianchang.yml) |\n| 芒果 TV | | ✓ | | | | | [![mgtv](https://github.com/iawia002/lux/actions/workflows/stream_mgtv.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_mgtv.yml) |\n| 糖豆广场舞 | | ✓ | | | | | [![tangdou](https://github.com/iawia002/lux/actions/workflows/stream_tangdou.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tangdou.yml) |\n| Tumblr | | ✓ | ✓ | | | | [![tumblr](https://github.com/iawia002/lux/actions/workflows/stream_tumblr.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tumblr.yml) |\n| Vimeo | | ✓ | | | | | [![vimeo](https://github.com/iawia002/lux/actions/workflows/stream_vimeo.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vimeo.yml) |\n| Facebook | | ✓ | | | | | [![facebook](https://github.com/iawia002/lux/actions/workflows/stream_facebook.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_facebook.yml) |\n| 斗鱼视频 | | ✓ | | | | | [![douyu](https://github.com/iawia002/lux/actions/workflows/stream_douyu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_douyu.yml) |\n| 秒拍 | | ✓ | | | | | [![miaopai](https://github.com/iawia002/lux/actions/workflows/stream_miaopai.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_miaopai.yml) |\n| 微博 | | ✓ | | | | | [![weibo](https://github.com/iawia002/lux/actions/workflows/stream_weibo.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_weibo.yml) |\n| Instagram | | ✓ | ✓ | | | | [![instagram](https://github.com/iawia002/lux/actions/workflows/stream_instagram.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_instagram.yml) |\n| Threads | | ✓ | ✓ | | | | [![threads](https://github.com/iawia002/lux/actions/workflows/stream_threads.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_threads.yml) |\n| Twitter | | ✓ | | | | | [![twitter](https://github.com/iawia002/lux/actions/workflows/stream_twitter.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_twitter.yml) |\n| 腾讯视频 | | ✓ | | | | | [![qq](https://github.com/iawia002/lux/actions/workflows/stream_qq.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_qq.yml) |\n| 网易云音乐 | | ✓ | | | | | [![netease](https://github.com/iawia002/lux/actions/workflows/stream_netease.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_netease.yml) |\n| 音悦台 | | ✓ | | | | | [![yinyuetai](https://github.com/iawia002/lux/actions/workflows/stream_yinyuetai.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_yinyuetai.yml) |\n| 极客时间 | | ✓ | | | | | [![geekbang](https://github.com/iawia002/lux/actions/workflows/stream_geekbang.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_geekbang.yml) |\n| Pornhub | | ✓ | | | | | [![pornhub](https://github.com/iawia002/lux/actions/workflows/stream_pornhub.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pornhub.yml) |\n| XVIDEOS | | ✓ | | | | | [![xvideos](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml) |\n| 聯合新聞網 | | ✓ | | | | | [![udn](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml) |\n| TikTok | | ✓ | | | | | [![tiktok](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml) |\n| Pinterest | | ✓ | | | | | [![pinterest](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml) |\n| 好看视频 | | ✓ | | | | | [![haokan](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml) |\n| AcFun | | ✓ | | | ✓ | | [![acfun](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml) |\n| Eporner | | ✓ | | | | | [![eporner](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml) |\n| StreamTape | | ✓ | | | | | [![streamtape](https://github.com/iawia002/lux/actions/workflows/stream_streamtape.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_streamtape.yml) |\n| 虎扑 | | ✓ | | | | | [![hupu](https://github.com/iawia002/lux/actions/workflows/stream_hupu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_hupu.yml) |\n| 虎牙视频 | | ✓ | | | | | [![huya](https://github.com/iawia002/lux/actions/workflows/stream_huya.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_huya.yml) |\n| 喜马拉雅 | | | | ✓ | | | [![ximalaya](https://github.com/iawia002/lux/actions/workflows/stream_ximalaya.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_ximalaya.yml) |\n| 快手 | | ✓ | | | | | [![kuaishou](https://github.com/iawia002/lux/actions/workflows/stream_kuaishou.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_kuaishou.yml) |\n| Reddit | | ✓ | ✓ | | | | [![reddit](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml) |\n| VKontakte | | ✓ | | | | | [![vk](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/) |\n| 知乎 | | ✓ | | | | | [![zhihu](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/) |\n| Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) |\n| 小红书 | | ✓ | | | | | [![xiaohongshu](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/) |\n| Zing MP3 | | ✓ | | ✓ | | | [![zingmp3](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/) |\n| Bitchute | | ✓ | | | | | [![bitchute](https://github.com/iawia002/lux/actions/workflows/stream_bitchute.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bitchute.yml/) |\n| Odysee | | ✓ | | ✓ | | | [![odysee](https://github.com/iawia002/lux/actions/workflows/stream_odysee.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_odysee.yml/) |\n\n\n## Known issues\n\n### 优酷\n\n优酷的 `ccode` 经常变化导致 lux 不可用，如果你知道有新的可用的 `ccode`，可以直接使用 `lux -ccode ...` 而不用等待 lux 更新（当然，也欢迎你给我们提一个 Pull request 来更新默认的 `ccode`）\n\n最好是每次下载都附带登录过的 Cookie 以避免部分 `ccode` 的问题\n\n### 西瓜/头条视频\n西瓜/头条视频必须带 Cookie 才能下载成功，西瓜和头条可共用西瓜视频的 Cookie，Cookie 的有效期可能较短，下载失败就更新 Cookie 尝试：\n\n```\n$ lux -c \"msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C...\" \"https://m.toutiao.com/is/iYbTfJ79/\"\n```\n\n## Contributing\n\nLux is an open source project and built on the top of open-source projects. Check out the [Contributing Guide](./CONTRIBUTING.md) to get started.\n\n## Authors\n\nCode with ❤️ by [iawia002](https://github.com/iawia002) and lovely [contributors](https://github.com/iawia002/lux/graphs/contributors)\n\n## Similar projects\n\n- [youtube](https://github.com/kkdai/youtube)\n- [youtube-dl](https://github.com/rg3/youtube-dl)\n- [you-get](https://github.com/soimort/you-get)\n- [ytdl](https://github.com/rylio/ytdl)\n\n## License\n\nMIT\n\nCopyright (c) 2018-present, iawia002\n" }, { "path": "app/app.go", "content": "package app\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/urfave/cli/v2\"\n\n\t\"github.com/iawia002/lux/downloader\"\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\n// Name is the name of this app.\nconst Name = \"lux\"\n\n// This value will be injected into the corresponding git tag value at build time using `-ldflags`.\nvar version = \"v0.0.0\"\n\nfunc init() {\n\tcli.VersionPrinter = func(c *cli.Context) {\n\t\tblue := color.New(color.FgBlue)\n\t\tcyan := color.New(color.FgCyan)\n\t\tfmt.Fprintf(\n\t\t\tcolor.Output,\n\t\t\t\"\\n%s: version %s, A fast and simple video downloader.\\n\\n\",\n\t\t\tcyan.Sprintf(Name),\n\t\t\tblue.Sprintf(c.App.Version),\n\t\t)\n\t}\n}\n\n// New returns the App instance.\nfunc New() *cli.App {\n\tapp := &cli.App{\n\t\tName: Name,\n\t\tUsage: \"A fast and simple video downloader.\",\n\t\tVersion: version,\n\t\tFlags: []cli.Flag{\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"debug\",\n\t\t\t\tAliases: []string{\"d\"},\n\t\t\t\tUsage: \"Debug mode\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"silent\",\n\t\t\t\tAliases: []string{\"s\"},\n\t\t\t\tUsage: \"Minimum outputs\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"info\",\n\t\t\t\tAliases: []string{\"i\"},\n\t\t\t\tUsage: \"Information only\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"json\",\n\t\t\t\tAliases: []string{\"j\"},\n\t\t\t\tUsage: \"Print extracted JSON data\",\n\t\t\t},\n\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"cookie\",\n\t\t\t\tAliases: []string{\"c\"},\n\t\t\t\tUsage: \"Cookie\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"playlist\",\n\t\t\t\tAliases: []string{\"p\"},\n\t\t\t\tUsage: \"Download playlist\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"user-agent\",\n\t\t\t\tAliases: []string{\"u\"},\n\t\t\t\tUsage: \"Use specified User-Agent\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"refer\",\n\t\t\t\tAliases: []string{\"r\"},\n\t\t\t\tUsage: \"Use specified Referrer\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"stream-format\",\n\t\t\t\tAliases: []string{\"f\"},\n\t\t\t\tUsage: \"Select specific stream to download\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"audio-only\",\n\t\t\t\tAliases: []string{\"ao\"},\n\t\t\t\tUsage: \"Download audio only at best quality\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"file\",\n\t\t\t\tAliases: []string{\"F\"},\n\t\t\t\tUsage: \"URLs file path\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"output-path\",\n\t\t\t\tAliases: []string{\"o\"},\n\t\t\t\tUsage: \"Specify the output path\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"output-name\",\n\t\t\t\tAliases: []string{\"O\"},\n\t\t\t\tUsage: \"Specify the output file name\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"file-name-length\",\n\t\t\t\tValue: 255,\n\t\t\t\tUsage: \"The maximum length of a file name, 0 means unlimited\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"caption\",\n\t\t\t\tAliases: []string{\"C\"},\n\t\t\t\tUsage: \"Download captions\",\n\t\t\t},\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"embed-subtitle\",\n\t\t\t\tAliases: []string{\"embed\"},\n\t\t\t\tUsage: \"Embed subtitles into the video (requires ffmpeg)\",\n\t\t\t},\n\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"start\",\n\t\t\t\tValue: 1,\n\t\t\t\tUsage: \"Define the starting item of a playlist or a file input\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"end\",\n\t\t\t\tValue: 0,\n\t\t\t\tUsage: \"Define the ending item of a playlist or a file input\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"items\",\n\t\t\t\tUsage: \"Define wanted items from a file or playlist. Separated by commas like: 1,5,6,8-10\",\n\t\t\t},\n\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"multi-thread\",\n\t\t\t\tAliases: []string{\"m\"},\n\t\t\t\tUsage: \"Multiple threads to download single video\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"retry\",\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"How many times to retry when the download failed\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"chunk-size\",\n\t\t\t\tAliases: []string{\"cs\"},\n\t\t\t\tValue: 1,\n\t\t\t\tUsage: \"HTTP chunk size for downloading (in MB)\",\n\t\t\t},\n\t\t\t&cli.UintFlag{\n\t\t\t\tName: \"thread\",\n\t\t\t\tAliases: []string{\"n\"},\n\t\t\t\tValue: 10,\n\t\t\t\tUsage: \"The number of download thread (only works for multiple-parts video)\",\n\t\t\t},\n\n\t\t\t// Aria2\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"aria2\",\n\t\t\t\tUsage: \"Use Aria2 RPC to download\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"aria2-token\",\n\t\t\t\tUsage: \"Aria2 RPC Token\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"aria2-addr\",\n\t\t\t\tValue: \"localhost:6800\",\n\t\t\t\tUsage: \"Aria2 Address\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"aria2-method\",\n\t\t\t\tValue: \"http\",\n\t\t\t\tUsage: \"Aria2 Method\",\n\t\t\t},\n\n\t\t\t// youku\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"youku-ccode\",\n\t\t\t\tAliases: []string{\"ccode\"},\n\t\t\t\tValue: \"0502\",\n\t\t\t\tUsage: \"Youku ccode\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"youku-ckey\",\n\t\t\t\tAliases: []string{\"ckey\"},\n\t\t\t\tValue: \"7B19C0AB12633B22E7FE81271162026020570708D6CC189E4924503C49D243A0DE6CD84A766832C2C99898FC5ED31F3709BB3CDD82C96492E721BDD381735026\",\n\t\t\t\tUsage: \"Youku ckey\",\n\t\t\t},\n\t\t\t&cli.StringFlag{\n\t\t\t\tName: \"youku-password\",\n\t\t\t\tAliases: []string{\"password\"},\n\t\t\t\tUsage: \"Youku password\",\n\t\t\t},\n\n\t\t\t&cli.BoolFlag{\n\t\t\t\tName: \"episode-title-only\",\n\t\t\t\tAliases: []string{\"eto\"},\n\t\t\t\tUsage: \"File name of each bilibili episode doesn't include the playlist title\",\n\t\t\t},\n\t\t},\n\t\tAction: func(c *cli.Context) error {\n\t\t\targs := c.Args().Slice()\n\n\t\t\tif c.Bool(\"debug\") {\n\t\t\t\tcli.VersionPrinter(c)\n\t\t\t}\n\n\t\t\tif file := c.String(\"file\"); file != \"\" {\n\t\t\t\tf, err := os.Open(file)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tdefer f.Close() // nolint\n\n\t\t\t\tfileItems := utils.ParseInputFile(f, c.String(\"items\"), int(c.Uint(\"start\")), int(c.Uint(\"end\")))\n\t\t\t\targs = append(args, fileItems...)\n\t\t\t}\n\n\t\t\tif len(args) < 1 {\n\t\t\t\treturn errors.New(\"too few arguments\")\n\t\t\t}\n\n\t\t\tcookie := c.String(\"cookie\")\n\t\t\tif cookie != \"\" {\n\t\t\t\t// If cookie is a file path, convert it to a string to ensure cookie is always string\n\t\t\t\tif _, fileErr := os.Stat(cookie); fileErr == nil {\n\t\t\t\t\t// Cookie is a file\n\t\t\t\t\tdata, err := os.ReadFile(cookie)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tcookie = strings.TrimSpace(string(data))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trequest.SetOptions(request.Options{\n\t\t\t\tRetryTimes: int(c.Uint(\"retry\")),\n\t\t\t\tCookie: cookie,\n\t\t\t\tUserAgent: c.String(\"user-agent\"),\n\t\t\t\tRefer: c.String(\"refer\"),\n\t\t\t\tDebug: c.Bool(\"debug\"),\n\t\t\t\tSilent: c.Bool(\"silent\"),\n\t\t\t})\n\n\t\t\tvar isErr bool\n\t\t\tfor _, videoURL := range args {\n\t\t\t\tif err := download(c, videoURL); err != nil {\n\t\t\t\t\tfmt.Fprintf(\n\t\t\t\t\t\tcolor.Output,\n\t\t\t\t\t\t\"Downloading %s error:\\n\",\n\t\t\t\t\t\tcolor.CyanString(\"%s\", videoURL),\n\t\t\t\t\t)\n\t\t\t\t\tfmt.Printf(\"%+v\\n\", err)\n\t\t\t\t\tisErr = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif isErr {\n\t\t\t\treturn cli.Exit(\"\", 1)\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t\tEnableBashCompletion: true,\n\t}\n\n\tsort.Sort(cli.FlagsByName(app.Flags))\n\treturn app\n}\n\nfunc download(c *cli.Context, videoURL string) error {\n\tdata, err := extractors.Extract(videoURL, extractors.Options{\n\t\tPlaylist: c.Bool(\"playlist\"),\n\t\tItems: c.String(\"items\"),\n\t\tItemStart: int(c.Uint(\"start\")),\n\t\tItemEnd: int(c.Uint(\"end\")),\n\t\tThreadNumber: int(c.Uint(\"thread\")),\n\t\tEpisodeTitleOnly: c.Bool(\"episode-title-only\"),\n\t\tCookie: c.String(\"cookie\"),\n\t\tYoukuCcode: c.String(\"youku-ccode\"),\n\t\tYoukuCkey: c.String(\"youku-ckey\"),\n\t\tYoukuPassword: c.String(\"youku-password\"),\n\t})\n\tif err != nil {\n\t\t// if this error occurs, it means that an error occurred before actually starting to extract data\n\t\t// (there is an error in the preparation step), and the data list is empty.\n\t\treturn err\n\t}\n\n\tif c.Bool(\"json\") {\n\t\te := json.NewEncoder(os.Stdout)\n\t\te.SetIndent(\"\", \"\\t\")\n\t\te.SetEscapeHTML(false)\n\t\tif err := e.Encode(data); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tdefaultDownloader := downloader.New(downloader.Options{\n\t\tSilent: c.Bool(\"silent\"),\n\t\tInfoOnly: c.Bool(\"info\"),\n\t\tStream: c.String(\"stream-format\"),\n\t\tAudioOnly: c.Bool(\"audio-only\"),\n\t\tRefer: c.String(\"refer\"),\n\t\tOutputPath: c.String(\"output-path\"),\n\t\tOutputName: c.String(\"output-name\"),\n\t\tFileNameLength: int(c.Uint(\"file-name-length\")),\n\t\tCaption: c.Bool(\"caption\"),\n\t\tEmbedSubtitle: c.Bool(\"embed-subtitle\"),\n\t\tMultiThread: c.Bool(\"multi-thread\"),\n\t\tThreadNumber: int(c.Uint(\"thread\")),\n\t\tRetryTimes: int(c.Uint(\"retry\")),\n\t\tChunkSizeMB: int(c.Uint(\"chunk-size\")),\n\t\tUseAria2RPC: c.Bool(\"aria2\"),\n\t\tAria2Token: c.String(\"aria2-token\"),\n\t\tAria2Method: c.String(\"aria2-method\"),\n\t\tAria2Addr: c.String(\"aria2-addr\"),\n\t})\n\terrors := make([]error, 0)\n\tfor _, item := range data {\n\t\tif item.Err != nil {\n\t\t\t// if this error occurs, the preparation step is normal, but the data extraction is wrong.\n\t\t\t// the data is an empty struct.\n\t\t\terrors = append(errors, item.Err)\n\t\t\tcontinue\n\t\t}\n\t\tif err = defaultDownloader.Download(item); err != nil {\n\t\t\terrors = append(errors, err)\n\t\t}\n\t}\n\tif len(errors) != 0 {\n\t\treturn errors[0]\n\t}\n\treturn nil\n}\n" }, { "path": "app/register.go", "content": "package app\n\nimport (\n\t_ \"github.com/iawia002/lux/extractors/acfun\"\n\t_ \"github.com/iawia002/lux/extractors/bcy\"\n\t_ \"github.com/iawia002/lux/extractors/bilibili\"\n\t_ \"github.com/iawia002/lux/extractors/bitchute\"\n\t_ \"github.com/iawia002/lux/extractors/douyin\"\n\t_ \"github.com/iawia002/lux/extractors/douyu\"\n\t_ \"github.com/iawia002/lux/extractors/eporner\"\n\t_ \"github.com/iawia002/lux/extractors/facebook\"\n\t_ \"github.com/iawia002/lux/extractors/geekbang\"\n\t_ \"github.com/iawia002/lux/extractors/haokan\"\n\t_ \"github.com/iawia002/lux/extractors/hupu\"\n\t_ \"github.com/iawia002/lux/extractors/huya\"\n\t_ \"github.com/iawia002/lux/extractors/instagram\"\n\t_ \"github.com/iawia002/lux/extractors/iqiyi\"\n\t_ \"github.com/iawia002/lux/extractors/ixigua\"\n\t_ \"github.com/iawia002/lux/extractors/kuaishou\"\n\t_ \"github.com/iawia002/lux/extractors/mgtv\"\n\t_ \"github.com/iawia002/lux/extractors/miaopai\"\n\t_ \"github.com/iawia002/lux/extractors/netease\"\n\t_ \"github.com/iawia002/lux/extractors/odysee\"\n\t_ \"github.com/iawia002/lux/extractors/pinterest\"\n\t_ \"github.com/iawia002/lux/extractors/pixivision\"\n\t_ \"github.com/iawia002/lux/extractors/pornhub\"\n\t_ \"github.com/iawia002/lux/extractors/qq\"\n\t_ \"github.com/iawia002/lux/extractors/reddit\"\n\t_ \"github.com/iawia002/lux/extractors/rumble\"\n\t_ \"github.com/iawia002/lux/extractors/streamtape\"\n\t_ \"github.com/iawia002/lux/extractors/tangdou\"\n\t_ \"github.com/iawia002/lux/extractors/threads\"\n\t_ \"github.com/iawia002/lux/extractors/tiktok\"\n\t_ \"github.com/iawia002/lux/extractors/tumblr\"\n\t_ \"github.com/iawia002/lux/extractors/twitter\"\n\t_ \"github.com/iawia002/lux/extractors/udn\"\n\t_ \"github.com/iawia002/lux/extractors/universal\"\n\t_ \"github.com/iawia002/lux/extractors/vimeo\"\n\t_ \"github.com/iawia002/lux/extractors/vk\"\n\t_ \"github.com/iawia002/lux/extractors/weibo\"\n\t_ \"github.com/iawia002/lux/extractors/xiaohongshu\"\n\t_ \"github.com/iawia002/lux/extractors/ximalaya\"\n\t_ \"github.com/iawia002/lux/extractors/xinpianchang\"\n\t_ \"github.com/iawia002/lux/extractors/xvideos\"\n\t_ \"github.com/iawia002/lux/extractors/yinyuetai\"\n\t_ \"github.com/iawia002/lux/extractors/youku\"\n\t_ \"github.com/iawia002/lux/extractors/youtube\"\n\t_ \"github.com/iawia002/lux/extractors/zhihu\"\n\t_ \"github.com/iawia002/lux/extractors/zingmp3\"\n)\n" }, { "path": "codecov.yml", "content": "codecov:\n token: e0f2d44f-c6a7-469a-a688-37c72c0f18f9\n" }, { "path": "config/config.go", "content": "package config\n\n// FakeHeaders fake http headers\nvar FakeHeaders = map[string]string{\n\t\"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n\t\"Accept-Charset\": \"UTF-8,*;q=0.5\",\n\t\"Accept-Encoding\": \"gzip,deflate,sdch\",\n\t\"Accept-Language\": \"en-US,en;q=0.8\",\n\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36\",\n}\n" }, { "path": "downloader/downloader.go", "content": "package downloader\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/cheggaaa/pb/v3\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\n// Options defines options used in downloading.\ntype Options struct {\n\tInfoOnly bool\n\tSilent bool\n\tStream string\n\tAudioOnly bool\n\tRefer string\n\tOutputPath string\n\tOutputName string\n\tFileNameLength int\n\tCaption bool\n\tEmbedSubtitle bool\n\n\tMultiThread bool\n\tThreadNumber int\n\tRetryTimes int\n\tChunkSizeMB int\n\t// Aria2\n\tUseAria2RPC bool\n\tAria2Token string\n\tAria2Method string\n\tAria2Addr string\n}\n\n// Downloader is the default downloader.\ntype Downloader struct {\n\tBar *pb.ProgressBar\n\toption Options\n}\n\nconst (\n\tDOWNLOAD_FILE_EXT = \".download\"\n)\n\nfunc progressBar(size int64) *pb.ProgressBar {\n\ttmpl := `{{counters .}} {{bar . \"[\" \"=\" \">\" \"-\" \"]\"}} {{speed .}} {{percent . | green}} {{rtime .}}`\n\treturn pb.New64(size).\n\t\tSet(pb.Bytes, true).\n\t\tSetMaxWidth(1000).\n\t\tSetTemplate(pb.ProgressBarTemplate(tmpl))\n}\n\n// New returns a new Downloader implementation.\nfunc New(option Options) *Downloader {\n\tdownloader := &Downloader{\n\t\toption: option,\n\t}\n\treturn downloader\n}\n\n// caption downloads danmaku, subtitles, etc\nfunc (downloader *Downloader) caption(url, fileName, ext string, transform func([]byte) ([]byte, error)) error {\n\trefer := downloader.option.Refer\n\tif refer == \"\" {\n\t\trefer = url\n\t}\n\tbody, err := request.GetByte(url, refer, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif transform != nil {\n\t\tbody, err = transform(body)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tfilePath, err := utils.FilePath(fileName, ext, downloader.option.FileNameLength, downloader.option.OutputPath, true)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfile, fileError := os.Create(filePath)\n\tif fileError != nil {\n\t\treturn fileError\n\t}\n\tdefer file.Close() // nolint\n\n\tif _, err = file.Write(body); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (downloader *Downloader) writeFile(url string, file *os.File, headers map[string]string) (int64, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, headers)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tbarWriter := downloader.Bar.NewProxyWriter(file)\n\t// Note that io.Copy reads 32kb(maximum) from input and writes them to output, then repeats.\n\t// So don't worry about memory.\n\twritten, copyErr := io.Copy(barWriter, res.Body)\n\tif copyErr != nil && copyErr != io.EOF {\n\t\treturn written, errors.Errorf(\"file copy error: %s\", copyErr)\n\t}\n\treturn written, nil\n}\n\nfunc (downloader *Downloader) save(part *extractors.Part, refer, fileName string) error {\n\tfilePath, err := utils.FilePath(fileName, part.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfileSize, exists, err := utils.FileSize(filePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Skip segment file\n\t// TODO: Live video URLs will not return the size\n\tif exists && fileSize == part.Size {\n\t\tdownloader.Bar.Add64(fileSize)\n\t\treturn nil\n\t}\n\n\ttempFilePath := filePath + DOWNLOAD_FILE_EXT\n\ttempFileSize, _, err := utils.FileSize(tempFilePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\theaders := map[string]string{\n\t\t\"Referer\": refer,\n\t}\n\tvar (\n\t\tfile *os.File\n\t\tfileError error\n\t)\n\tif tempFileSize > 0 {\n\t\t// range start from 0, 0-1023 means the first 1024 bytes of the file\n\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-\", tempFileSize)\n\t\tfile, fileError = os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY, 0644)\n\t\tdownloader.Bar.Add64(tempFileSize)\n\t} else {\n\t\tfile, fileError = os.Create(tempFilePath)\n\t}\n\tif fileError != nil {\n\t\treturn fileError\n\t}\n\n\t// close and rename temp file at the end of this function\n\tdefer func() {\n\t\t// must close the file before rename or it will cause\n\t\t// `The process cannot access the file because it is being used by another process.` error.\n\t\tfile.Close() // nolint\n\t\tif err == nil {\n\t\t\tos.Rename(tempFilePath, filePath) // nolint\n\t\t}\n\t}()\n\n\tif downloader.option.ChunkSizeMB > 0 {\n\t\tvar start, end, chunkSize int64\n\t\tchunkSize = int64(downloader.option.ChunkSizeMB) * 1024 * 1024\n\t\tremainingSize := part.Size\n\t\tif tempFileSize > 0 {\n\t\t\tstart = tempFileSize\n\t\t\tremainingSize -= tempFileSize\n\t\t}\n\t\tchunk := remainingSize / chunkSize\n\t\tif remainingSize%chunkSize != 0 {\n\t\t\tchunk++\n\t\t}\n\t\tvar i int64 = 1\n\t\tfor ; i <= chunk; i++ {\n\t\t\tend = start + chunkSize - 1\n\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-%d\", start, end)\n\t\t\ttemp := start\n\t\t\tfor i := 0; ; i++ {\n\t\t\t\twritten, err := downloader.writeFile(part.URL, file, headers)\n\t\t\t\tif err == nil {\n\t\t\t\t\tbreak\n\t\t\t\t} else if i+1 >= downloader.option.RetryTimes {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\ttemp += written\n\t\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-%d\", temp, end)\n\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t}\n\t\t\tstart = end + 1\n\t\t}\n\t} else {\n\t\ttemp := tempFileSize\n\t\tfor i := 0; ; i++ {\n\t\t\twritten, err := downloader.writeFile(part.URL, file, headers)\n\t\t\tif err == nil {\n\t\t\t\tbreak\n\t\t\t} else if i+1 >= downloader.option.RetryTimes {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttemp += written\n\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-\", temp)\n\t\t\ttime.Sleep(1 * time.Second)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (downloader *Downloader) multiThreadSave(dataPart *extractors.Part, refer, fileName string) error {\n\tfilePath, err := utils.FilePath(fileName, dataPart.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfileSize, exists, err := utils.FileSize(filePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Skip segment file\n\t// TODO: Live video URLs will not return the size\n\tif exists && fileSize == dataPart.Size {\n\t\tdownloader.Bar.Add64(fileSize)\n\t\treturn nil\n\t}\n\ttmpFilePath := filePath + DOWNLOAD_FILE_EXT\n\ttmpFileSize, tmpExists, err := utils.FileSize(tmpFilePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif tmpExists {\n\t\tif tmpFileSize == dataPart.Size {\n\t\t\tdownloader.Bar.Add64(dataPart.Size)\n\t\t\treturn os.Rename(tmpFilePath, filePath)\n\t\t}\n\n\t\tif err = os.Remove(tmpFilePath); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Scan all parts\n\tparts, err := readDirAllFilePart(filePath, fileName, dataPart.Ext)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar unfinishedPart []*FilePartMeta\n\tsavedSize := int64(0)\n\tif len(parts) > 0 {\n\t\tlastEnd := int64(-1)\n\t\tfor i, part := range parts {\n\t\t\t// If some parts are lost, re-insert one part.\n\t\t\tif part.Start-lastEnd != 1 {\n\t\t\t\tnewPart := &FilePartMeta{\n\t\t\t\t\tIndex: part.Index - 0.000001,\n\t\t\t\t\tStart: lastEnd + 1,\n\t\t\t\t\tEnd: part.Start - 1,\n\t\t\t\t\tCur: lastEnd + 1,\n\t\t\t\t}\n\t\t\t\ttmp := append([]*FilePartMeta{}, parts[:i]...)\n\t\t\t\ttmp = append(tmp, newPart)\n\t\t\t\tparts = append(tmp, parts[i:]...)\n\t\t\t\tunfinishedPart = append(unfinishedPart, newPart)\n\t\t\t}\n\t\t\t// When the part has been downloaded in whole, part.Cur is equal to part.End + 1\n\t\t\tif part.Cur <= part.End+1 {\n\t\t\t\tsavedSize += part.Cur - part.Start\n\t\t\t\tif part.Cur < part.End+1 {\n\t\t\t\t\tunfinishedPart = append(unfinishedPart, part)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// The size of this part has been saved greater than the part size, delete it transparently and re-download.\n\t\t\t\terr = os.Remove(filePartPath(filePath, part))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tpart.Cur = part.Start\n\t\t\t\tunfinishedPart = append(unfinishedPart, part)\n\t\t\t}\n\t\t\tlastEnd = part.End\n\t\t}\n\t\tif lastEnd != dataPart.Size-1 {\n\t\t\tnewPart := &FilePartMeta{\n\t\t\t\tIndex: parts[len(parts)-1].Index + 1,\n\t\t\t\tStart: lastEnd + 1,\n\t\t\t\tEnd: dataPart.Size - 1,\n\t\t\t\tCur: lastEnd + 1,\n\t\t\t}\n\t\t\tparts = append(parts, newPart)\n\t\t\tunfinishedPart = append(unfinishedPart, newPart)\n\t\t}\n\t} else {\n\t\tvar start, end, partSize int64\n\t\tvar i float32\n\t\tpartSize = dataPart.Size / int64(downloader.option.ThreadNumber)\n\t\ti = 0\n\t\tfor start < dataPart.Size {\n\t\t\tend = start + partSize - 1\n\t\t\tif end > dataPart.Size {\n\t\t\t\tend = dataPart.Size - 1\n\t\t\t} else if int(i+1) == downloader.option.ThreadNumber && end < dataPart.Size {\n\t\t\t\tend = dataPart.Size - 1\n\t\t\t}\n\t\t\tpart := &FilePartMeta{\n\t\t\t\tIndex: i,\n\t\t\t\tStart: start,\n\t\t\t\tEnd: end,\n\t\t\t\tCur: start,\n\t\t\t}\n\t\t\tparts = append(parts, part)\n\t\t\tunfinishedPart = append(unfinishedPart, part)\n\t\t\tstart = end + 1\n\t\t\ti++\n\t\t}\n\t}\n\tif savedSize > 0 {\n\t\tdownloader.Bar.Add64(savedSize)\n\t\tif savedSize == dataPart.Size {\n\t\t\treturn mergeMultiPart(filePath, parts)\n\t\t}\n\t}\n\n\twgp := utils.NewWaitGroupPool(downloader.option.ThreadNumber)\n\tvar errs []error\n\tvar mu sync.Mutex\n\tfor _, part := range unfinishedPart {\n\t\twgp.Add()\n\t\tgo func(part *FilePartMeta) {\n\t\t\tfile, err := os.OpenFile(filePartPath(filePath, part), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)\n\t\t\tif err != nil {\n\t\t\t\tmu.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tmu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer func() {\n\t\t\t\tfile.Close() // nolint\n\t\t\t\twgp.Done()\n\t\t\t}()\n\n\t\t\tvar end, chunkSize int64\n\t\t\theaders := map[string]string{\n\t\t\t\t\"Referer\": refer,\n\t\t\t}\n\t\t\tif downloader.option.ChunkSizeMB <= 0 {\n\t\t\t\tchunkSize = part.End - part.Start + 1\n\t\t\t} else {\n\t\t\t\tchunkSize = int64(downloader.option.ChunkSizeMB) * 1024 * 1024\n\t\t\t}\n\t\t\tremainingSize := part.End - part.Cur + 1\n\t\t\tif part.Cur == part.Start {\n\t\t\t\t// Only write part to new file.\n\t\t\t\terr = writeFilePartMeta(file, part)\n\t\t\t\tif err != nil {\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor remainingSize > 0 {\n\t\t\t\tend = computeEnd(part.Cur, chunkSize, part.End)\n\t\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-%d\", part.Cur, end)\n\t\t\t\ttemp := part.Cur\n\t\t\t\tfor i := 0; ; i++ {\n\t\t\t\t\twritten, err := downloader.writeFile(dataPart.URL, file, headers)\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tremainingSize -= chunkSize\n\t\t\t\t\t\tbreak\n\t\t\t\t\t} else if i+1 >= downloader.option.RetryTimes {\n\t\t\t\t\t\tmu.Lock()\n\t\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t\t\tmu.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\ttemp += written\n\t\t\t\t\theaders[\"Range\"] = fmt.Sprintf(\"bytes=%d-%d\", temp, end)\n\t\t\t\t}\n\t\t\t\tpart.Cur = end + 1\n\t\t\t}\n\t\t}(part)\n\t}\n\twgp.Wait()\n\tif len(errs) > 0 {\n\t\treturn errs[0]\n\t}\n\treturn mergeMultiPart(filePath, parts)\n}\n\nfunc filePartPath(filepath string, part *FilePartMeta) string {\n\treturn fmt.Sprintf(\"%s.part%f\", filepath, part.Index)\n}\n\nfunc computeEnd(s, chunkSize, max int64) int64 {\n\tvar end int64\n\tend = s + chunkSize - 1\n\tif end > max {\n\t\tend = max\n\t}\n\treturn end\n}\n\nfunc readDirAllFilePart(filePath, filename, extname string) ([]*FilePartMeta, error) {\n\tdirPath := filepath.Dir(filePath)\n\tdir, err := os.Open(dirPath)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer dir.Close() // nolint\n\tfns, err := dir.Readdir(0)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar metas []*FilePartMeta\n\treg := regexp.MustCompile(fmt.Sprintf(\"%s.%s.part.+\", regexp.QuoteMeta(filename), extname))\n\tfor _, fn := range fns {\n\t\tif reg.MatchString(fn.Name()) {\n\t\t\tmeta, err := parseFilePartMeta(path.Join(dirPath, fn.Name()), fn.Size())\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tmetas = append(metas, meta)\n\t\t}\n\t}\n\tsort.SliceStable(metas, func(i, j int) bool {\n\t\treturn metas[i].Index < metas[j].Index\n\t})\n\treturn metas, nil\n}\n\nfunc parseFilePartMeta(filepath string, fileSize int64) (*FilePartMeta, error) {\n\tmeta := new(FilePartMeta)\n\tsize := binary.Size(*meta)\n\tfile, err := os.OpenFile(filepath, os.O_RDWR, 0666)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer file.Close() // nolint\n\tvar buf [512]byte\n\treadSize, err := file.ReadAt(buf[0:size], 0)\n\tif err != nil && err != io.EOF {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif readSize < size {\n\t\treturn nil, errors.Errorf(\"the file has been broken, please delete all part files and re-download\")\n\t}\n\terr = binary.Read(bytes.NewBuffer(buf[:size]), binary.LittleEndian, meta)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsavedSize := fileSize - int64(binary.Size(meta))\n\tmeta.Cur = meta.Start + savedSize\n\treturn meta, nil\n}\n\nfunc writeFilePartMeta(file *os.File, meta *FilePartMeta) error {\n\treturn binary.Write(file, binary.LittleEndian, meta)\n}\n\nfunc mergeMultiPart(filepath string, parts []*FilePartMeta) error {\n\ttempFilePath := filepath + DOWNLOAD_FILE_EXT\n\ttempFile, err := os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar partFiles []*os.File\n\tdefer func() {\n\t\tfor _, f := range partFiles {\n\t\t\tf.Close() // nolint\n\t\t\tos.Remove(f.Name()) // nolint\n\t\t}\n\t}()\n\tfor _, part := range parts {\n\t\tfile, err := os.Open(filePartPath(filepath, part))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tpartFiles = append(partFiles, file)\n\t\t_, err = file.Seek(int64(binary.Size(part)), 0)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err = io.Copy(tempFile, file)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\ttempFile.Close() // nolint\n\terr = os.Rename(tempFilePath, filepath)\n\treturn err\n}\n\nfunc (downloader *Downloader) aria2(title string, stream *extractors.Stream) error {\n\trpcData := Aria2RPCData{\n\t\tJSONRPC: \"2.0\",\n\t\tID: \"lux\", // can be modified\n\t\tMethod: \"aria2.addUri\",\n\t}\n\trpcData.Params[0] = \"token:\" + downloader.option.Aria2Token\n\tvar urls []string\n\tfor _, p := range stream.Parts {\n\t\turls = append(urls, p.URL)\n\t}\n\tvar inputs Aria2Input\n\tinputs.Header = append(inputs.Header, \"Referer: \"+downloader.option.Refer)\n\tfor i := range urls {\n\t\trpcData.Params[1] = urls[i : i+1]\n\t\tinputs.Out = fmt.Sprintf(\"%s[%d].%s\", title, i, stream.Parts[0].Ext)\n\t\trpcData.Params[2] = &inputs\n\t\tjsonData, err := json.Marshal(rpcData)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treqURL := fmt.Sprintf(\"%s://%s/jsonrpc\", downloader.option.Aria2Method, downloader.option.Aria2Addr)\n\t\treq, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewBuffer(jsonData))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treq.Header.Set(\"Content-Type\", \"application/json\")\n\n\t\tvar client = http.Client{Timeout: 30 * time.Second}\n\t\tres, err := client.Do(req)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// The http Client and Transport guarantee that Body is always\n\t\t// non-nil, even on responses without a body or responses with\n\t\t// a zero-length body.\n\t\tres.Body.Close() // nolint\n\t}\n\treturn nil\n}\n\n// Download download urls\nfunc (downloader *Downloader) Download(data *extractors.Data) error {\n\tif len(data.Streams) == 0 {\n\t\treturn errors.Errorf(\"no streams in title %s\", data.Title)\n\t}\n\n\tsortedStreams := genSortedStreams(data.Streams)\n\tif downloader.option.InfoOnly {\n\t\tprintInfo(data, sortedStreams)\n\t\treturn nil\n\t}\n\n\ttitle := downloader.option.OutputName\n\tif title == \"\" {\n\t\ttitle = data.Title\n\t}\n\ttitle = utils.FileName(title, \"\", downloader.option.FileNameLength)\n\n\tstreamName := downloader.option.Stream\n\tif streamName == \"\" {\n\t\tstreamName = sortedStreams[0].ID\n\t}\n\tstream, ok := data.Streams[streamName]\n\tif !ok {\n\t\treturn errors.Errorf(\"no stream named %s\", streamName)\n\t}\n\n\tif downloader.option.AudioOnly {\n\t\tvar isFound bool\n\t\treg, err := regexp.Compile(\"audio+\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, s := range sortedStreams {\n\t\t\t// Looking for the best quality\n\t\t\tif reg.MatchString(s.Quality) {\n\t\t\t\tisFound = true\n\t\t\t\tstream = data.Streams[s.ID]\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tfor _, part := range s.Parts {\n\t\t\t\tif part.Ext == \"m4a\" {\n\t\t\t\t\tisFound = true\n\t\t\t\t\tstream = data.Streams[s.ID]\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif !isFound {\n\t\t\treturn errors.Errorf(\"No audio stream found\")\n\t\t}\n\t}\n\n\tif !downloader.option.Silent {\n\t\tprintStreamInfo(data, stream)\n\t}\n\n\t// download caption\n\tvar subtitlePaths []string\n\tvar subtitleLangs []string\n\tvar subtitleFilesToDelete []string\n\tif downloader.option.Caption && data.Captions != nil {\n\t\tfmt.Println(\"\\nDownloading captions...\")\n\t\tfor k, v := range data.Captions {\n\t\t\tif v != nil {\n\t\t\t\tfmt.Printf(\"Downloading %s ...\\n\", k)\n\t\t\t\tif err := downloader.caption(v.URL, title, v.Ext, v.Transform); err != nil {\n\t\t\t\t\t// nolint\n\t\t\t\t} else if downloader.option.EmbedSubtitle {\n\t\t\t\t\tsubtitlePath, _ := utils.FilePath(title, v.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, true)\n\t\t\t\t\tsubtitleFilesToDelete = append(subtitleFilesToDelete, subtitlePath)\n\t\t\t\t\tif strings.HasSuffix(v.Ext, \"xml\") {\n\t\t\t\t\t\tif srtPath, err := utils.ConvertXMLFileToSRT(subtitlePath); err == nil {\n\t\t\t\t\t\t\tsubtitlePath = srtPath\n\t\t\t\t\t\t\tsubtitleFilesToDelete = append(subtitleFilesToDelete, srtPath)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tsubtitlePaths = append(subtitlePaths, subtitlePath)\n\t\t\t\t\tsubtitleLangs = append(subtitleLangs, k)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Use aria2 rpc to download\n\tif downloader.option.UseAria2RPC {\n\t\treturn downloader.aria2(title, stream)\n\t}\n\n\t// Skip the complete file that has been merged\n\tmergedFilePath, err := utils.FilePath(title, stream.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, mergedFileExists, err := utils.FileSize(mergedFilePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// After the merge, the file size has changed, so we do not check whether the size matches\n\tif mergedFileExists {\n\t\tfmt.Printf(\"%s: file already exists, skipping\\n\", mergedFilePath)\n\t\treturn nil\n\t}\n\n\tdownloader.Bar = progressBar(stream.Size)\n\tif !downloader.option.Silent {\n\t\tdownloader.Bar.Start()\n\t}\n\tif len(stream.Parts) == 1 {\n\t\t// only one fragment\n\t\tvar err error\n\t\tif downloader.option.MultiThread {\n\t\t\terr = downloader.multiThreadSave(stream.Parts[0], data.URL, title)\n\t\t} else {\n\t\t\terr = downloader.save(stream.Parts[0], data.URL, title)\n\t\t}\n\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdownloader.Bar.Finish()\n\n\t\tif downloader.option.EmbedSubtitle && len(subtitlePaths) > 0 {\n\t\t\tif !downloader.option.Silent {\n\t\t\t\tfmt.Println(\"Embedding subtitles...\")\n\t\t\t}\n\t\t\tif err := utils.EmbedSubtitles(mergedFilePath, subtitlePaths, subtitleLangs); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, path := range subtitleFilesToDelete {\n\t\t\t\tos.Remove(path)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\twgp := utils.NewWaitGroupPool(downloader.option.ThreadNumber)\n\t// multiple fragments\n\terrs := make([]error, 0)\n\tlock := sync.Mutex{}\n\tparts := make([]string, len(stream.Parts))\n\tfor index, part := range stream.Parts {\n\t\tif len(errs) > 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tif downloader.option.AudioOnly && (part.Ext != \"m4a\") {\n\t\t\tcontinue\n\t\t}\n\n\t\tpartFileName := fmt.Sprintf(\"%s[%d]\", title, index)\n\t\tpartFilePath, err := utils.FilePath(partFileName, part.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tparts[index] = partFilePath\n\n\t\twgp.Add()\n\t\tgo func(part *extractors.Part, fileName string) {\n\t\t\tdefer wgp.Done()\n\t\t\tvar err error\n\t\t\tif downloader.option.MultiThread {\n\t\t\t\terr = downloader.multiThreadSave(part, data.URL, fileName)\n\t\t\t} else {\n\t\t\t\terr = downloader.save(part, data.URL, fileName)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t}\n\t\t}(part, partFileName)\n\t}\n\twgp.Wait()\n\tif len(errs) > 0 {\n\t\treturn errs[0]\n\t}\n\tdownloader.Bar.Finish()\n\n\tif data.Type != extractors.DataTypeVideo || downloader.option.AudioOnly {\n\t\treturn nil\n\t}\n\n\tif !downloader.option.Silent {\n\t\tfmt.Printf(\"Merging video parts into %s\\n\", mergedFilePath)\n\t}\n\tif stream.Ext != \"mp4\" || stream.NeedMux {\n\t\tif err := utils.MergeFilesWithSameExtension(parts, mergedFilePath); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tif err := utils.MergeToMP4(parts, mergedFilePath, title); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif downloader.option.EmbedSubtitle && len(subtitlePaths) > 0 {\n\t\tif !downloader.option.Silent {\n\t\t\tfmt.Println(\"Embedding subtitles...\")\n\t\t}\n\t\tif err := utils.EmbedSubtitles(mergedFilePath, subtitlePaths, subtitleLangs); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, path := range subtitleFilesToDelete {\n\t\t\tos.Remove(path)\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "downloader/downloader_test.go", "content": "package downloader\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttestCases := []struct {\n\t\tname string\n\t\tdata *extractors.Data\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\tdata: &extractors.Data{\n\t\t\t\tSite: \"douyin\",\n\t\t\t\tTitle: \"test\",\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t\tURL: \"https://www.douyin.com\",\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"default\": {\n\t\t\t\t\t\tID: \"default\",\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: \"https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f9a0000bc117isuatl67cees890&line=0\",\n\t\t\t\t\t\t\t\tSize: 4927877,\n\t\t\t\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"multi-stream test\",\n\t\t\tdata: &extractors.Data{\n\t\t\t\tSite: \"douyin\",\n\t\t\t\tTitle: \"test2\",\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t\tURL: \"https://www.douyin.com\",\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"miaopai\": {\n\t\t\t\t\t\tID: \"miaopai\",\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: \"https://txycdn.miaopai.com/stream/KwR26jUGh2ySnVjYbQiFmomNjP14LtMU3vi6sQ__.mp4?ssig=6594aa01a78e78f50c65c164d186ba9e&time_stamp=1537070910786\",\n\t\t\t\t\t\t\t\tSize: 4011590,\n\t\t\t\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSize: 4011590,\n\t\t\t\t\t},\n\t\t\t\t\t\"douyin\": {\n\t\t\t\t\t\tID: \"douyin\",\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: \"https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f9a0000bc117isuatl67cees890&line=0\",\n\t\t\t\t\t\t\t\tSize: 4927877,\n\t\t\t\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSize: 4927877,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\tdata: &extractors.Data{\n\t\t\t\tSite: \"bcy\",\n\t\t\t\tTitle: \"bcy image test\",\n\t\t\t\tType: extractors.DataTypeImage,\n\t\t\t\tURL: \"https://www.bcyimg.com\",\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"default\": {\n\t\t\t\t\t\tID: \"default\",\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: \"http://img5.bcyimg.com/coser/143767/post/c0j7x/0d713eb41a614053ac6a3b146914f6bc.jpg/w650\",\n\t\t\t\t\t\t\t\tSize: 56107,\n\t\t\t\t\t\t\t\tExt: \"jpg\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: \"http://img9.bcyimg.com/coser/143767/post/c0j7x/d17e9b8587794d939a1363c5f715014b.jpg/w650\",\n\t\t\t\t\t\t\t\tSize: 142100,\n\t\t\t\t\t\t\t\tExt: \"jpg\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, testCase := range testCases {\n\t\terr := New(Options{}).Download(testCase.data)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n}\n" }, { "path": "downloader/types.go", "content": "package downloader\n\n// Aria2RPCData defines the data structure of json RPC 2.0 info for Aria2\ntype Aria2RPCData struct {\n\t// More info about RPC interface please refer to\n\t// https://aria2.github.io/manual/en/html/aria2c.html#rpc-interface\n\tJSONRPC string `json:\"jsonrpc\"`\n\tID string `json:\"id\"`\n\t// For a simple download, only implemented `addUri`\n\tMethod string `json:\"method\"`\n\t// secret, uris, options\n\tParams [3]interface{} `json:\"params\"`\n}\n\n// Aria2Input is options for `aria2.addUri`\n// https://aria2.github.io/manual/en/html/aria2c.html#id3\ntype Aria2Input struct {\n\t// The file name of the downloaded file\n\tOut string `json:\"out\"`\n\t// For a simple download, only add headers\n\tHeader []string `json:\"header\"`\n}\n\n// FilePartMeta defines the data structure of file meta info.\ntype FilePartMeta struct {\n\tIndex float32\n\tStart int64\n\tEnd int64\n\tCur int64\n}\n" }, { "path": "downloader/utils.go", "content": "package downloader\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/fatih/color\"\n\n\t\"github.com/iawia002/lux/extractors\"\n)\n\nvar (\n\tblue = color.New(color.FgBlue)\n\tcyan = color.New(color.FgCyan)\n)\n\nfunc genSortedStreams(streams map[string]*extractors.Stream) []*extractors.Stream {\n\tsortedStreams := make([]*extractors.Stream, 0, len(streams))\n\tfor _, data := range streams {\n\t\tsortedStreams = append(sortedStreams, data)\n\t}\n\tif len(sortedStreams) > 1 {\n\t\tsort.SliceStable(\n\t\t\tsortedStreams, func(i, j int) bool { return sortedStreams[i].Size > sortedStreams[j].Size },\n\t\t)\n\t}\n\treturn sortedStreams\n}\n\nfunc printHeader(data *extractors.Data) {\n\tfmt.Println()\n\tcyan.Printf(\" Site: \") // nolint\n\tfmt.Println(data.Site)\n\tcyan.Printf(\" Title: \") // nolint\n\tfmt.Println(data.Title)\n\tcyan.Printf(\" Type: \") // nolint\n\tfmt.Println(data.Type)\n}\n\nfunc printStream(stream *extractors.Stream) {\n\tblue.Println(fmt.Sprintf(\" [%s] -------------------\", stream.ID)) // nolint\n\tif stream.Quality != \"\" {\n\t\tcyan.Printf(\" Quality: \") // nolint\n\t\tfmt.Println(stream.Quality)\n\t}\n\tcyan.Printf(\" Size: \") // nolint\n\tfmt.Printf(\"%.2f MiB (%d Bytes)\\n\", float64(stream.Size)/(1024*1024), stream.Size)\n\tcyan.Printf(\" # download with: \") // nolint\n\tfmt.Printf(\"lux -f %s ...\\n\\n\", stream.ID)\n}\n\nfunc printInfo(data *extractors.Data, sortedStreams []*extractors.Stream) {\n\tprintHeader(data)\n\tif len(data.Captions) > 0 {\n\t\tcyan.Printf(\" Captions: \") // nolint\n\t\tlanguages := make([]string, 0, len(data.Captions))\n\t\tfor lang := range data.Captions {\n\t\t\tlanguages = append(languages, lang)\n\t\t}\n\t\tsort.Strings(languages)\n\t\tcaptionList := \"\"\n\t\tfor _, lang := range languages {\n\t\t\tcaption := data.Captions[lang]\n\t\t\tif caption == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcaptionList += fmt.Sprintf(\"%s \", lang)\n\t\t}\n\t\tfmt.Println(captionList)\n\t}\n\tcyan.Printf(\" Streams: \") // nolint\n\tfmt.Println(\"# All available quality\")\n\tfor _, stream := range sortedStreams {\n\t\tprintStream(stream)\n\t}\n}\n\nfunc printStreamInfo(data *extractors.Data, stream *extractors.Stream) {\n\tprintHeader(data)\n\n\tcyan.Printf(\" Stream: \") // nolint\n\tfmt.Println()\n\tprintStream(stream)\n}\n" }, { "path": "extractors/acfun/acfun.go", "content": "package acfun\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"regexp\"\n\n\tjsoniter \"github.com/json-iterator/go\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"acfun\", New())\n}\n\nconst (\n\tbangumiDataPattern = \"window.pageInfo = window.bangumiData = (.*);\"\n\tbangumiListPattern = \"window.bangumiList = (.*);\"\n\n\tbangumiHTMLURL = \"https://www.acfun.cn/bangumi/aa%d_36188_%d\"\n\n\treferer = \"https://www.acfun.cn\"\n)\n\ntype extractor struct{}\n\n// New returns a new acfun bangumi extractor\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract ...\nfunc (e *extractor) Extract(URL string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.GetByte(URL, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tepDatas := make([]*episodeData, 0)\n\n\tif option.Playlist {\n\t\tlist, err := resolvingEpisodes(html)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\titems := utils.NeedDownloadList(option.Items, option.ItemStart, option.ItemEnd, len(list.Episodes))\n\n\t\tfor _, item := range items {\n\t\t\tepDatas = append(epDatas, list.Episodes[item-1])\n\t\t}\n\t} else {\n\t\tbgData, _, err := resolvingData(html)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tepDatas = append(epDatas, &bgData.episodeData)\n\t}\n\n\tdatas := make([]*extractors.Data, 0)\n\n\twgp := utils.NewWaitGroupPool(option.ThreadNumber)\n\tfor _, epData := range epDatas {\n\t\tt := epData\n\t\twgp.Add()\n\t\tgo func() {\n\t\t\tdefer wgp.Done()\n\t\t\tdatas = append(datas, extractBangumi(concatURL(t)))\n\t\t}()\n\t}\n\twgp.Wait()\n\treturn datas, nil\n}\n\nfunc concatURL(epData *episodeData) string {\n\treturn fmt.Sprintf(bangumiHTMLURL, epData.BangumiID, epData.ItemID)\n}\n\nfunc extractBangumi(URL string) *extractors.Data {\n\tvar err error\n\thtml, err := request.GetByte(URL, referer, nil)\n\tif err != nil {\n\t\treturn extractors.EmptyData(URL, err)\n\t}\n\n\t_, vInfo, err := resolvingData(html)\n\tif err != nil {\n\t\treturn extractors.EmptyData(URL, err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\n\tfor _, stm := range vInfo.AdaptationSet[0].Streams {\n\t\tm3u8URL, err := url.Parse(stm.URL)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(URL, err)\n\t\t}\n\n\t\turls, err := utils.M3u8URLs(m3u8URL.String())\n\t\tif err != nil {\n\t\t\t_, err = url.Parse(stm.URL)\n\t\t\tif err != nil {\n\t\t\t\treturn extractors.EmptyData(URL, err)\n\t\t\t}\n\n\t\t\turls, err = utils.M3u8URLs(stm.BackURL)\n\t\t\tif err != nil {\n\t\t\t\treturn extractors.EmptyData(URL, err)\n\t\t\t}\n\t\t}\n\n\t\t// There is no size information in the m3u8 file and the calculation will take too much time, just ignore it.\n\t\tparts := make([]*extractors.Part, 0)\n\t\tfor _, u := range urls {\n\t\t\tparts = append(parts, &extractors.Part{\n\t\t\t\tURL: u,\n\t\t\t\tExt: \"ts\",\n\t\t\t})\n\t\t}\n\t\tstreams[stm.QualityLabel] = &extractors.Stream{\n\t\t\tID: stm.QualityType,\n\t\t\tParts: parts,\n\t\t\tQuality: stm.QualityType,\n\t\t\tNeedMux: false,\n\t\t}\n\t}\n\n\tdoc, err := parser.GetDoc(string(html))\n\tif err != nil {\n\t\treturn extractors.EmptyData(URL, err)\n\t}\n\tdata := &extractors.Data{\n\t\tSite: \"AcFun acfun.cn\",\n\t\tTitle: parser.Title(doc),\n\t\tType: extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tURL: URL,\n\t}\n\treturn data\n}\n\nfunc resolvingData(html []byte) (*bangumiData, *videoInfo, error) {\n\tbgData := &bangumiData{}\n\tvInfo := &videoInfo{}\n\n\tpattern, _ := regexp.Compile(bangumiDataPattern)\n\n\tgroups := pattern.FindSubmatch(html)\n\terr := jsoniter.Unmarshal(groups[1], bgData)\n\tif err != nil {\n\t\treturn nil, nil, errors.WithStack(err)\n\t}\n\n\terr = jsoniter.UnmarshalFromString(bgData.CurrentVideoInfo.KsPlayJSON, vInfo)\n\tif err != nil {\n\t\treturn nil, nil, errors.WithStack(err)\n\t}\n\treturn bgData, vInfo, nil\n}\n\nfunc resolvingEpisodes(html []byte) (*episodeList, error) {\n\tlist := &episodeList{}\n\tpattern, _ := regexp.Compile(bangumiListPattern)\n\n\tgroups := pattern.FindSubmatch(html)\n\terr := jsoniter.Unmarshal(groups[1], list)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn list, nil\n}\n" }, { "path": "extractors/acfun/acfun_test.go", "content": "package acfun\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.acfun.cn/bangumi/aa6000686_36188_1704167\",\n\t\t\t\tTitle: \"瑞克和莫蒂 第四季 ：第2话 注释版\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/acfun/types.go", "content": "package acfun\n\ntype episodeData struct {\n\tItemID int64 `json:\"itemId\"`\n\tEpisodeName string `json:\"episodeName\"`\n\tBangumiID int64 `json:\"bangumiId\"`\n\tVideoID int64 `json:\"videoId\"`\n}\n\ntype bangumiData struct {\n\tepisodeData\n\tBangumiTitle string `json:\"bangumiTitle\"`\n\tCurrentVideoInfo struct {\n\t\tKsPlayJSON string `json:\"ksPlayJson\"`\n\t} `json:\"currentVideoInfo\"`\n}\n\ntype videoInfo struct {\n\tAdaptationSet []struct {\n\t\tStreams streams `json:\"representation\"`\n\t} `json:\"adaptationSet\"`\n}\n\ntype streams []stream\n\ntype episodeList struct {\n\tEpisodes []*episodeData `json:\"items\"`\n}\n\ntype stream struct {\n\tID int64 `json:\"id\"`\n\tBackURL string `json:\"backUrl\"`\n\tCodecs string `json:\"codecs\"`\n\tURL string `json:\"url\"`\n\tBitRate int64 `json:\"avgBitrate\"`\n\tQualityType string `json:\"qualityType\"`\n\tQualityLabel string `json:\"qualityLabel\"`\n}\n" }, { "path": "extractors/bcy/bcy.go", "content": "package bcy\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"bcy\", New())\n}\n\ntype bcyData struct {\n\tDetail struct {\n\t\tPostData struct {\n\t\t\tMulti []struct {\n\t\t\t\tOriginalPath string `json:\"original_path\"`\n\t\t\t} `json:\"multi\"`\n\t\t} `json:\"post_data\"`\n\t} `json:\"detail\"`\n}\n\ntype extractor struct{}\n\n// New returns a bcy extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// parse json data\n\trep := strings.NewReplacer(`\\\"`, `\"`, `\\\\`, `\\`)\n\trealURLs := utils.MatchOneOf(html, `JSON.parse\\(\"(.+?)\"\\);`)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := rep.Replace(realURLs[1])\n\n\tvar data bcyData\n\tif err = json.Unmarshal([]byte(jsonString), &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle := strings.Replace(parser.Title(doc), \" - 半次元 banciyuan - ACG爱好者社区\", \"\", -1)\n\n\tparts := make([]*extractors.Part, 0, len(data.Detail.PostData.Multi))\n\tvar totalSize int64\n\tfor _, img := range data.Detail.PostData.Multi {\n\t\tsize, err := request.Size(img.OriginalPath, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\ttotalSize += size\n\t\t_, ext, err := utils.GetNameAndExt(img.OriginalPath)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL: img.OriginalPath,\n\t\t\tSize: size,\n\t\t\tExt: ext,\n\t\t})\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"半次元 bcy.net\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/bcy/bcy_test.go", "content": "package bcy\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://bcy.net/item/detail/6558738153367142664\",\n\t\t\t\tTitle: \"cos正片 命运石之门 牧濑红莉栖 克里斯蒂娜… - 半次元 - ACG爱好者社区\",\n\t\t\t\tSize: 13035763,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/bilibili/bilibili.go", "content": "package bilibili\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"slices\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\tbilibiliExtractor := New()\n\textractors.Register(\"bilibili\", bilibiliExtractor)\n\textractors.Register(\"b23\", bilibiliExtractor)\n}\n\nconst (\n\tbilibiliAPI = \"https://api.bilibili.com/x/player/playurl?\"\n\tbilibiliBangumiAPI = \"https://api.bilibili.com/pgc/player/web/playurl?\"\n\tbilibiliTokenAPI = \"https://api.bilibili.com/x/player/playurl/token?\"\n)\n\nconst referer = \"https://www.bilibili.com\"\n\nvar utoken string\n\nfunc genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (string, error) {\n\tvar (\n\t\terr error\n\t\tbaseAPIURL string\n\t\tparams string\n\t)\n\tif cookie != \"\" && utoken == \"\" {\n\t\tutoken, err = request.Get(\n\t\t\tfmt.Sprintf(\"%said=%d&cid=%d\", bilibiliTokenAPI, aid, cid),\n\t\t\treferer,\n\t\t\tnil,\n\t\t)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tvar t token\n\t\terr = json.Unmarshal([]byte(utoken), &t)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tif t.Code != 0 {\n\t\t\treturn \"\", errors.Errorf(\"cookie error: %s\", t.Message)\n\t\t}\n\t\tutoken = t.Data.Token\n\t}\n\tvar api string\n\tif bangumi {\n\t\t// The parameters need to be sorted by name\n\t\t// qn=0 flag makes the CDN address different every time\n\t\t// quality=120(4k) is the highest quality so far\n\t\tparams = fmt.Sprintf(\n\t\t\t\"cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16\",\n\t\t\tcid, bvid, quality,\n\t\t)\n\t\tbaseAPIURL = bilibiliBangumiAPI\n\t} else {\n\t\tparams = fmt.Sprintf(\n\t\t\t\"avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=2000\",\n\t\t\taid, cid, bvid, quality,\n\t\t)\n\t\tbaseAPIURL = bilibiliAPI\n\t}\n\tapi = baseAPIURL + params\n\t// bangumi utoken also need to put in params to sign, but the ordinary video doesn't need\n\tif !bangumi && utoken != \"\" {\n\t\tapi = fmt.Sprintf(\"%s&utoken=%s\", api, utoken)\n\t}\n\treturn api, nil\n}\n\ntype bilibiliOptions struct {\n\turl string\n\thtml string\n\tbangumi bool\n\taid int\n\tcid int\n\tbvid string\n\tpage int\n\tsubtitle string\n}\n\nfunc extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tdataString := utils.MatchOneOf(html, `const playurlSSRData = ({[\\s\\S]+})`)[1]\n\tepArrayString := utils.MatchOneOf(dataString, `\"episode_info\"\\s*:\\s*(.+?)\\s*,\\s*\"season_info\"`)[1]\n\tfullVideoIdString := utils.MatchOneOf(dataString, `\"videoId\"\\s*:\\s*\"(ep|ss)(\\d+)\"`)\n\tepSsString := fullVideoIdString[1] // \"ep\" or \"ss\"\n\tvideoIdString := fullVideoIdString[2]\n\n\tvar epArray EpVideoInfo\n\terr := json.Unmarshal([]byte(epArrayString), &epArray)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar data bangumiData\n\n\tvideoId, err := strconv.ParseInt(videoIdString, 10, 0)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif epArray.EpID == int(videoId) || (epSsString == \"ss\" && epArray.Title == \"第1话\") {\n\t\tdata.EpInfo = epArray\n\t}\n\tdata.EpList = append(data.EpList, epArray)\n\n\tsort.Slice(data.EpList, func(i, j int) bool {\n\t\treturn data.EpList[i].EpID < data.EpList[j].EpID\n\t})\n\n\tif !extractOption.Playlist {\n\t\taid := data.EpInfo.Aid\n\t\tcid := data.EpInfo.Cid\n\t\tbvid := data.EpInfo.Bvid\n\t\ttitleFormat := data.EpInfo.Title\n\t\tlongTitle := data.EpInfo.LongTitle\n\t\tif aid <= 0 || cid <= 0 || bvid == \"\" {\n\t\t\taid = data.EpList[0].Aid\n\t\t\tcid = data.EpList[0].Cid\n\t\t\tbvid = data.EpList[0].Bvid\n\t\t\ttitleFormat = data.EpList[0].Title\n\t\t\tlongTitle = data.EpList[0].LongTitle\n\t\t}\n\t\toptions := bilibiliOptions{\n\t\t\turl: url,\n\t\t\thtml: html,\n\t\t\tbangumi: true,\n\t\t\taid: aid,\n\t\t\tcid: cid,\n\t\t\tbvid: bvid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", titleFormat, longTitle),\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle bangumi playlist\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(data.EpList))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range data.EpList {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\tid := u.EpID\n\t\tif id == 0 {\n\t\t\tid = u.EpID\n\t\t}\n\t\t// html content can't be reused here\n\t\toptions := bilibiliOptions{\n\t\t\turl: fmt.Sprintf(\"https://www.bilibili.com/bangumi/play/ep%d\", id),\n\t\t\tbangumi: true,\n\t\t\taid: u.Aid,\n\t\t\tcid: u.Cid,\n\t\t\tbvid: u.Bvid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", u.Title, u.LongTitle),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\nfunc getMultiPageData(html string) (*multiPage, error) {\n\tvar data multiPage\n\tmultiPageDataString := utils.MatchOneOf(\n\t\thtml, `window.__INITIAL_STATE__=(.+?);\\(function`,\n\t)\n\tif multiPageDataString == nil {\n\t\treturn &data, errors.New(\"this page has no playlist\")\n\t}\n\terr := json.Unmarshal([]byte(multiPageDataString[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn &data, nil\n}\n\nfunc extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tmatches := utils.MatchAll(html, \"<\\\\s*script[^>]*>\\\\s*window\\\\.__INITIAL_STATE__=([\\\\s\\\\S]*?);\\\\s?\\\\(function[\\\\s\\\\S]*?<\\\\/\\\\s*script\\\\s*>\")\n\tif len(matches) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tif len(matches[0]) < 2 {\n\t\treturn nil, errors.New(\"could not find video in page\")\n\t}\n\n\tvar festivalData festival\n\terr := json.Unmarshal([]byte(matches[0][1]), &festivalData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\toptions := bilibiliOptions{\n\t\turl: url,\n\t\thtml: html,\n\t\taid: festivalData.VideoInfo.Aid,\n\t\tbvid: festivalData.VideoInfo.BVid,\n\t\tcid: festivalData.VideoInfo.Cid,\n\t\tpage: 0,\n\t}\n\n\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n}\n\nfunc extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tpageData, err := getMultiPageData(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif !extractOption.Playlist {\n\t\t// handle URL that has a playlist, mainly for unified titles\n\t\t//

tag does not include subtitles\n\t\t// bangumi doesn't need this\n\t\tpageString := utils.MatchOneOf(url, `\\?p=(\\d+)`)\n\t\tvar p int\n\t\tif pageString == nil {\n\t\t\t// https://www.bilibili.com/video/av20827366/\n\t\t\tp = 1\n\t\t} else {\n\t\t\t// https://www.bilibili.com/video/av20827366/?p=2\n\t\t\tp, _ = strconv.Atoi(pageString[1])\n\t\t}\n\n\t\tif len(pageData.VideoData.Pages) < p || p < 1 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tpage := pageData.VideoData.Pages[p-1]\n\t\toptions := bilibiliOptions{\n\t\t\turl: url,\n\t\t\thtml: html,\n\t\t\taid: pageData.Aid,\n\t\t\tbvid: pageData.BVid,\n\t\t\tcid: page.Cid,\n\t\t\tpage: p,\n\t\t}\n\t\t// \"part\":\"\" or \"part\":\"Untitled\"\n\t\tif page.Part == \"Untitled\" || len(pageData.VideoData.Pages) == 1 {\n\t\t\toptions.subtitle = \"\"\n\t\t} else {\n\t\t\toptions.subtitle = page.Part\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle normal video playlist\n\tif len(pageData.Sections) == 0 {\n\t\t// https://www.bilibili.com/video/av20827366/?p=* each video in playlist has different p=?\n\t\treturn multiPageDownload(url, html, extractOption, pageData)\n\t}\n\t// handle another kind of playlist\n\t// https://www.bilibili.com/video/av*** each video in playlist has different av/bv id\n\treturn multiEpisodeDownload(url, html, extractOption, pageData)\n}\n\n// handle multi episode download\nfunc multiEpisodeDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.Sections[0].Episodes))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.Sections[0].Episodes {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl: url,\n\t\t\thtml: html,\n\t\t\taid: u.Aid,\n\t\t\tbvid: u.BVid,\n\t\t\tcid: u.Cid,\n\t\t\tsubtitle: fmt.Sprintf(\"%s P%d\", u.Title, index+1),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\n// handle multi page download\nfunc multiPageDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.VideoData.Pages))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.VideoData.Pages {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl: url,\n\t\t\thtml: html,\n\t\t\taid: pageData.Aid,\n\t\t\tbvid: pageData.BVid,\n\t\t\tcid: u.Cid,\n\t\t\tsubtitle: u.Part,\n\t\t\tpage: u.Page,\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\ntype extractor struct{}\n\n// New returns a bilibili extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\tif strings.Contains(url, \"bangumi\") {\n\t\t// handle bangumi\n\t\treturn extractBangumi(url, html, option)\n\t} else if strings.Contains(url, \"festival\") {\n\t\treturn extractFestival(url, html, option)\n\t} else {\n\t\t// handle normal video\n\t\treturn extractNormalVideo(url, html, option)\n\t}\n}\n\n// bilibiliDownload is the download function for a single URL\nfunc bilibiliDownload(options bilibiliOptions, extractOption extractors.Options) *extractors.Data {\n\tvar (\n\t\terr error\n\t\thtml string\n\t)\n\tif options.html != \"\" {\n\t\t// reuse html string, but this can't be reused in case of playlist\n\t\thtml = options.html\n\t} else {\n\t\thtml, err = request.Get(options.url, referer, nil)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t}\n\n\t// Get \"accept_quality\" and \"accept_description\"\n\t// \"accept_description\":[\"超高清 8K\",\"超清 4K\",\"高清 1080P+\",\"高清 1080P\",\"高清 720P\",\"清晰 480P\",\"流畅 360P\"],\n\t// \"accept_quality\":[127，120,112,80,48,32,16],\n\tapi, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tjsonString, err := request.Get(api, referer, nil)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\n\tvar data dash\n\terr = json.Unmarshal([]byte(jsonString), &data)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tvar dashData dashInfo\n\tif data.Data.Description == nil {\n\t\tdashData = data.Result\n\t} else {\n\t\tdashData = data.Data\n\t}\n\n\tvar audioPart *extractors.Part\n\tif dashData.Streams.Audio != nil {\n\t\t// Get audio part\n\t\tvar audioID int\n\t\taudios := map[int]string{}\n\t\tbandwidth := 0\n\t\tfor _, stream := range dashData.Streams.Audio {\n\t\t\tif stream.Bandwidth > bandwidth {\n\t\t\t\taudioID = stream.ID\n\t\t\t\tbandwidth = stream.Bandwidth\n\t\t\t}\n\t\t\taudios[stream.ID] = stream.BaseURL\n\t\t}\n\t\ts, err := request.Size(audios[audioID], referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\taudioPart = &extractors.Part{\n\t\t\tURL: audios[audioID],\n\t\t\tSize: s,\n\t\t\tExt: \"m4a\",\n\t\t}\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(dashData.Quality))\n\tfor _, stream := range dashData.Streams.Video {\n\t\ts, err := request.Size(stream.BaseURL, referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\tparts := make([]*extractors.Part, 0, 2)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL: stream.BaseURL,\n\t\t\tSize: s,\n\t\t\tExt: getExtFromMimeType(stream.MimeType),\n\t\t})\n\t\tif audioPart != nil {\n\t\t\tparts = append(parts, audioPart)\n\t\t}\n\t\tvar size int64\n\t\tfor _, part := range parts {\n\t\t\tsize += part.Size\n\t\t}\n\t\tid := fmt.Sprintf(\"%d-%d\", stream.ID, stream.Codecid)\n\t\tstreams[id] = &extractors.Stream{\n\t\t\tParts: parts,\n\t\t\tSize: size,\n\t\t\tQuality: fmt.Sprintf(\"%s %s\", qualityString[stream.ID], stream.Codecs),\n\t\t}\n\t\tif audioPart != nil {\n\t\t\tstreams[id].NeedMux = true\n\t\t}\n\t}\n\n\tfor _, durl := range dashData.DURLs {\n\t\tvar ext string\n\t\tswitch dashData.DURLFormat {\n\t\tcase \"flv\", \"flv480\":\n\t\t\text = \"flv\"\n\t\tcase \"mp4\", \"hdmp4\": // nolint\n\t\t\text = \"mp4\"\n\t\t}\n\n\t\tparts := make([]*extractors.Part, 0, 1)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL: durl.URL,\n\t\t\tSize: durl.Size,\n\t\t\tExt: ext,\n\t\t})\n\n\t\tstreams[strconv.Itoa(dashData.CurQuality)] = &extractors.Stream{\n\t\t\tParts: parts,\n\t\t\tSize: durl.Size,\n\t\t\tQuality: qualityString[dashData.CurQuality],\n\t\t}\n\t}\n\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\ttitle := parser.Title(doc)\n\tif options.subtitle != \"\" {\n\t\tpageString := \"\"\n\t\tif options.page > 0 {\n\t\t\tpageString = fmt.Sprintf(\"P%d \", options.page)\n\t\t}\n\t\tif extractOption.EpisodeTitleOnly {\n\t\t\ttitle = fmt.Sprintf(\"%s%s\", pageString, options.subtitle)\n\t\t} else {\n\t\t\ttitle = fmt.Sprintf(\"%s %s%s\", title, pageString, options.subtitle)\n\t\t}\n\t}\n\n\treturn &extractors.Data{\n\t\tSite: \"哔哩哔哩 bilibili.com\",\n\t\tTitle: title,\n\t\tType: extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tCaptions: map[string]*extractors.CaptionPart{\n\t\t\t\"danmaku\": {\n\t\t\t\tPart: extractors.Part{\n\t\t\t\t\tURL: fmt.Sprintf(\"https://comment.bilibili.com/%d.xml\", options.cid),\n\t\t\t\t\tExt: \"xml\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"subtitle\": getSubTitleCaptionPart(options.aid, options.cid),\n\t\t},\n\t\tURL: options.url,\n\t}\n}\n\nfunc getExtFromMimeType(mimeType string) string {\n\texts := strings.Split(mimeType, \"/\")\n\tif len(exts) == 2 {\n\t\treturn exts[1]\n\t}\n\treturn \"mp4\"\n}\n\nfunc getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart {\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"http://api.bilibili.com/x/player/wbi/v2?aid=%d&cid=%d\", aid, cid), referer, nil,\n\t)\n\tif err != nil {\n\t\treturn nil\n\t}\n\tstu := bilibiliWebInterface{}\n\terr = json.Unmarshal([]byte(jsonString), &stu)\n\tif err != nil || len(stu.Data.SubtitleInfo.SubtitleList) == 0 {\n\t\treturn nil\n\t}\n\treturn &extractors.CaptionPart{\n\t\tPart: extractors.Part{\n\t\t\tURL: fmt.Sprintf(\"https:%s\", stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl),\n\t\t\tExt: \"srt\",\n\t\t},\n\t\tTransform: subtitleTransform,\n\t}\n}\n\nfunc subtitleTransform(body []byte) ([]byte, error) {\n\tbytes := \"\"\n\tcaptionData := bilibiliSubtitleFormat{}\n\terr := json.Unmarshal(body, &captionData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor i := 0; i < len(captionData.Body); i++ {\n\t\tbytes += fmt.Sprintf(\"%d\\n%s --> %s\\n%s\\n\\n\",\n\t\t\ti,\n\t\t\ttime.Unix(0, int64(captionData.Body[i].From*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\ttime.Unix(0, int64(captionData.Body[i].To*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\tcaptionData.Body[i].Content,\n\t\t)\n\t}\n\treturn []byte(bytes), nil\n}\n" }, { "path": "extractors/bilibili/bilibili_test.go", "content": "package bilibili\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestBilibili(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/video/av20203945/\",\n\t\t\t\tTitle: \"【2018拜年祭单品】相遇day by day\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/video/av41301960\",\n\t\t\t\tTitle: \"【英雄联盟】2019赛季CG 《觉醒》\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/bangumi/play/ep167000\",\n\t\t\t\tTitle: \"狐妖小红娘 第70话 苏苏智商上线\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/bangumi/play/ss5050\",\n\t\t\t\tTitle: \"一人之下：第1话 异人刀兵起，道炁携阴阳\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/video/av16907446/\",\n\t\t\t\tTitle: \"\\\"不要相信歌词，他们为了押韵什么都干得出来\\\"\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"8k test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/video/BV1qM4y1w716\",\n\t\t\t\tTitle: \"【8K演示片】B站首发！你的设备还顶得住吗？\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"b23 test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://b23.tv/Fc9i7QF\",\n\t\t\t\tTitle: \"【十年榜】2000-2009年最强华语金曲TOP100 P1 100爱转角-罗志祥\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"festival test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt\",\n\t\t\t\tTitle: \"洛天依十周年官方演唱会\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// for playlist, we don't check the data\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist: true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "extractors/bilibili/types.go", "content": "package bilibili\n\n// {\"code\":0,\"message\":\"0\",\"ttl\":1,\"data\":{\"token\":\"aaa\"}}\n// {\"code\":-101,\"message\":\"账号未登录\",\"ttl\":1}\ntype tokenData struct {\n\tToken string `json:\"token\"`\n}\n\ntype token struct {\n\tCode int `json:\"code\"`\n\tMessage string `json:\"message\"`\n\tData tokenData `json:\"data\"`\n}\n\ntype Interaction struct {\n\tInteraction bool `json:\"interaction\"`\n}\n\ntype EpVideoInfo struct {\n\tAid int `json:\"aid\"`\n\tBvid string `json:\"bvid\"`\n\tCid int `json:\"cid\"`\n\tDeliveryBusinessFragmentVideo bool `json:\"delivery_business_fragment_video\"`\n\tDeliveryFragmentVideo bool `json:\"delivery_fragment_video\"`\n\tEpID int `json:\"ep_id\"`\n\tEpStatus int `json:\"ep_status\"`\n\tInteraction Interaction `json:\"interaction\"`\n\tLongTitle string `json:\"long_title\"`\n\tTitle string `json:\"title\"`\n}\n\ntype bangumiData struct {\n\tEpInfo EpVideoInfo `json:\"epInfo\"`\n\tEpList []EpVideoInfo `json:\"epList\"`\n}\n\ntype videoPagesData struct {\n\tCid int `json:\"cid\"`\n\tPart string `json:\"part\"`\n\tPage int `json:\"page\"`\n}\n\ntype multiPageVideoData struct {\n\tTitle string `json:\"title\"`\n\tPages []videoPagesData `json:\"pages\"`\n}\n\ntype episode struct {\n\tAid int `json:\"aid\"`\n\tCid int `json:\"cid\"`\n\tTitle string `json:\"title\"`\n\tBVid string `json:\"bvid\"`\n}\n\ntype multiEpisodeData struct {\n\tSeasionid int `json:\"season_id\"`\n\tEpisodes []episode `json:\"episodes\"`\n}\n\ntype multiPage struct {\n\tAid int `json:\"aid\"`\n\tBVid string `json:\"bvid\"`\n\tSections []multiEpisodeData `json:\"sections\"`\n\tVideoData multiPageVideoData `json:\"videoData\"`\n}\n\ntype dashStream struct {\n\tID int `json:\"id\"`\n\tBaseURL string `json:\"baseUrl\"`\n\tBandwidth int `json:\"bandwidth\"`\n\tMimeType string `json:\"mimeType\"`\n\tCodecid int `json:\"codecid\"`\n\tCodecs string `json:\"codecs\"`\n}\n\ntype dashStreams struct {\n\tVideo []dashStream `json:\"video\"`\n\tAudio []dashStream `json:\"audio\"`\n}\n\ntype dashInfo struct {\n\tCurQuality int `json:\"quality\"`\n\tDescription []string `json:\"accept_description\"`\n\tQuality []int `json:\"accept_quality\"`\n\tStreams dashStreams `json:\"dash\"`\n\tDURLFormat string `json:\"format\"`\n\tDURLs []dURL `json:\"durl\"`\n}\n\ntype dURL struct {\n\tURL string `json:\"url\"`\n\tSize int64 `json:\"size\"`\n}\n\ntype dash struct {\n\tCode int `json:\"code\"`\n\tMessage string `json:\"message\"`\n\tData dashInfo `json:\"data\"`\n\tResult dashInfo `json:\"result\"`\n}\n\nvar qualityString = map[int]string{\n\t127: \"超高清 8K\",\n\t120: \"超清 4K\",\n\t116: \"高清 1080P60\",\n\t74: \"高清 720P60\",\n\t112: \"高清 1080P+\",\n\t80: \"高清 1080P\",\n\t64: \"高清 720P\",\n\t48: \"高清 720P\",\n\t32: \"清晰 480P\",\n\t16: \"流畅 360P\",\n\t15: \"流畅 360P\",\n}\n\ntype subtitleData struct {\n\tFrom float32 `json:\"from\"`\n\tTo float32 `json:\"to\"`\n\tLocation int `json:\"location\"`\n\tContent string `json:\"content\"`\n}\n\ntype bilibiliSubtitleFormat struct {\n\tFontSize float32 `json:\"font_size\"`\n\tFontColor string `json:\"font_color\"`\n\tBackgroundAlpha float32 `json:\"background_alpha\"`\n\tBackgroundColor string `json:\"background_color\"`\n\tStroke string `json:\"Stroke\"`\n\tBody []subtitleData `json:\"body\"`\n}\n\ntype subtitleProperty struct {\n\tID int64 `json:\"id\"`\n\tLan string `json:\"lan\"`\n\tLanDoc string `json:\"lan_doc\"`\n\tSubtitleUrl string `json:\"subtitle_url\"`\n}\n\ntype subtitleInfo struct {\n\tAllowSubmit bool `json:\"allow_submit\"`\n\tSubtitleList []subtitleProperty `json:\"subtitles\"`\n}\n\ntype bilibiliWebInterfaceData struct {\n\tBvid string `json:\"bvid\"`\n\tSubtitleInfo subtitleInfo `json:\"subtitle\"`\n}\n\ntype bilibiliWebInterface struct {\n\tCode int `json:\"code\"`\n\tData bilibiliWebInterfaceData `json:\"data\"`\n}\n\ntype festival struct {\n\tVideoSections []struct {\n\t\tId int64 `json:\"id\"`\n\t\tTitle string `json:\"title\"`\n\t\tType int `json:\"type\"`\n\t} `json:\"videoSections\"`\n\tEpisodes []episode `json:\"episodes\"`\n\tVideoInfo struct {\n\t\tAid int `json:\"aid\"`\n\t\tBVid string `json:\"bvid\"`\n\t\tCid int `json:\"cid\"`\n\t\tTitle string `json:\"title\"`\n\t\tDesc string `json:\"desc\"`\n\t\tPages []struct {\n\t\t\tCid int `json:\"cid\"`\n\t\t\tDuration int `json:\"duration\"`\n\t\t\tPage int `json:\"page\"`\n\t\t\tPart string `json:\"part\"`\n\t\t\tDimension struct {\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tRotate int `json:\"rotate\"`\n\t\t\t} `json:\"dimension\"`\n\t\t} `json:\"pages\"`\n\t} `json:\"videoInfo\"`\n}\n" }, { "path": "extractors/bitchute/bitchute.go", "content": "package bitchute\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"bitchute\", New())\n}\n\ntype extractor struct{}\n\n// New returns a bitchute extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) {\n\tregVideoID := regexp.MustCompile(`/video/([^?/]+)`)\n\tmatchVideoID := regVideoID.FindStringSubmatch(u)\n\tif len(matchVideoID) < 2 {\n\t\treturn nil, errors.New(\"Invalid video URL: Missing video ID parameter\")\n\t}\n\tembedURL := fmt.Sprintf(\"https://www.bitchute.com/api/beta9/embed/?videoID=%s\", matchVideoID[1])\n\n\tres, err := request.Request(http.MethodGet, embedURL, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// There is also an API that provides meta data\n\t// POST https://api.bitchute.com/api/beta9/video {\"video_id\": }\n\thtml := string(b)\n\tregMediaURL := regexp.MustCompile(`media_url\\s*=\\s*['|\"](https:\\/\\/[^.]+\\.bitchute\\.com\\/.*\\.mp4)`)\n\tmatchMediaURL := regMediaURL.FindStringSubmatch(html)\n\tif len(matchMediaURL) < 2 {\n\t\treturn nil, errors.New(\"Could not extract media URL from page\")\n\t}\n\tmediaURL := matchMediaURL[1]\n\n\tregVideoName := regexp.MustCompile(`(?m)video_name\\s*=\\s*[\"|']\\\\?\"?(.*)[\"|'];?$`)\n\tmatchVideoName := regVideoName.FindStringSubmatch(html)\n\tif len(matchVideoName) < 2 {\n\t\treturn nil, errors.New(\"Could not extract media name from page\")\n\t}\n\tvideoName := strings.ReplaceAll(matchVideoName[1], `\\\"`, \"\")\n\n\tstreams := make(map[string]*extractors.Stream, 1)\n\tsize, err := request.Size(mediaURL, u)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tstreams[\"Default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{\n\t\t\t{\n\t\t\t\tURL: mediaURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt: \"mp4\",\n\t\t\t},\n\t\t},\n\t\tSize: size,\n\t\tQuality: \"Default\",\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Bitchute bitchute.com\",\n\t\t\tTitle: videoName,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: u,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/bitchute/bitchute_test.go", "content": "package bitchute\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bitchute.com/video/C17naZ6WlWPo\",\n\t\t\t\tTitle: \"Everybody Dance Now\",\n\t\t\t\tSize: 1794720,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.bitchute.com/video/HFgoUz6HrvQd\",\n\t\t\t\tTitle: \"Bear Level 1\",\n\t\t\t\tSize: 971698,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/douyin/douyin.go", "content": "package douyin\n\nimport (\n\t\"crypto/rand\"\n\t_ \"embed\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"douyin\", e)\n\textractors.Register(\"iesdouyin\", e)\n}\n\n//go:embed sign.js\nvar script string\n\ntype extractor struct{}\n\n// New returns a douyin extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tif strings.Contains(url, \"v.douyin.com\") {\n\t\treq, err := http.NewRequest(\"GET\", url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tc := http.Client{\n\t\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\t\treturn http.ErrUseLastResponse\n\t\t\t},\n\t\t}\n\t\tresp, err := c.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\turl = resp.Header.Get(\"location\")\n\t}\n\n\titemIds := utils.MatchOneOf(url, `/video/(\\d+)`)\n\tif len(itemIds) == 0 {\n\t\treturn nil, errors.New(\"unable to get video ID\")\n\t}\n\tif itemIds == nil || len(itemIds) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\titemId := itemIds[len(itemIds)-1]\n\n\t// dynamic generate cookie\n\tcookie, err := createCookie()\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tapi := \"https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=\" + itemId\n\t// parse api query params string\n\tquery, err := netURL.Parse(api)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\t// define request headers and sign agent\n\theaders := map[string]string{}\n\theaders[\"Cookie\"] = cookie\n\theaders[\"Referer\"] = \"https://www.douyin.com/\"\n\theaders[\"User-Agent\"] = \"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66\"\n\n\t// init JavaScripts runtime\n\tvm := goja.New()\n\t// load sign scripts\n\t_, _ = vm.RunString(script)\n\t// sign\n\tsign, err := vm.RunString(fmt.Sprintf(\"sign('%s', '%s')\", query.RawQuery, headers[\"User-Agent\"]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tapi = fmt.Sprintf(\"%s&X-Bogus=%s\", api, sign)\n\n\tjsonData, err := request.Get(api, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar douyin douyinData\n\tif err = json.Unmarshal([]byte(jsonData), &douyin); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := make([]*extractors.Part, 0)\n\tvar douyinType extractors.DataType\n\tvar totalSize int64\n\t// AwemeType: 0:video 68:image\n\tif douyin.AwemeDetail.AwemeType == 68 {\n\t\tdouyinType = extractors.DataTypeImage\n\t\tfor _, img := range douyin.AwemeDetail.Images {\n\t\t\trealURL := img.URLList[len(img.URLList)-1]\n\t\t\tsize, err := request.Size(realURL, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turlData = append(urlData, &extractors.Part{\n\t\t\t\tURL: realURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt: ext,\n\t\t\t})\n\t\t}\n\t} else {\n\t\tdouyinType = extractors.DataTypeVideo\n\t\trealURL := douyin.AwemeDetail.Video.PlayAddr.URLList[0]\n\t\ttotalSize, err = request.Size(realURL, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData = append(urlData, &extractors.Part{\n\t\t\tURL: realURL,\n\t\t\tSize: totalSize,\n\t\t\tExt: \"mp4\",\n\t\t})\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urlData,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"抖音 douyin.com\",\n\t\t\tTitle: douyin.AwemeDetail.Desc,\n\t\t\tType: douyinType,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n\nfunc createCookie() (string, error) {\n\tv1, err := msToken(107)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv2, err := ttwid()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv3 := \"324fb4ea4a89c0c05827e18a1ed9cf9bf8a17f7705fcc793fec935b637867e2a5a9b8168c885554d029919117a18ba69\"\n\tv4 := \"eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWNsaWVudC1jc3IiOiItLS0tLUJFR0lOIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLVxyXG5NSUlCRFRDQnRRSUJBREFuTVFzd0NRWURWUVFHRXdKRFRqRVlNQllHQTFVRUF3d1BZbVJmZEdsamEyVjBYMmQxXHJcbllYSmtNRmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVKUDZzbjNLRlFBNUROSEcyK2F4bXAwNG5cclxud1hBSTZDU1IyZW1sVUE5QTZ4aGQzbVlPUlI4NVRLZ2tXd1FJSmp3Nyszdnc0Z2NNRG5iOTRoS3MvSjFJc3FBc1xyXG5NQ29HQ1NxR1NJYjNEUUVKRGpFZE1Cc3dHUVlEVlIwUkJCSXdFSUlPZDNkM0xtUnZkWGxwYmk1amIyMHdDZ1lJXHJcbktvWkl6ajBFQXdJRFJ3QXdSQUlnVmJkWTI0c0RYS0c0S2h3WlBmOHpxVDRBU0ROamNUb2FFRi9MQnd2QS8xSUNcclxuSURiVmZCUk1PQVB5cWJkcytld1QwSDZqdDg1czZZTVNVZEo5Z2dmOWlmeTBcclxuLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tXHJcbiJ9\"\n\tcookie := fmt.Sprintf(\"msToken=%s;ttwid=%s;odin_tt=%s;bd_ticket_guard_client_data=%s;\", v1, v2, v3, v4)\n\treturn cookie, nil\n}\n\nfunc msToken(length int) (string, error) {\n\tconst characters = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\"\n\trandomBytes := make([]byte, length)\n\tif _, err := rand.Read(randomBytes); err != nil {\n\t\treturn \"\", err\n\t}\n\ttoken := make([]byte, length)\n\tfor i, b := range randomBytes {\n\t\ttoken[i] = characters[int(b)%len(characters)]\n\t}\n\treturn string(token), nil\n}\n\nfunc ttwid() (string, error) {\n\tbody := map[string]interface{}{\n\t\t\"aid\": 1768,\n\t\t\"union\": true,\n\t\t\"needFid\": false,\n\t\t\"region\": \"cn\",\n\t\t\"cbUrlProtocol\": \"https\",\n\t\t\"service\": \"www.ixigua.com\",\n\t\t\"migrate_info\": map[string]string{\"ticket\": \"\", \"source\": \"node\"},\n\t}\n\tbytes, err := json.Marshal(body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tpayload := strings.NewReader(string(bytes))\n\tresp, err := request.Request(http.MethodPost, \"https://ttwid.bytedance.com/ttwid/union/register/\", payload, nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close() // nolint\n\tcookie := resp.Header.Get(\"Set-Cookie\")\n\tre := regexp.MustCompile(`ttwid=([^;]+)`)\n\tif match := re.FindStringSubmatch(cookie); match != nil {\n\t\treturn match[1], nil\n\t}\n\treturn \"\", errors.New(\"douyin ttwid request failed\")\n}\n" }, { "path": "extractors/douyin/douyin_test.go", "content": "package douyin\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home\",\n\t\t\t\tTitle: \"是爱情，让父子相认#陈翔六点半 #关于爱情\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://v.douyin.com/LvCYKvV\",\n\t\t\t\tTitle: \"黑发限定#开春必备\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/douyin/sign.js", "content": "var window = null;\n\nfunction _0x5cd844(e) {\n var b = {\n exports: {}\n };\n return e(b, b.exports), b.exports\n}\n\njsvmp = function(e, b, a) {\n function f(e, b, a) {\n return (f = function() {\n if (\"undefined\" == typeof Reflect || !Reflect.construct || Reflect.construct.sham) return !1;\n if (\"function\" == typeof Proxy) return !0;\n try {\n return Date.prototype.toString.call(Reflect.construct(Date, [], function() {})), !0\n } catch (e) {\n return !1\n }\n }() ? Reflect.construct : function(e, b, a) {\n var f = [null];\n f.push.apply(f, b);\n var c = new(Function.bind.apply(e, f));\n return a && function(e, b) {\n (Object.setPrototypeOf || function(e, b) {\n return e.__proto__ = b, e\n })(e, b)\n }(c, a.prototype), c\n }).apply(null, arguments)\n }\n\n function c(e) {\n return function(e) {\n if (Array.isArray(e)) {\n for (var b = 0, a = new Array(e.length); b < e.length; b++) a[b] = e[b];\n return a\n }\n }(e) || function(e) {\n if (Symbol.iterator in Object(e) || \"[object Arguments]\" === Object.prototype.toString.call(e)) return Array.from(e)\n }(e) || function() {\n throw new TypeError(\"Invalid attempt to spread non-iterable instance\")\n }()\n }\n for (var r = [], t = 0, d = [], i = 0, n = function(e, b) {\n var a = e[b++],\n f = e[b],\n c = parseInt(\"\" + a + f, 16);\n if (c >> 7 == 0) return [1, c];\n if (c >> 6 == 2) {\n var r = parseInt(\"\" + e[++b] + e[++b], 16);\n return c &= 63, [2, r = (c <<= 8) + r]\n }\n if (c >> 6 == 3) {\n var t = parseInt(\"\" + e[++b] + e[++b], 16),\n d = parseInt(\"\" + e[++b] + e[++b], 16);\n return c &= 63, [3, d = (c <<= 16) + (t <<= 8) + d]\n }\n }, s = function(e, b) {\n var a = parseInt(\"\" + e[b] + e[b + 1], 16);\n return a > 127 ? -256 + a : a\n }, o = function(e, b) {\n var a = parseInt(\"\" + e[b] + e[b + 1] + e[b + 2] + e[b + 3], 16);\n return a > 32767 ? -65536 + a : a\n }, l = function(e, b) {\n var a = parseInt(\"\" + e[b] + e[b + 1] + e[b + 2] + e[b + 3] + e[b + 4] + e[b + 5] + e[b + 6] + e[b + 7], 16);\n return a > 2147483647 ? 0 + a : a\n }, _ = function(e, b) {\n return parseInt(\"\" + e[b] + e[b + 1], 16)\n }, x = function(e, b) {\n return parseInt(\"\" + e[b] + e[b + 1] + e[b + 2] + e[b + 3], 16)\n }, u = u || this || window, h = (e.length, 0), p = \"\", y = h; y < h + 16; y++) {\n var v = \"\" + e[y++] + e[y];\n v = parseInt(v, 16), p += String.fromCharCode(v)\n }\n if (\"HNOJ@?RC\" != p) throw new Error(\"error magic number \" + p);\n parseInt(\"\" + e[h += 16] + e[h + 1], 16), h += 8, t = 0;\n for (var g = 0; g < 4; g++) {\n var w = h + 2 * g,\n A = parseInt(\"\" + e[w++] + e[w], 16);\n t += (3 & A) << 2 * g\n }\n h += 16;\n var C = parseInt(\"\" + e[h += 8] + e[h + 1] + e[h + 2] + e[h + 3] + e[h + 4] + e[h + 5] + e[h + 6] + e[h + 7], 16),\n m = C,\n S = h += 8,\n z = x(e, h += C);\n z[1], h += 4, r = {\n p: [],\n q: []\n };\n for (var B = 0; B < z; B++) {\n for (var R = n(e, h), q = h += 2 * R[0], I = r.p.length, k = 0; k < R[1]; k++) {\n var j = n(e, q);\n r.p.push(j[1]), q += 2 * j[0]\n }\n h = q, r.q.push([I, r.p.length])\n }\n var O = {\n 5: 1,\n 6: 1,\n 70: 1,\n 22: 1,\n 23: 1,\n 37: 1,\n 73: 1\n },\n U = {\n 72: 1\n },\n D = {\n 74: 1\n },\n N = {\n 11: 1,\n 12: 1,\n 24: 1,\n 26: 1,\n 27: 1,\n 31: 1\n },\n J = {\n 10: 1\n },\n L = {\n 2: 1,\n 29: 1,\n 30: 1,\n 20: 1\n },\n T = [],\n E = [];\n\n function M(e, b, a) {\n for (var f = b; f < b + a;) {\n var c = _(e, f);\n T[f] = c, f += 2, U[c] ? (E[f] = s(e, f), f += 2) : O[c] ? (E[f] = o(e, f), f += 4) : D[c] ? (E[f] = l(e, f), f += 8) : N[c] ? (E[f] = _(e, f), f += 2) : J[c] ? (E[f] = x(e, f), f += 4) : L[c] && (E[f] = x(e, f), f += 4)\n }\n }\n return F(e, S, m / 2, [], b, a);\n\n function P(e, b, a, n, h, p, y, v) {\n null == p && (p = this);\n var g, w, A, C, m = [],\n S = 0;\n y && (w = y);\n var z, B, R = b,\n q = R + 2 * a;\n if (!v)\n for (; R < q;) {\n var I = parseInt(\"\" + e[R] + e[R + 1], 16);\n R += 2;\n var j = 3 & (z = 13 * I % 241);\n if (z >>= 2, j < 1)\n if (j = 3 & z, z >>= 2, j < 1) {\n if ((j = z) < 1) return [1, m[S--]];\n j < 5 ? (w = m[S--], m[S] = m[S] * w) : j < 7 ? (w = m[S--], m[S] = m[S] != w) : j < 14 ? (A = m[S--], C = m[S--], (j = m[S--]).x === P ? j.y >= 1 ? m[++S] = F(e, j.c, j.l, A, j.z, C, null, 1) : (m[++S] = F(e, j.c, j.l, A, j.z, C, null, 0), j.y++) : m[++S] = j.apply(C, A)) : j < 16 && (B = o(e, R), (g = function b() {\n var a = arguments;\n return b.y > 0 || b.y++, F(e, b.c, b.l, a, b.z, this, null, 0)\n }).c = R + 4, g.l = B - 2, g.x = P, g.y = 0, g.z = h, m[S] = g, R += 2 * B - 2)\n } else if (j < 2)(j = z) > 8 ? (w = m[S--], m[S] = typeof w) : j > 4 ? m[S -= 1] = m[S][m[S + 1]] : j > 2 && (A = m[S--], (j = m[S]).x === P ? j.y >= 1 ? m[S] = F(e, j.c, j.l, [A], j.z, C, null, 1) : (m[S] = F(e, j.c, j.l, [A], j.z, C, null, 0), j.y++) : m[S] = j(A));\n else if (j < 3) {\n if ((j = z) < 9) {\n for (w = m[S--], B = x(e, R), j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n R += 4, m[S--][j] = w\n } else if (j < 13) throw m[S--]\n } else(j = z) < 1 ? m[++S] = null : j < 3 ? (w = m[S--], m[S] = m[S] >= w) : j < 12 && (m[++S] = void 0);\n else if (j < 2)\n if (j = 3 & z, z >>= 2, j < 1)\n if ((j = z) < 5) {\n B = o(e, R);\n try {\n if (d[i][2] = 1, 1 == (w = P(e, R + 4, B - 3, [], h, p, null, 0))[0]) return w\n } catch (b) {\n if (d[i] && d[i][1] && 1 == (w = P(e, d[i][1][0], d[i][1][1], [], h, p, b, 0))[0]) return w\n } finally {\n if (d[i] && d[i][0] && 1 == (w = P(e, d[i][0][0], d[i][0][1], [], h, p, null, 0))[0]) return w;\n d[i] = 0, i--\n }\n R += 2 * B - 2\n } else j < 7 ? (B = _(e, R), R += 2, m[S -= B] = 0 === B ? new m[S] : f(m[S], c(m.slice(S + 1, S + B + 1)))) : j < 9 && (w = m[S--], m[S] = m[S] & w);\n else if (j < 2)\n if ((j = z) > 12) m[++S] = s(e, R), R += 2;\n else if (j > 10) w = m[S--], m[S] = m[S] << w;\n else if (j > 8) {\n for (B = x(e, R), j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n R += 4, m[S] = m[S][j]\n } else j > 6 && (A = m[S--], w = delete m[S--][A]);\n else if (j < 3)(j = z) < 2 ? m[++S] = w : j < 11 ? (w = m[S -= 2][m[S + 1]] = m[S + 2], S--) : j < 13 && (w = m[S], m[++S] = w);\n else if ((j = z) > 12) m[++S] = p;\n else if (j > 5) w = m[S--], m[S] = m[S] !== w;\n else if (j > 3) w = m[S--], m[S] = m[S] / w;\n else if (j > 1) {\n if ((B = o(e, R)) < 0) {\n v = 1, M(e, b, 2 * a), R += 2 * B - 2;\n break\n }\n R += 2 * B - 2\n } else j > -1 && (m[S] = !m[S]);\n else if (j < 3)\n if (j = 3 & z, z >>= 2, j < 1)(j = z) > 13 ? (m[++S] = o(e, R), R += 4) : j > 11 ? (w = m[S--], m[S] = m[S] >> w) : j > 9 ? (B = _(e, R), R += 2, w = m[S--], h[B] = w) : j > 7 ? (B = x(e, R), R += 4, A = S + 1, m[S -= B - 1] = B ? m.slice(S, A) : []) : j > 0 && (w = m[S--], m[S] = m[S] > w);\n else if (j < 2)(j = z) > 12 ? (w = m[S - 1], A = m[S], m[++S] = w, m[++S] = A) : j > 3 ? (w = m[S--], m[S] = m[S] == w) : j > 1 ? (w = m[S--], m[S] = m[S] + w) : j > -1 && (m[++S] = u);\n else if (j < 3) {\n if ((j = z) > 13) m[++S] = !1;\n else if (j > 6) w = m[S--], m[S] = m[S] instanceof w;\n else if (j > 4) w = m[S--], m[S] = m[S] % w;\n else if (j > 2)\n if (m[S--]) R += 4;\n else {\n if ((B = o(e, R)) < 0) {\n v = 1, M(e, b, 2 * a), R += 2 * B - 2;\n break\n }\n R += 2 * B - 2\n }\n else if (j > 0) {\n for (B = x(e, R), w = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) w += String.fromCharCode(t ^ r.p[k]);\n m[++S] = w, R += 4\n }\n } else(j = z) > 7 ? (w = m[S--], m[S] = m[S] | w) : j > 5 ? (B = _(e, R), R += 2, m[++S] = h[\"$\" + B]) : j > 3 && (B = o(e, R), d[i][0] && !d[i][2] ? d[i][1] = [R + 4, B - 3] : d[i++] = [0, [R + 4, B - 3], 0], R += 2 * B - 2);\n else if (j = 3 & z, z >>= 2, j > 2)(j = z) > 13 ? (m[++S] = l(e, R), R += 8) : j > 11 ? (w = m[S--], m[S] = m[S] >>> w) : j > 9 ? m[++S] = !0 : j > 7 ? (B = _(e, R), R += 2, m[S] = m[S][B]) : j > 0 && (w = m[S--], m[S] = m[S] < w);\n else if (j > 1)(j = z) > 10 ? (B = o(e, R), d[++i] = [\n [R + 4, B - 3], 0, 0\n ], R += 2 * B - 2) : j > 8 ? (w = m[S--], m[S] = m[S] ^ w) : j > 6 && (w = m[S--]);\n else if (j > 0) {\n if ((j = z) > 7) w = m[S--], m[S] = m[S] in w;\n else if (j > 5) m[S] = ++m[S];\n else if (j > 3) B = _(e, R), R += 2, w = h[B], m[++S] = w;\n else if (j > 1) {\n var O = 0,\n U = m[S].length,\n D = m[S];\n m[++S] = function() {\n var e = O < U;\n if (e) {\n var b = D[O++];\n m[++S] = b\n }\n m[++S] = e\n }\n }\n } else if ((j = z) > 13) w = m[S], m[S] = m[S - 1], m[S - 1] = w;\n else if (j > 4) w = m[S--], m[S] = m[S] === w;\n else if (j > 2) w = m[S--], m[S] = m[S] - w;\n else if (j > 0) {\n for (B = x(e, R), j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n j = +j, R += 4, m[++S] = j\n }\n }\n if (v)\n for (; R < q;)\n if (I = T[R], R += 2, j = 3 & (z = 13 * I % 241), z >>= 2, j > 2)\n if (j = 3 & z, z >>= 2, j > 2)(j = z) < 2 ? (w = m[S--], m[S] = m[S] < w) : j < 9 ? (B = E[R], R += 2, m[S] = m[S][B]) : j < 11 ? m[++S] = !0 : j < 13 ? (w = m[S--], m[S] = m[S] >>> w) : j < 15 && (m[++S] = E[R], R += 8);\n else if (j > 1)(j = z) < 6 || (j < 8 ? w = m[S--] : j < 10 ? (w = m[S--], m[S] = m[S] ^ w) : j < 12 && (B = E[R], d[++i] = [\n [R + 4, B - 3], 0, 0\n ], R += 2 * B - 2));\n else if (j > 0)(j = z) > 7 ? (w = m[S--], m[S] = m[S] in w) : j > 5 ? m[S] = ++m[S] : j > 3 ? (B = E[R], R += 2, w = h[B], m[++S] = w) : j > 1 && (O = 0, U = m[S].length, D = m[S], m[++S] = function() {\n var e = O < U;\n if (e) {\n var b = D[O++];\n m[++S] = b\n }\n m[++S] = e\n });\n else if ((j = z) < 2) {\n for (B = E[R], j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n j = +j, R += 4, m[++S] = j\n } else j < 4 ? (w = m[S--], m[S] = m[S] - w) : j < 6 ? (w = m[S--], m[S] = m[S] === w) : j < 15 && (w = m[S], m[S] = m[S - 1], m[S - 1] = w);\n else if (j > 1)\n if (j = 3 & z, z >>= 2, j < 1)(j = z) > 13 ? (m[++S] = E[R], R += 4) : j > 11 ? (w = m[S--], m[S] = m[S] >> w) : j > 9 ? (B = E[R], R += 2, w = m[S--], h[B] = w) : j > 7 ? (B = E[R], R += 4, A = S + 1, m[S -= B - 1] = B ? m.slice(S, A) : []) : j > 0 && (w = m[S--], m[S] = m[S] > w);\n else if (j < 2)(j = z) < 1 ? m[++S] = u : j < 3 ? (w = m[S--], m[S] = m[S] + w) : j < 5 ? (w = m[S--], m[S] = m[S] == w) : j < 14 && (w = m[S - 1], A = m[S], m[++S] = w, m[++S] = A);\n else if (j < 3) {\n if ((j = z) > 13) m[++S] = !1;\n else if (j > 6) w = m[S--], m[S] = m[S] instanceof w;\n else if (j > 4) w = m[S--], m[S] = m[S] % w;\n else if (j > 2) m[S--] ? R += 4 : R += 2 * (B = E[R]) - 2;\n else if (j > 0) {\n for (B = E[R], w = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) w += String.fromCharCode(t ^ r.p[k]);\n m[++S] = w, R += 4\n }\n } else(j = z) > 7 ? (w = m[S--], m[S] = m[S] | w) : j > 5 ? (B = E[R], R += 2, m[++S] = h[\"$\" + B]) : j > 3 && (B = E[R], d[i][0] && !d[i][2] ? d[i][1] = [R + 4, B - 3] : d[i++] = [0, [R + 4, B - 3], 0], R += 2 * B - 2);\n else if (j > 0)\n if (j = 3 & z, z >>= 2, j < 1) {\n if ((j = z) > 9);\n else if (j > 7) w = m[S--], m[S] = m[S] & w;\n else if (j > 5) B = E[R], R += 2, m[S -= B] = 0 === B ? new m[S] : f(m[S], c(m.slice(S + 1, S + B + 1)));\n else if (j > 3) {\n B = E[R];\n try {\n if (d[i][2] = 1, 1 == (w = P(e, R + 4, B - 3, [], h, p, null, 0))[0]) return w\n } catch (b) {\n if (d[i] && d[i][1] && 1 == (w = P(e, d[i][1][0], d[i][1][1], [], h, p, b, 0))[0]) return w\n } finally {\n if (d[i] && d[i][0] && 1 == (w = P(e, d[i][0][0], d[i][0][1], [], h, p, null, 0))[0]) return w;\n d[i] = 0, i--\n }\n R += 2 * B - 2\n }\n } else if (j < 2)\n if ((j = z) < 8) A = m[S--], w = delete m[S--][A];\n else if (j < 10) {\n for (B = E[R], j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n R += 4, m[S] = m[S][j]\n } else j < 12 ? (w = m[S--], m[S] = m[S] << w) : j < 14 && (m[++S] = E[R], R += 2);\n else j < 3 ? (j = z) < 2 ? m[++S] = w : j < 11 ? (w = m[S -= 2][m[S + 1]] = m[S + 2], S--) : j < 13 && (w = m[S], m[++S] = w) : (j = z) > 12 ? m[++S] = p : j > 5 ? (w = m[S--], m[S] = m[S] !== w) : j > 3 ? (w = m[S--], m[S] = m[S] / w) : j > 1 ? R += 2 * (B = E[R]) - 2 : j > -1 && (m[S] = !m[S]);\n else if (j = 3 & z, z >>= 2, j < 1) {\n if ((j = z) < 1) return [1, m[S--]];\n j < 5 ? (w = m[S--], m[S] = m[S] * w) : j < 7 ? (w = m[S--], m[S] = m[S] != w) : j < 14 ? (A = m[S--], C = m[S--], (j = m[S--]).x === P ? j.y >= 1 ? m[++S] = F(e, j.c, j.l, A, j.z, C, null, 1) : (m[++S] = F(e, j.c, j.l, A, j.z, C, null, 0), j.y++) : m[++S] = j.apply(C, A)) : j < 16 && (B = E[R], (g = function b() {\n var a = arguments;\n return b.y > 0 || b.y++, F(e, b.c, b.l, a, b.z, this, null, 0)\n }).c = R + 4, g.l = B - 2, g.x = P, g.y = 0, g.z = h, m[S] = g, R += 2 * B - 2)\n } else if (j < 2)(j = z) > 8 ? (w = m[S--], m[S] = typeof w) : j > 4 ? m[S -= 1] = m[S][m[S + 1]] : j > 2 && (A = m[S--], (j = m[S]).x === P ? j.y >= 1 ? m[S] = F(e, j.c, j.l, [A], j.z, C, null, 1) : (m[S] = F(e, j.c, j.l, [A], j.z, C, null, 0), j.y++) : m[S] = j(A));\n else if (j < 3) {\n if ((j = z) < 9) {\n for (w = m[S--], B = E[R], j = \"\", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]);\n R += 4, m[S--][j] = w\n } else if (j < 13) throw m[S--]\n } else(j = z) < 1 ? m[++S] = null : j < 3 ? (w = m[S--], m[S] = m[S] >= w) : j < 12 && (m[++S] = void 0);\n return [0, null]\n }\n\n function F(e, b, a, f, c, r, t, d) {\n null == r && (r = this), c && !c.d && (c.d = 0, c.$0 = c, c[1] = {});\n var i, n, s = {},\n o = s.d = c ? c.d + 1 : 0;\n for (s[\"$\" + o] = s, n = 0; n < o; n++) s[i = \"$\" + n] = c[i];\n for (n = 0, o = s.length = f.length; n < o; n++) s[n] = f[n];\n return d && !T[b] && M(e, b, 2 * a), T[b] ? P(e, b, a, 0, s, r, null, 1)[1] : P(e, b, a, 0, s, r, null, 0)[1]\n }\n};\nvar _0x397dc7 = \"undefined\" != typeof globalThis ? globalThis : void 0 !== window ? window : \"undefined\" != typeof global ? global : \"undefined\" != typeof self ? self : {},\n _0x124d1a = _0x5cd844(function(_0x770f81) {\n ! function() {\n var _0x250d36 = \"input is invalid type\",\n _0x4cfaee = !1,\n _0x1702f9 = {},\n _0x5ccbb3 = !_0x4cfaee && \"object\" == typeof self,\n _0x54d876 = !_0x1702f9.JS_MD5_NO_NODE_JS && \"object\" == typeof process && process.versions && process.versions.node,\n _0x185caf;\n _0x54d876 ? _0x1702f9 = _0x397dc7 : _0x5ccbb3 && (_0x1702f9 = self);\n var _0x17dcbf = !_0x1702f9.JS_MD5_NO_COMMON_JS && _0x770f81.exports,\n _0x554fed = !1,\n _0x2de28f = !_0x1702f9.JS_MD5_NO_ARRAY_BUFFER && \"undefined\" != typeof ArrayBuffer,\n _0x3a9a1b = [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n _0x465562 = [128, 32768, 8388608, -2147483648],\n _0x20b37e = [0, 8, 16, 24],\n _0x323604 = [\"hex\", \"array\", \"digest\", \"buffer\", \"arrayBuffer\", \"base64\"],\n _0x2c185e = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\", \"K\", \"L\", \"M\", \"N\", \"O\", \"P\", \"Q\", \"R\", \"S\", \"T\", \"U\", \"V\", \"W\", \"X\", \"Y\", \"Z\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"m\", \"n\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\", \"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"+\", \"/\"],\n _0x4b59e0 = [];\n if (_0x2de28f) {\n var _0x395837 = new ArrayBuffer(68);\n _0x185caf = new Uint8Array(_0x395837), _0x4b59e0 = new Uint32Array(_0x395837)\n }!_0x1702f9.JS_MD5_NO_NODE_JS && Array.isArray || (Array.isArray = function(e) {\n return \"[object Array]\" === Object.prototype.toString.call(e)\n }), _0x2de28f && (_0x1702f9.JS_MD5_NO_ARRAY_BUFFER_IS_VIEW || !ArrayBuffer.isView) && (ArrayBuffer.isView = function(e) {\n return \"object\" == typeof e && e.buffer && e.buffer.constructor === ArrayBuffer\n });\n var _0x4e9930 = function(e) {\n return function(b) {\n return new _0x5887c8(!0).update(b)[e]()\n }\n },\n _0x38ba77 = function() {\n var e = _0x4e9930(\"hex\");\n _0x54d876 && (e = _0x474989(e)), e.create = function() {\n return new _0x5887c8\n }, e.update = function(b) {\n return e.create().update(b)\n };\n for (var b = 0; b < _0x323604.length; ++b) {\n var a = _0x323604[b];\n e[a] = _0x4e9930(a)\n }\n return e\n },\n _0x474989 = function(_0x57eeaa) {\n var _0x114910, _0x226465 = eval(\"require('crypto');\"),\n _0x1f6ae0 = eval(\"require('buffer')['Buffer'];\");\n return function(e) {\n if (\"string\" == typeof e) return _0x226465.createHash(\"md5\").update(e, \"utf8\").digest(\"hex\");\n if (null == e) throw _0x250d36;\n return e.constructor === ArrayBuffer && (e = new Uint8Array(e)), Array.isArray(e) || ArrayBuffer.isView(e) || e.constructor === _0x1f6ae0 ? _0x226465.createHash(\"md5\").update(new _0x1f6ae0.from(e)).digest(\"hex\") : _0x57eeaa(e)\n }\n };\n\n function _0x5887c8(e) {\n if (e) _0x4b59e0[0] = _0x4b59e0[16] = _0x4b59e0[1] = _0x4b59e0[2] = _0x4b59e0[3] = _0x4b59e0[4] = _0x4b59e0[5] = _0x4b59e0[6] = _0x4b59e0[7] = _0x4b59e0[8] = _0x4b59e0[9] = _0x4b59e0[10] = _0x4b59e0[11] = _0x4b59e0[12] = _0x4b59e0[13] = _0x4b59e0[14] = _0x4b59e0[15] = 0, this.blocks = _0x4b59e0, this.buffer8 = _0x185caf;\n else if (_0x2de28f) {\n var b = new ArrayBuffer(68);\n this.buffer8 = new Uint8Array(b), this.blocks = new Uint32Array(b)\n } else this.blocks = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];\n this.h0 = this.h1 = this.h2 = this.h3 = this.start = this.bytes = this.hBytes = 0, this.finalized = this.hashed = !1, this.first = !0\n }\n _0x5887c8.prototype.update = function(e) {\n if (!this.finalized) {\n var b, a = typeof e;\n if (\"string\" !== a) {\n if (\"object\" !== a || null === e) throw _0x250d36;\n if (_0x2de28f && e.constructor === ArrayBuffer) e = new Uint8Array(e);\n else if (!(Array.isArray(e) || _0x2de28f && ArrayBuffer.isView(e))) throw _0x250d36;\n b = !0\n }\n for (var f, c, r = 0, t = e.length, d = this.blocks, i = this.buffer8; r < t;) {\n if (this.hashed && (this.hashed = !1, d[0] = d[16], d[16] = d[1] = d[2] = d[3] = d[4] = d[5] = d[6] = d[7] = d[8] = d[9] = d[10] = d[11] = d[12] = d[13] = d[14] = d[15] = 0), b)\n if (_0x2de28f)\n for (c = this.start; r < t && c < 64; ++r) i[c++] = e[r];\n else\n for (c = this.start; r < t && c < 64; ++r) d[c >> 2] |= e[r] << _0x20b37e[3 & c++];\n else if (_0x2de28f)\n for (c = this.start; r < t && c < 64; ++r)(f = e.charCodeAt(r)) < 128 ? i[c++] = f : f < 2048 ? (i[c++] = 192 | f >> 6, i[c++] = 128 | 63 & f) : f < 55296 || f >= 57344 ? (i[c++] = 224 | f >> 12, i[c++] = 128 | f >> 6 & 63, i[c++] = 128 | 63 & f) : (f = 65536 + ((1023 & f) << 10 | 1023 & e.charCodeAt(++r)), i[c++] = 240 | f >> 18, i[c++] = 128 | f >> 12 & 63, i[c++] = 128 | f >> 6 & 63, i[c++] = 128 | 63 & f);\n else\n for (c = this.start; r < t && c < 64; ++r)(f = e.charCodeAt(r)) < 128 ? d[c >> 2] |= f << _0x20b37e[3 & c++] : f < 2048 ? (d[c >> 2] |= (192 | f >> 6) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]) : f < 55296 || f >= 57344 ? (d[c >> 2] |= (224 | f >> 12) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 6 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]) : (f = 65536 + ((1023 & f) << 10 | 1023 & e.charCodeAt(++r)), d[c >> 2] |= (240 | f >> 18) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 12 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 6 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]);\n this.lastByteIndex = c, this.bytes += c - this.start, c >= 64 ? (this.start = c - 64, this.hash(), this.hashed = !0) : this.start = c\n }\n return this.bytes > 4294967295 && (this.hBytes += this.bytes / 4294967296 << 0, this.bytes = this.bytes % 4294967296), this\n }\n }, _0x5887c8.prototype.finalize = function() {\n if (!this.finalized) {\n this.finalized = !0;\n var e = this.blocks,\n b = this.lastByteIndex;\n e[b >> 2] |= _0x465562[3 & b], b >= 56 && (this.hashed || this.hash(), e[0] = e[16], e[16] = e[1] = e[2] = e[3] = e[4] = e[5] = e[6] = e[7] = e[8] = e[9] = e[10] = e[11] = e[12] = e[13] = e[14] = e[15] = 0), e[14] = this.bytes << 3, e[15] = this.hBytes << 3 | this.bytes >>> 29, this.hash()\n }\n }, _0x5887c8.prototype.hash = function() {\n var e, b, a, f, c, r, t = this.blocks;\n this.first ? b = ((b = ((e = ((e = t[0] - 680876937) << 7 | e >>> 25) - 271733879 << 0) ^ (a = ((a = (-271733879 ^ (f = ((f = (-1732584194 ^ 2004318071 & e) + t[1] - 117830708) << 12 | f >>> 20) + e << 0) & (-271733879 ^ e)) + t[2] - 1126478375) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[3] - 1316259209) << 22 | b >>> 10) + a << 0 : (e = this.h0, b = this.h1, a = this.h2, b = ((b += ((e = ((e += ((f = this.h3) ^ b & (a ^ f)) + t[0] - 680876936) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[1] - 389564586) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[2] + 606105819) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[3] - 1044525330) << 22 | b >>> 10) + a << 0), b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[4] - 176418897) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[5] + 1200080426) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[6] - 1473231341) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[7] - 45705983) << 22 | b >>> 10) + a << 0, b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[8] + 1770035416) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[9] - 1958414417) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[10] - 42063) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[11] - 1990404162) << 22 | b >>> 10) + a << 0, b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[12] + 1804603682) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[13] - 40341101) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[14] - 1502002290) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[15] + 1236535329) << 22 | b >>> 10) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[1] - 165796510) << 5 | e >>> 27) + b << 0) ^ b)) + t[6] - 1069501632) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[11] + 643717713) << 14 | a >>> 18) + f << 0) ^ f)) + t[0] - 373897302) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[5] - 701558691) << 5 | e >>> 27) + b << 0) ^ b)) + t[10] + 38016083) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[15] - 660478335) << 14 | a >>> 18) + f << 0) ^ f)) + t[4] - 405537848) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[9] + 568446438) << 5 | e >>> 27) + b << 0) ^ b)) + t[14] - 1019803690) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[3] - 187363961) << 14 | a >>> 18) + f << 0) ^ f)) + t[8] + 1163531501) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[13] - 1444681467) << 5 | e >>> 27) + b << 0) ^ b)) + t[2] - 51403784) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[7] + 1735328473) << 14 | a >>> 18) + f << 0) ^ f)) + t[12] - 1926607734) << 20 | b >>> 12) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[5] - 378558) << 4 | e >>> 28) + b << 0)) + t[8] - 2022574463) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[11] + 1839030562) << 16 | a >>> 16) + f << 0)) + t[14] - 35309556) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[1] - 1530992060) << 4 | e >>> 28) + b << 0)) + t[4] + 1272893353) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[7] - 155497632) << 16 | a >>> 16) + f << 0)) + t[10] - 1094730640) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[13] + 681279174) << 4 | e >>> 28) + b << 0)) + t[0] - 358537222) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[3] - 722521979) << 16 | a >>> 16) + f << 0)) + t[6] + 76029189) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[9] - 640364487) << 4 | e >>> 28) + b << 0)) + t[12] - 421815835) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[15] + 530742520) << 16 | a >>> 16) + f << 0)) + t[2] - 995338651) << 23 | b >>> 9) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[0] - 198630844) << 6 | e >>> 26) + b << 0) | ~a)) + t[7] + 1126891415) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[14] - 1416354905) << 15 | a >>> 17) + f << 0) | ~e)) + t[5] - 57434055) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[12] + 1700485571) << 6 | e >>> 26) + b << 0) | ~a)) + t[3] - 1894986606) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[10] - 1051523) << 15 | a >>> 17) + f << 0) | ~e)) + t[1] - 2054922799) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[8] + 1873313359) << 6 | e >>> 26) + b << 0) | ~a)) + t[15] - 30611744) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[6] - 1560198380) << 15 | a >>> 17) + f << 0) | ~e)) + t[13] + 1309151649) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[4] - 145523070) << 6 | e >>> 26) + b << 0) | ~a)) + t[11] - 1120210379) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[2] + 718787259) << 15 | a >>> 17) + f << 0) | ~e)) + t[9] - 343485551) << 21 | b >>> 11) + a << 0, this.first ? (this.h0 = e + 1732584193 << 0, this.h1 = b - 271733879 << 0, this.h2 = a - 1732584194 << 0, this.h3 = f + 271733878 << 0, this.first = !1) : (this.h0 = this.h0 + e << 0, this.h1 = this.h1 + b << 0, this.h2 = this.h2 + a << 0, this.h3 = this.h3 + f << 0)\n }, _0x5887c8.prototype.hex = function() {\n this.finalize();\n var e = this.h0,\n b = this.h1,\n a = this.h2,\n f = this.h3;\n return _0x3a9a1b[e >> 4 & 15] + _0x3a9a1b[15 & e] + _0x3a9a1b[e >> 12 & 15] + _0x3a9a1b[e >> 8 & 15] + _0x3a9a1b[e >> 20 & 15] + _0x3a9a1b[e >> 16 & 15] + _0x3a9a1b[e >> 28 & 15] + _0x3a9a1b[e >> 24 & 15] + _0x3a9a1b[b >> 4 & 15] + _0x3a9a1b[15 & b] + _0x3a9a1b[b >> 12 & 15] + _0x3a9a1b[b >> 8 & 15] + _0x3a9a1b[b >> 20 & 15] + _0x3a9a1b[b >> 16 & 15] + _0x3a9a1b[b >> 28 & 15] + _0x3a9a1b[b >> 24 & 15] + _0x3a9a1b[a >> 4 & 15] + _0x3a9a1b[15 & a] + _0x3a9a1b[a >> 12 & 15] + _0x3a9a1b[a >> 8 & 15] + _0x3a9a1b[a >> 20 & 15] + _0x3a9a1b[a >> 16 & 15] + _0x3a9a1b[a >> 28 & 15] + _0x3a9a1b[a >> 24 & 15] + _0x3a9a1b[f >> 4 & 15] + _0x3a9a1b[15 & f] + _0x3a9a1b[f >> 12 & 15] + _0x3a9a1b[f >> 8 & 15] + _0x3a9a1b[f >> 20 & 15] + _0x3a9a1b[f >> 16 & 15] + _0x3a9a1b[f >> 28 & 15] + _0x3a9a1b[f >> 24 & 15]\n }, _0x5887c8.prototype.toString = _0x5887c8.prototype.hex, _0x5887c8.prototype.digest = function() {\n this.finalize();\n var e = this.h0,\n b = this.h1,\n a = this.h2,\n f = this.h3;\n return [255 & e, e >> 8 & 255, e >> 16 & 255, e >> 24 & 255, 255 & b, b >> 8 & 255, b >> 16 & 255, b >> 24 & 255, 255 & a, a >> 8 & 255, a >> 16 & 255, a >> 24 & 255, 255 & f, f >> 8 & 255, f >> 16 & 255, f >> 24 & 255]\n }, _0x5887c8.prototype.array = _0x5887c8.prototype.digest, _0x5887c8.prototype.arrayBuffer = function() {\n this.finalize();\n var e = new ArrayBuffer(16),\n b = new Uint32Array(e);\n return b[0] = this.h0, b[1] = this.h1, b[2] = this.h2, b[3] = this.h3, e\n }, _0x5887c8.prototype.buffer = _0x5887c8.prototype.arrayBuffer, _0x5887c8.prototype.base64 = function() {\n for (var e, b, a, f = \"\", c = this.array(), r = 0; r < 15;) e = c[r++], b = c[r++], a = c[r++], f += _0x2c185e[e >>> 2] + _0x2c185e[63 & (e << 4 | b >>> 4)] + _0x2c185e[63 & (b << 2 | a >>> 6)] + _0x2c185e[63 & a];\n return f + (_0x2c185e[(e = c[r]) >>> 2] + _0x2c185e[e << 4 & 63] + \"==\")\n };\n var _0x4dd781 = _0x38ba77();\n _0x17dcbf ? _0x770f81.exports = _0x4dd781 : (_0x1702f9.md5 = _0x4dd781, _0x554fed && (void 0)(function() {\n return _0x4dd781\n }))\n }()\n });\n\nfunction _0x178cef(e) {\n return jsvmp(\"484e4f4a403f52430038001eab0015840e8ee21a00000000000000621b000200001d000146000306000e271f001b000200021d00010500121b001b000b021b000b04041d0001071b000b0500000003000126207575757575757575757575757575757575757575757575757575757575757575\", [, , void 0 !== _0x124d1a ? _0x124d1a : void 0, _0x178cef, e])\n}\nfor (var _0xb55f3e = {\n boe: !1,\n aid: 0,\n dfp: !1,\n sdi: !1,\n enablePathList: [],\n _enablePathListRegex: [],\n urlRewriteRules: [],\n _urlRewriteRules: [],\n initialized: !1,\n enableTrack: !1,\n track: {\n unitTime: 0,\n unitAmount: 0,\n fre: 0\n },\n triggerUnload: !1,\n region: \"\",\n regionConf: {},\n umode: 0,\n v: !1,\n perf: !1,\n xxbg: !0\n}, _0x3eaf64 = {\n debug: function(e, b) {\n let a = !1;\n a = !1\n }\n}, _0x233455 = [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\"], _0x2e9f6d = [], _0x511f86 = [], _0x3d35de = 0; _0x3d35de < 256; _0x3d35de++) _0x2e9f6d[_0x3d35de] = _0x233455[_0x3d35de >> 4 & 15] + _0x233455[15 & _0x3d35de], _0x3d35de < 16 && (_0x3d35de < 10 ? _0x511f86[48 + _0x3d35de] = _0x3d35de : _0x511f86[87 + _0x3d35de] = _0x3d35de);\nvar _0x2ce54d = function(e) {\n for (var b = e.length, a = \"\", f = 0; f < b;) a += _0x2e9f6d[e[f++]];\n return a\n },\n _0x5960a2 = function(e) {\n for (var b = e.length >> 1, a = b << 1, f = new Uint8Array(b), c = 0, r = 0; r < a;) f[c++] = _0x511f86[e.charCodeAt(r++)] << 4 | _0x511f86[e.charCodeAt(r++)];\n return f\n },\n _0x4e46b6 = {\n encode: _0x2ce54d,\n decode: _0x5960a2\n };\n\nfunction sign(e, b) {\n return jsvmp(\"484e4f4a403f5243001f240fbf2031ccf317480300000000000007181b0002012e1d00921b000b171b000b02402217000a1c1b000b1726402217000c1c1b000b170200004017002646000306000e271f001b000200021d00920500121b001b000b031b000b17041d0092071b000b041e012f17000d1b000b05260a0000101c1b000b06260a0000101c1b001b000b071e01301d00931b001b000b081e00081d00941b0048021d00951b001b000b1b1d00961b0048401d009e1b001b000b031b000b16041d009f1b001b000b09221e0131241b000b031b000b09221e0131241b000b1e0a000110040a0001101d00d51b001b000b09221e0131241b000b031b000b09221e0131241b000b180a000110040a0001101d00d71b001b000b0a1e00101d00d91b001b000b0b261b000b1a1b000b190a0002101d00db1b001b000b0c261b000b221b000b210a0002101d00dc1b001b000b0d261b000b230200200a0002101d00dd1b001b000b09221e0131241b000b031b000b24040a0001101d00df1b001b000b0e1a00221e00de240a0000104903e82b1d00e31b001b000b0f260a0000101d00e41b001b000b1d1d00e71b001b000b1a4901002b1d00e81b001b000b1a4901002c1d00ea1b001b000b191d00f21b001b000b1f480e191d00f81b001b000b1f480f191d00f91b001b000b20480e191d00fb1b001b000b20480f191d00fe1b001b000b25480e191d01001b001b000b25480f191d01011b001b000b264818344900ff2f1d01031b001b000b264810344900ff2f1d01321b001b000b264808344900ff2f1d01331b001b000b264800344900ff2f1d01341b001b000b274818344900ff2f1d01351b001b000b274810344900ff2f1d01361b001b000b274808344900ff2f1d01371b001b000b274800344900ff2f1d01381b001b000b281b000b29311b000b2a311b000b2b311b000b2c311b000b2d311b000b2e311b000b2f311b000b30311b000b31311b000b32311b000b33311b000b34311b000b35311b000b36311b000b37311b000b38311b000b39311d01391b004900ff1d013a1b001b000b10261b000b281b000b2a1b000b2c1b000b2e1b000b301b000b321b000b341b000b361b000b381b000b3a1b000b291b000b2b1b000b2d1b000b2f1b000b311b000b331b000b351b000b371b000b390a0013101d013b1b001b000b0c261b000b111b000b3b041b000b3c0a0002101d013c1b001b000b12261b000b1c1b000b3b1b000b3d0a0003101d013d1b001b000b13261b000b3e0200240a0002101d013e1b000b3f0000013f000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b726152670f487c717976706733447a7d777c644e08577c70667e767d6712487c7179767067335d72657a7472677c614e057960777c7e10487c7179767067335b7a60677c616a4e07637f66747a7d60084c637b727d677c7e0b70727f7f437b727d677c7e0b4c4c7d7a747b677e726176055266777a7c1850727d65726041767d7776617a7d74507c7d67766b6721570964767177617a657661137476675c647d43617c637661676a5d727e7660097f727d74667274766006707b617c7e760761667d677a7e7607707c7d7d767067144c4c64767177617a6576614c7665727f66726776134c4c60767f767d7a667e4c7665727f667267761b4c4c64767177617a6576614c6070617a63674c75667d70677a7c7d174c4c64767177617a6576614c6070617a63674c75667d70154c4c64767177617a6576614c6070617a63674c757d134c4c756b77617a6576614c7665727f66726776124c4c77617a6576614c667d64617263637677154c4c64767177617a6576614c667d64617263637677114c4c77617a6576614c7665727f66726776144c4c60767f767d7a667e4c667d64617263637677144c4c756b77617a6576614c667d64617263637677094c60767f767d7a667e0c70727f7f40767f767d7a667e164c40767f767d7a667e4c5a57564c4176707c6177766108777c70667e767d670478766a60057e7267707b06417674566b630a4f3748723e694e77704c067072707b764c04607c7e7608707675407b72616308507675407b72616305767c72637a16767c44767151617c64607661577a60637267707b76610f717a7d775c717976706752606a7d700e7a60565c44767151617c646076610120047c63767d0467766067097a7d707c747d7a677c077c7d7661617c6104707c77761242465c47524c564b5056565756574c5641410e607660607a7c7d40677c61727476076076675a67767e10607c7e7658766a5b766176516a6776770a61767e7c65765a67767e097a7d77766b767757510c437c7a7d6776615665767d670e5e40437c7a7d6776615665767d670d706176726776567f767e767d670670727d65726009677c5772677246415f076176637f727076034f603901740a7d72677a6576707c777614487c717976706733437f66747a7d526161726a4e4a4d7b676763602c294f3c4f3c3b48233e2a4e68223f206e3b4f3d48233e2a4e68223f206e3a68206e6f48723e75233e2a4e68223f276e3b2948723e75233e2a4e68223f276e3a68246e3a0127087f7c7072677a7c7d047b61767504757a7f76107b676763293c3c7f7c70727f7b7c606708637f7267757c617e02222102222007647a7d777c646002222703647a7d02222607727d77617c7a77022225057f7a7d666b022224067a637b7c7d7602222b047a63727702222a047a637c77022123037e7270022122097e72707a7d677c607b0c7e72704c637c64766163703a0470617c60036b22220570617a7c6005756b7a7c6004637a787602212102212002212702212602212502212402212b08757a6176757c6b3c067c637661723c05337c63613c05337c63673c07707b617c7e763c0867617a77767d673c047e607a7602212a0220230665767d777c6106547c7c747f760e4c637261727e40647a67707b5c7d0a777a61767067407a747d0a707c7d607a6067767d670660647a67707b03777c7e07637b727d677c7e047b7c7c7840525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e3d03727a77017d01750161096067726167477a7e7601670972717a7f7a677a76600a677a7e766067727e6322137b72617764726176507c7d70666161767d706a0c7776657a70765e767e7c616a087f727d74667274760a6176607c7f66677a7c7d0f7265727a7f4176607c7f66677a7c7d0960706176767d477c630a60706176767d5f767567107776657a7076437a6b767f4172677a7c0a63617c77667067406671077172676776616a016309677c66707b5a7d757c08677a7e76697c7d760a677a7e766067727e6321077463665a7d757c0b7960557c7d67605f7a60670b637f66747a7d605f7a60670a677a7e766067727e63200a76657661507c7c787a760767674c60707a77017e0b606a7d67726b5661617c610c7d72677a65765f767d74677b056167705a43097563457661607a7c7d0b4c4c657661607a7c7d4c4c08707f7a767d675a770a677a7e766067727e63270b766b67767d77557a767f77046366607b03727f7f04677b767d097172607625274c707b0c75617c7e507b7261507c7776067125274c2023022022087172607625274c23022021087172607625274c22022020087172607625274c2102202702202602202507747667477a7e760220240b777c7e5d7c6745727f7a77096066716067617a7d740863617c677c707c7f02202b02202a01230e222323232323232322222323232302272302272207757c616176727f02272104717c776a096067617a7d747a756a02686e0b717c776a45727f216067610a717c776a4c7b72607b2e01350366617f02272005626676616a0a72607c7f774c607a747d096372677b7d727e762e0967674c6476717a772e063566667a772e0227270227260e4c716a6776774c6076704c777a770227250a27212a272a2524212a25097576457661607a7c7d0227240e4c232151274925647c232323232202272b02272a05607f7a7076022623074056505a5d555c037d7c6409677a7e766067727e6305757f7c7c610661727d777c7e0f7476674747447671507c7c787a7660056767647a770867674c6476717a770767674476715a770b67674c6476717a774c65210967674476717a7745210761667d7d7a7d7405757f66607b087e7c65765f7a60670660637f7a70760671765e7c657609707f7a70785f7a6067077176507f7a70780c78766a717c7261775f7a60670a717658766a717c7261770b7270677a657640677267760b647a7d777c6440677267760360477e05676172707808667d7a67477a7e76037270700a667d7a67527e7c667d670871767b72657a7c61077e6074476a637603645a5707727a775f7a60670b63617a6572706a5e7c777606706660677c7e067260607a747d0f4456514c5756455a50564c5a5d555c0479607c7d0a6176747a7c7d507c7d75096176637c616746617f04766b7a67094b3e5e403e404746510c4b3e5e403e43524a5f5c525720232323232323232323232323232323232323232323232323232323232323232320772722772b70772a2b75232371212327762a2b23232a2a2b7670752b272124760165066671707c7776067776707c777602262202262102262002262702262602262502262402262b02262a022523022522022521022520\", [, , void 0, void 0 !== _0x178cef ? _0x178cef : void 0, {\n boe: !1,\n aid: 0,\n dfp: !1,\n sdi: !1,\n enablePathList: [],\n _enablePathListRegex: [/\\/web\\/report/],\n urlRewriteRules: [],\n _urlRewriteRules: [],\n initialized: !1,\n enableTrack: !1,\n track: {\n unitTime: 0,\n unitAmount: 0,\n fre: 0\n },\n triggerUnload: !1,\n region: \"\",\n regionConf: {},\n umode: 0,\n v: !1,\n perf: !1,\n xxbg: !0\n }, () => 0, () => \"03v\", {\n ubcode: 0\n }, {\n bogusIndex: 0,\n msNewTokenList: [],\n moveList: [],\n clickList: [],\n keyboardList: [],\n activeState: [],\n aidList: [],\n envcode: 0,\n msToken: \"\",\n msStatus: 0,\n __ac_testid: \"\",\n ttwid: \"\",\n tt_webid: \"\",\n tt_webid_v2: \"\"\n }, void 0 !== _0x4e46b6 ? _0x4e46b6 : void 0, {\n userAgent: b\n }, (e, b) => {\n let a = new Uint8Array(3);\n return a[0] = e / 256, a[1] = e % 256, a[2] = b % 256, String.fromCharCode.apply(null, a)\n }, (e, b) => {\n let a, f = [],\n c = 0,\n r = \"\";\n for (let e = 0; e < 256; e++) f[e] = e;\n for (let b = 0; b < 256; b++) c = (c + f[b] + e.charCodeAt(b % e.length)) % 256, a = f[b], f[b] = f[c], f[c] = a;\n let t = 0;\n c = 0;\n for (let e = 0; e < b.length; e++) c = (c + f[t = (t + 1) % 256]) % 256, a = f[t], f[t] = f[c], f[c] = a, r += String.fromCharCode(b.charCodeAt(e) ^ f[(f[t] + f[c]) % 256]);\n return r\n }, (e, b) => jsvmp(\"484e4f4a403f524300281018f7b851f02d296e5b00000000000004a21b0002001d1d001e1b00131e00061a001d001f1b000b070200200200210d1b000b070200220200230d1b000b070200240200250d1b000b070200260200270d1b001b000b071b000b05191d00031b000200001d00281b0048001d00291b000b041e002a1b000b0b4803283b1700f11b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f480833301b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b08221e002d241b000b0a490fc02f4806340a000110281d00281b00220b091b000b08221e002d241b000b0a483f2f0a000110281d002816ff031b000b041e002a1b000b0b294800391700e01b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b041e002a1b000b0b3917001e1b000b04221e002b241b000b0b0a0001104900ff2f4808331600054800301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b041e002a1b000b0b3917001e1b000b08221e002d241b000b0a490fc02f4806340a0001101600071b000b06281d00281b00220b091b000b06281d00281b000b090000002e000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b72615267\", [, , , , e, b]), \"undefined\" != typeof Date ? Date : void 0, () => 0, (e, b, a, f, c, r, t, d, i, n, s, o, l, _, x, u, h, p, y) => {\n let v = new Uint8Array(19);\n return v[0] = e, v[1] = s, v[2] = b, v[3] = o, v[4] = a, v[5] = l, v[6] = f, v[7] = _, v[8] = c, v[9] = x, v[10] = r, v[11] = u, v[12] = t, v[13] = h, v[14] = d, v[15] = p, v[16] = i, v[17] = y, v[18] = n, String.fromCharCode.apply(null, v)\n }, e => String.fromCharCode(e), (e, b, a) => String.fromCharCode(e) + String.fromCharCode(b) + a, (e, b) => jsvmp(\"484e4f4a403f524300281018f7b851f02d296e5b00000000000004a21b0002001d1d001e1b00131e00061a001d001f1b000b070200200200210d1b000b070200220200230d1b000b070200240200250d1b000b070200260200270d1b001b000b071b000b05191d00031b000200001d00281b0048001d00291b000b041e002a1b000b0b4803283b1700f11b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f480833301b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b08221e002d241b000b0a490fc02f4806340a000110281d00281b00220b091b000b08221e002d241b000b0a483f2f0a000110281d002816ff031b000b041e002a1b000b0b294800391700e01b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b041e002a1b000b0b3917001e1b000b04221e002b241b000b0b0a0001104900ff2f4808331600054800301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b041e002a1b000b0b3917001e1b000b08221e002d241b000b0a490fc02f4806340a0001101600071b000b06281d00281b00220b091b000b06281d00281b000b090000002e000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b72615267\", [, , , , e, b]), , sign, e, void 0])\n}\n\nmodule.exports = {\n sign\n};\n" }, { "path": "extractors/douyin/types.go", "content": "package douyin\n\ntype douyinData struct {\n\tStatusCode int `json:\"status_code\"`\n\tAwemeDetail struct {\n\t\tAdmireAuth struct {\n\t\t\tAdmireButton int `json:\"admire_button\"`\n\t\t\tIsAdmire int `json:\"is_admire\"`\n\t\t\tIsShowAdmireButton int `json:\"is_show_admire_button\"`\n\t\t\tIsShowAdmireTab int `json:\"is_show_admire_tab\"`\n\t\t} `json:\"admire_auth\"`\n\t\tAnchors interface{} `json:\"anchors\"`\n\t\tAuthor struct {\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tCfList interface{} `json:\"cf_list\"`\n\t\t\tCloseFriendType int `json:\"close_friend_type\"`\n\t\t\tContactsStatus int `json:\"contacts_status\"`\n\t\t\tContrailList interface{} `json:\"contrail_list\"`\n\t\t\tCoverURL []struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_url\"`\n\t\t\tCreateTime int `json:\"create_time\"`\n\t\t\tCustomVerify string `json:\"custom_verify\"`\n\t\t\tDataLabelList interface{} `json:\"data_label_list\"`\n\t\t\tEndorsementInfoList interface{} `json:\"endorsement_info_list\"`\n\t\t\tEnterpriseVerifyReason string `json:\"enterprise_verify_reason\"`\n\t\t\tFavoritingCount int `json:\"favoriting_count\"`\n\t\t\tFollowStatus int `json:\"follow_status\"`\n\t\t\tFollowerCount int `json:\"follower_count\"`\n\t\t\tFollowerListSecondaryInformationStruct interface{} `json:\"follower_list_secondary_information_struct\"`\n\t\t\tFollowerStatus int `json:\"follower_status\"`\n\t\t\tFollowingCount int `json:\"following_count\"`\n\t\t\tImRoleIds interface{} `json:\"im_role_ids\"`\n\t\t\tIsAdFake bool `json:\"is_ad_fake\"`\n\t\t\tIsBlockedV2 bool `json:\"is_blocked_v2\"`\n\t\t\tIsBlockingV2 bool `json:\"is_blocking_v2\"`\n\t\t\tIsCf int `json:\"is_cf\"`\n\t\t\tMaxFollowerCount int `json:\"max_follower_count\"`\n\t\t\tNickname string `json:\"nickname\"`\n\t\t\tNotSeenItemIDList interface{} `json:\"not_seen_item_id_list\"`\n\t\t\tNotSeenItemIDListV2 interface{} `json:\"not_seen_item_id_list_v2\"`\n\t\t\tOfflineInfoList interface{} `json:\"offline_info_list\"`\n\t\t\tPersonalTagList interface{} `json:\"personal_tag_list\"`\n\t\t\tPreventDownload bool `json:\"prevent_download\"`\n\t\t\tRiskNoticeText string `json:\"risk_notice_text\"`\n\t\t\tSecUID string `json:\"sec_uid\"`\n\t\t\tSecret int `json:\"secret\"`\n\t\t\tShareInfo struct {\n\t\t\t\tShareDesc string `json:\"share_desc\"`\n\t\t\t\tShareDescInfo string `json:\"share_desc_info\"`\n\t\t\t\tShareQrcodeURL struct {\n\t\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth int `json:\"width\"`\n\t\t\t\t} `json:\"share_qrcode_url\"`\n\t\t\t\tShareTitle string `json:\"share_title\"`\n\t\t\t\tShareTitleMyself string `json:\"share_title_myself\"`\n\t\t\t\tShareTitleOther string `json:\"share_title_other\"`\n\t\t\t\tShareURL string `json:\"share_url\"`\n\t\t\t\tShareWeiboDesc string `json:\"share_weibo_desc\"`\n\t\t\t} `json:\"share_info\"`\n\t\t\tShortID string `json:\"short_id\"`\n\t\t\tSignature string `json:\"signature\"`\n\t\t\tSignatureExtra interface{} `json:\"signature_extra\"`\n\t\t\tSpecialPeopleLabels interface{} `json:\"special_people_labels\"`\n\t\t\tStatus int `json:\"status\"`\n\t\t\tTextExtra interface{} `json:\"text_extra\"`\n\t\t\tTotalFavorited int `json:\"total_favorited\"`\n\t\t\tUID string `json:\"uid\"`\n\t\t\tUniqueID string `json:\"unique_id\"`\n\t\t\tUserAge int `json:\"user_age\"`\n\t\t\tUserCanceled bool `json:\"user_canceled\"`\n\t\t\tUserPermissions interface{} `json:\"user_permissions\"`\n\t\t\tVerificationType int `json:\"verification_type\"`\n\t\t} `json:\"author\"`\n\t\tAuthorMaskTag int `json:\"author_mask_tag\"`\n\t\tAuthorUserID int64 `json:\"author_user_id\"`\n\t\tAwemeACL struct {\n\t\t\tDownloadMaskPanel struct {\n\t\t\t\tCode int `json:\"code\"`\n\t\t\t\tShowType int `json:\"show_type\"`\n\t\t\t} `json:\"download_mask_panel\"`\n\t\t} `json:\"aweme_acl\"`\n\t\tAwemeControl struct {\n\t\t\tCanComment bool `json:\"can_comment\"`\n\t\t\tCanForward bool `json:\"can_forward\"`\n\t\t\tCanShare bool `json:\"can_share\"`\n\t\t\tCanShowComment bool `json:\"can_show_comment\"`\n\t\t} `json:\"aweme_control\"`\n\t\tAwemeID string `json:\"aweme_id\"`\n\t\tAwemeType int `json:\"aweme_type\"`\n\t\tChallengePosition interface{} `json:\"challenge_position\"`\n\t\tChapterList interface{} `json:\"chapter_list\"`\n\t\tCollectStat int `json:\"collect_stat\"`\n\t\tCommentGid int64 `json:\"comment_gid\"`\n\t\tCommentList interface{} `json:\"comment_list\"`\n\t\tCommentPermissionInfo struct {\n\t\t\tCanComment bool `json:\"can_comment\"`\n\t\t\tCommentPermissionStatus int `json:\"comment_permission_status\"`\n\t\t\tItemDetailEntry bool `json:\"item_detail_entry\"`\n\t\t\tPressEntry bool `json:\"press_entry\"`\n\t\t\tToastGuide bool `json:\"toast_guide\"`\n\t\t} `json:\"comment_permission_info\"`\n\t\tCommerceConfigData interface{} `json:\"commerce_config_data\"`\n\t\tCommonBarInfo string `json:\"common_bar_info\"`\n\t\tComponentInfoV2 string `json:\"component_info_v2\"`\n\t\tCoverLabels interface{} `json:\"cover_labels\"`\n\t\tCreateTime int `json:\"create_time\"`\n\t\tDesc string `json:\"desc\"`\n\t\tDiggLottie struct {\n\t\t\tCanBomb int `json:\"can_bomb\"`\n\t\t\tLottieID string `json:\"lottie_id\"`\n\t\t} `json:\"digg_lottie\"`\n\t\tDisableRelationBar int `json:\"disable_relation_bar\"`\n\t\tDislikeDimensionList interface{} `json:\"dislike_dimension_list\"`\n\t\tDuetAggregateInMusicTab bool `json:\"duet_aggregate_in_music_tab\"`\n\t\tDuration int `json:\"duration\"`\n\t\tFeedCommentConfig struct {\n\t\t\tAuthorAuditStatus int `json:\"author_audit_status\"`\n\t\t\tInputConfigText string `json:\"input_config_text\"`\n\t\t} `json:\"feed_comment_config\"`\n\t\tGeofencing []interface{} `json:\"geofencing\"`\n\t\tGeofencingRegions interface{} `json:\"geofencing_regions\"`\n\t\tGroupID string `json:\"group_id\"`\n\t\tHybridLabel interface{} `json:\"hybrid_label\"`\n\t\tImageAlbumMusicInfo struct {\n\t\t\tBeginTime int `json:\"begin_time\"`\n\t\t\tEndTime int `json:\"end_time\"`\n\t\t\tVolume int `json:\"volume\"`\n\t\t} `json:\"image_album_music_info\"`\n\t\tImageInfos interface{} `json:\"image_infos\"`\n\t\tImageList interface{} `json:\"image_list\"`\n\t\tImages []struct {\n\t\t\tDownloadURLList []string `json:\"download_url_list\"`\n\t\t\tHeight int `json:\"height\"`\n\t\t\tMaskURLList interface{} `json:\"mask_url_list\"`\n\t\t\tURI string `json:\"uri\"`\n\t\t\tURLList []string `json:\"url_list\"`\n\t\t\tWidth int `json:\"width\"`\n\t\t} `json:\"images\"`\n\t\tImgBitrate []struct {\n\t\t\tImages []struct {\n\t\t\t\tDownloadURLList []string `json:\"download_url_list\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tMaskURLList interface{} `json:\"mask_url_list\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"images\"`\n\t\t\tName string `json:\"name\"`\n\t\t} `json:\"img_bitrate\"`\n\t\tImpressionData struct {\n\t\t\tGroupIDListA []int64 `json:\"group_id_list_a\"`\n\t\t\tGroupIDListB []int64 `json:\"group_id_list_b\"`\n\t\t\tSimilarIDListA interface{} `json:\"similar_id_list_a\"`\n\t\t\tSimilarIDListB interface{} `json:\"similar_id_list_b\"`\n\t\t} `json:\"impression_data\"`\n\t\tInteractionStickers interface{} `json:\"interaction_stickers\"`\n\t\tIsAds bool `json:\"is_ads\"`\n\t\tIsCollectsSelected int `json:\"is_collects_selected\"`\n\t\tIsDuetSing bool `json:\"is_duet_sing\"`\n\t\tIsImageBeat bool `json:\"is_image_beat\"`\n\t\tIsLifeItem bool `json:\"is_life_item\"`\n\t\tIsMultiContent int `json:\"is_multi_content\"`\n\t\tIsStory int `json:\"is_story\"`\n\t\tIsTop int `json:\"is_top\"`\n\t\tItemWarnNotification struct {\n\t\t\tContent string `json:\"content\"`\n\t\t\tShow bool `json:\"show\"`\n\t\t\tType int `json:\"type\"`\n\t\t} `json:\"item_warn_notification\"`\n\t\tLabelTopText interface{} `json:\"label_top_text\"`\n\t\tLongVideo interface{} `json:\"long_video\"`\n\t\tMusic struct {\n\t\t\tAlbum string `json:\"album\"`\n\t\t\tArtistUserInfos interface{} `json:\"artist_user_infos\"`\n\t\t\tArtists []interface{} `json:\"artists\"`\n\t\t\tAuditionDuration int `json:\"audition_duration\"`\n\t\t\tAuthor string `json:\"author\"`\n\t\t\tAuthorDeleted bool `json:\"author_deleted\"`\n\t\t\tAuthorPosition interface{} `json:\"author_position\"`\n\t\t\tAuthorStatus int `json:\"author_status\"`\n\t\t\tAvatarLarge struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"avatar_large\"`\n\t\t\tAvatarMedium struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"avatar_medium\"`\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tBindedChallengeID int `json:\"binded_challenge_id\"`\n\t\t\tCanBackgroundPlay bool `json:\"can_background_play\"`\n\t\t\tCollectStat int `json:\"collect_stat\"`\n\t\t\tCoverColorHsv struct {\n\t\t\t\tH int `json:\"h\"`\n\t\t\t\tS int `json:\"s\"`\n\t\t\t\tV int `json:\"v\"`\n\t\t\t} `json:\"cover_color_hsv\"`\n\t\t\tCoverHd struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_hd\"`\n\t\t\tCoverLarge struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_large\"`\n\t\t\tCoverMedium struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_medium\"`\n\t\t\tCoverThumb struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_thumb\"`\n\t\t\tDmvAutoShow bool `json:\"dmv_auto_show\"`\n\t\t\tDspStatus int `json:\"dsp_status\"`\n\t\t\tDuration int `json:\"duration\"`\n\t\t\tEndTime int `json:\"end_time\"`\n\t\t\tExternalSongInfo []interface{} `json:\"external_song_info\"`\n\t\t\tExtra string `json:\"extra\"`\n\t\t\tID int64 `json:\"id\"`\n\t\t\tIDStr string `json:\"id_str\"`\n\t\t\tIsAudioURLWithCookie bool `json:\"is_audio_url_with_cookie\"`\n\t\t\tIsCommerceMusic bool `json:\"is_commerce_music\"`\n\t\t\tIsDelVideo bool `json:\"is_del_video\"`\n\t\t\tIsMatchedMetadata bool `json:\"is_matched_metadata\"`\n\t\t\tIsOriginal bool `json:\"is_original\"`\n\t\t\tIsOriginalSound bool `json:\"is_original_sound\"`\n\t\t\tIsPgc bool `json:\"is_pgc\"`\n\t\t\tIsRestricted bool `json:\"is_restricted\"`\n\t\t\tIsVideoSelfSee bool `json:\"is_video_self_see\"`\n\t\t\tLunaInfo struct {\n\t\t\t\tHasCopyright bool `json:\"has_copyright\"`\n\t\t\t\tIsLunaUser bool `json:\"is_luna_user\"`\n\t\t\t} `json:\"luna_info\"`\n\t\t\tLyricShortPosition interface{} `json:\"lyric_short_position\"`\n\t\t\tMatchedPgcSound struct {\n\t\t\t\tAuthor string `json:\"author\"`\n\t\t\t\tCoverMedium struct {\n\t\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth int `json:\"width\"`\n\t\t\t\t} `json:\"cover_medium\"`\n\t\t\t\tMixedAuthor string `json:\"mixed_author\"`\n\t\t\t\tMixedTitle string `json:\"mixed_title\"`\n\t\t\t\tTitle string `json:\"title\"`\n\t\t\t} `json:\"matched_pgc_sound\"`\n\t\t\tMid string `json:\"mid\"`\n\t\t\tMusicChartRanks interface{} `json:\"music_chart_ranks\"`\n\t\t\tMusicStatus int `json:\"music_status\"`\n\t\t\tMusicianUserInfos interface{} `json:\"musician_user_infos\"`\n\t\t\tMuteShare bool `json:\"mute_share\"`\n\t\t\tOfflineDesc string `json:\"offline_desc\"`\n\t\t\tOwnerHandle string `json:\"owner_handle\"`\n\t\t\tOwnerID string `json:\"owner_id\"`\n\t\t\tOwnerNickname string `json:\"owner_nickname\"`\n\t\t\tPgcMusicType int `json:\"pgc_music_type\"`\n\t\t\tPlayURL struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLKey string `json:\"url_key\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"play_url\"`\n\t\t\tPosition interface{} `json:\"position\"`\n\t\t\tPreventDownload bool `json:\"prevent_download\"`\n\t\t\tPreventItemDownloadStatus int `json:\"prevent_item_download_status\"`\n\t\t\tPreviewEndTime int `json:\"preview_end_time\"`\n\t\t\tPreviewStartTime float64 `json:\"preview_start_time\"`\n\t\t\tReasonType int `json:\"reason_type\"`\n\t\t\tRedirect bool `json:\"redirect\"`\n\t\t\tSchemaURL string `json:\"schema_url\"`\n\t\t\tSearchImpr struct {\n\t\t\t\tEntityID string `json:\"entity_id\"`\n\t\t\t} `json:\"search_impr\"`\n\t\t\tSecUID string `json:\"sec_uid\"`\n\t\t\tShootDuration int `json:\"shoot_duration\"`\n\t\t\tSong struct {\n\t\t\t\tArtists interface{} `json:\"artists\"`\n\t\t\t\tID int64 `json:\"id\"`\n\t\t\t\tIDStr string `json:\"id_str\"`\n\t\t\t} `json:\"song\"`\n\t\t\tSourcePlatform int `json:\"source_platform\"`\n\t\t\tStartTime int `json:\"start_time\"`\n\t\t\tStatus int `json:\"status\"`\n\t\t\tTagList interface{} `json:\"tag_list\"`\n\t\t\tTitle string `json:\"title\"`\n\t\t\tUnshelveCountries interface{} `json:\"unshelve_countries\"`\n\t\t\tUserCount int `json:\"user_count\"`\n\t\t\tVideoDuration int `json:\"video_duration\"`\n\t\t} `json:\"music\"`\n\t\tNicknamePosition interface{} `json:\"nickname_position\"`\n\t\tOriginCommentIds interface{} `json:\"origin_comment_ids\"`\n\t\tOriginTextExtra []interface{} `json:\"origin_text_extra\"`\n\t\tOriginalImages interface{} `json:\"original_images\"`\n\t\tPackedClips interface{} `json:\"packed_clips\"`\n\t\tPhotoSearchEntrance struct {\n\t\t\tEcomType int `json:\"ecom_type\"`\n\t\t} `json:\"photo_search_entrance\"`\n\t\tPosition interface{} `json:\"position\"`\n\t\tPressPanelInfo string `json:\"press_panel_info\"`\n\t\tPreviewTitle string `json:\"preview_title\"`\n\t\tPreviewVideoStatus int `json:\"preview_video_status\"`\n\t\tPromotions []interface{} `json:\"promotions\"`\n\t\tRate int `json:\"rate\"`\n\t\tRegion string `json:\"region\"`\n\t\tRelationLabels interface{} `json:\"relation_labels\"`\n\t\tSearchImpr struct {\n\t\t\tEntityID string `json:\"entity_id\"`\n\t\t\tEntityType string `json:\"entity_type\"`\n\t\t} `json:\"search_impr\"`\n\t\tSeriesPaidInfo struct {\n\t\t\tItemPrice int `json:\"item_price\"`\n\t\t\tSeriesPaidStatus int `json:\"series_paid_status\"`\n\t\t} `json:\"series_paid_info\"`\n\t\tShareInfo struct {\n\t\t\tShareDesc string `json:\"share_desc\"`\n\t\t\tShareDescInfo string `json:\"share_desc_info\"`\n\t\t\tShareLinkDesc string `json:\"share_link_desc\"`\n\t\t\tShareURL string `json:\"share_url\"`\n\t\t} `json:\"share_info\"`\n\t\tShareURL string `json:\"share_url\"`\n\t\tShouldOpenAdReport bool `json:\"should_open_ad_report\"`\n\t\tShowFollowButton struct {\n\t\t} `json:\"show_follow_button\"`\n\t\tSocialTagList interface{} `json:\"social_tag_list\"`\n\t\tStandardBarInfoList interface{} `json:\"standard_bar_info_list\"`\n\t\tStatistics struct {\n\t\t\tAdmireCount int `json:\"admire_count\"`\n\t\t\tAwemeID string `json:\"aweme_id\"`\n\t\t\tCollectCount int `json:\"collect_count\"`\n\t\t\tCommentCount int `json:\"comment_count\"`\n\t\t\tDiggCount int `json:\"digg_count\"`\n\t\t\tPlayCount int `json:\"play_count\"`\n\t\t\tShareCount int `json:\"share_count\"`\n\t\t} `json:\"statistics\"`\n\t\tStatus struct {\n\t\t\tAllowShare bool `json:\"allow_share\"`\n\t\t\tAwemeID string `json:\"aweme_id\"`\n\t\t\tInReviewing bool `json:\"in_reviewing\"`\n\t\t\tIsDelete bool `json:\"is_delete\"`\n\t\t\tIsProhibited bool `json:\"is_prohibited\"`\n\t\t\tListenVideoStatus int `json:\"listen_video_status\"`\n\t\t\tPartSee int `json:\"part_see\"`\n\t\t\tPrivateStatus int `json:\"private_status\"`\n\t\t\tReviewResult struct {\n\t\t\t\tReviewStatus int `json:\"review_status\"`\n\t\t\t} `json:\"review_result\"`\n\t\t} `json:\"status\"`\n\t\tTextExtra []struct {\n\t\t\tEnd int `json:\"end\"`\n\t\t\tHashtagID string `json:\"hashtag_id\"`\n\t\t\tHashtagName string `json:\"hashtag_name\"`\n\t\t\tIsCommerce bool `json:\"is_commerce\"`\n\t\t\tStart int `json:\"start\"`\n\t\t\tType int `json:\"type\"`\n\t\t} `json:\"text_extra\"`\n\t\tUniqidPosition interface{} `json:\"uniqid_position\"`\n\t\tUserDigged int `json:\"user_digged\"`\n\t\tVideo struct {\n\t\t\tBigThumbs []struct {\n\t\t\t\tDuration float64 `json:\"duration\"`\n\t\t\t\tFext string `json:\"fext\"`\n\t\t\t\tImgNum int `json:\"img_num\"`\n\t\t\t\tImgURL string `json:\"img_url\"`\n\t\t\t\tImgXLen int `json:\"img_x_len\"`\n\t\t\t\tImgXSize int `json:\"img_x_size\"`\n\t\t\t\tImgYLen int `json:\"img_y_len\"`\n\t\t\t\tImgYSize int `json:\"img_y_size\"`\n\t\t\t\tInterval float64 `json:\"interval\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t} `json:\"big_thumbs\"`\n\t\t\tBitRate []struct {\n\t\t\t\tFPS int `json:\"FPS\"`\n\t\t\t\tHDRBit string `json:\"HDR_bit\"`\n\t\t\t\tHDRType string `json:\"HDR_type\"`\n\t\t\t\tBitRate int `json:\"bit_rate\"`\n\t\t\t\tGearName string `json:\"gear_name\"`\n\t\t\t\tIsBytevc1 int `json:\"is_bytevc1\"`\n\t\t\t\tIsH265 int `json:\"is_h265\"`\n\t\t\t\tPlayAddr struct {\n\t\t\t\t\tDataSize int `json:\"data_size\"`\n\t\t\t\t\tFileCs string `json:\"file_cs\"`\n\t\t\t\t\tFileHash string `json:\"file_hash\"`\n\t\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\t\tURLKey string `json:\"url_key\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth int `json:\"width\"`\n\t\t\t\t} `json:\"play_addr\"`\n\t\t\t\tQualityType int `json:\"quality_type\"`\n\t\t\t} `json:\"bit_rate\"`\n\t\t\tCover struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover\"`\n\t\t\tCoverOriginalScale struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"cover_original_scale\"`\n\t\t\tDuration int `json:\"duration\"`\n\t\t\tDynamicCover struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"dynamic_cover\"`\n\t\t\tHeight int `json:\"height\"`\n\t\t\tIsH265 int `json:\"is_h265\"`\n\t\t\tIsLongVideo int `json:\"is_long_video\"`\n\t\t\tIsSourceHDR int `json:\"is_source_HDR\"`\n\t\t\tMeta string `json:\"meta\"`\n\t\t\tOriginCover struct {\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"origin_cover\"`\n\t\t\tPlayAddr struct {\n\t\t\t\tDataSize int `json:\"data_size\"`\n\t\t\t\tFileCs string `json:\"file_cs\"`\n\t\t\t\tFileHash string `json:\"file_hash\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLKey string `json:\"url_key\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"play_addr\"`\n\t\t\tPlayAddr265 struct {\n\t\t\t\tDataSize int `json:\"data_size\"`\n\t\t\t\tFileCs string `json:\"file_cs\"`\n\t\t\t\tFileHash string `json:\"file_hash\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLKey string `json:\"url_key\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"play_addr_265\"`\n\t\t\tPlayAddrH264 struct {\n\t\t\t\tDataSize int `json:\"data_size\"`\n\t\t\t\tFileCs string `json:\"file_cs\"`\n\t\t\t\tFileHash string `json:\"file_hash\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tURI string `json:\"uri\"`\n\t\t\t\tURLKey string `json:\"url_key\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth int `json:\"width\"`\n\t\t\t} `json:\"play_addr_h264\"`\n\t\t\tRatio string `json:\"ratio\"`\n\t\t\tVideoModel string `json:\"video_model\"`\n\t\t\tWidth int `json:\"width\"`\n\t\t} `json:\"video\"`\n\t\tVideoLabels interface{} `json:\"video_labels\"`\n\t\tVideoTag []struct {\n\t\t\tLevel int `json:\"level\"`\n\t\t\tTagID int `json:\"tag_id\"`\n\t\t\tTagName string `json:\"tag_name\"`\n\t\t} `json:\"video_tag\"`\n\t\tVideoText []interface{} `json:\"video_text\"`\n\t\tWannaTag struct {\n\t\t} `json:\"wanna_tag\"`\n\t} `json:\"aweme_detail\"`\n\tExtra struct {\n\t\tNow int64 `json:\"now\"`\n\t\tLogid string `json:\"logid\"`\n\t} `json:\"extra\"`\n}\n" }, { "path": "extractors/douyu/douyu.go", "content": "package douyu\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"douyu\", New())\n}\n\ntype douyuData struct {\n\tError int `json:\"error\"`\n\tData struct {\n\t\tVideoURL string `json:\"video_url\"`\n\t} `json:\"data\"`\n}\n\ntype douyuURLInfo struct {\n\tURL string\n\tSize int64\n}\n\nfunc douyuM3u8(url string) ([]douyuURLInfo, int64, error) {\n\tvar (\n\t\tdata []douyuURLInfo\n\t\ttemp douyuURLInfo\n\t\tsize, totalSize int64\n\t\terr error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tfor _, u := range urls {\n\t\tsize, err = request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = douyuURLInfo{\n\t\t\tURL: u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\ntype extractor struct{}\n\n// New returns a douyu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tliveVid := utils.MatchOneOf(url, `https?://www.douyu.com/(\\S+)`)\n\tif liveVid != nil {\n\t\treturn nil, errors.New(\"暂不支持斗鱼直播\")\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `(.*?)`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\tvids := utils.MatchOneOf(url, `https?://v.douyu.com/show/(\\S+)`)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tdataString, err := request.Get(\"http://vmobile.douyu.com/video/getInfo?vid=\"+vid, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdataDict := new(douyuData)\n\tif err := json.Unmarshal([]byte(dataString), dataDict); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tm3u8URLs, totalSize, err := douyuM3u8(dataDict.Data.VideoURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turls := make([]*extractors.Part, len(m3u8URLs))\n\tfor index, u := range m3u8URLs {\n\t\turls[index] = &extractors.Part{\n\t\t\tURL: u.URL,\n\t\t\tSize: u.Size,\n\t\t\tExt: \"ts\",\n\t\t}\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"斗鱼 douyu.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/douyu/douyu_test.go", "content": "package douyu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://v.douyu.com/show/l0Q8mMY3wZqv49Ad\",\n\t\t\t\tTitle: \"每日撸报_每日撸报：有些人死了其实它还可以把你带走_斗鱼视频 - 最6的弹幕视频网站\",\n\t\t\t\tSize: 10558080,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n" }, { "path": "extractors/eporner/eporner.go", "content": "package eporner\n\nimport (\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/PuerkitoBio/goquery\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"eporner\", New())\n}\n\nconst (\n\tdownloadclass = \".dloaddivcol\"\n)\n\ntype src struct {\n\turl string\n\tquality string\n\tsizestr string\n\tsize int64\n}\n\nfunc getSrcMeta(text string) *src {\n\tsti := strings.Index(text, \"(\")\n\tste := strings.Index(text, \")\")\n\titext := text[sti+1 : ste]\n\tstrs := strings.Split(itext, \",\")\n\ts := &src{}\n\n\tif len(strs) == 2 {\n\t\ts.quality = strings.Trim(strs[0], \" \")\n\t\ts.sizestr = strings.Trim(strs[1], \" \")\n\t}\n\n\tif s.sizestr == \"\" {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\n\tvalunit := strings.Split(s.sizestr, \" \")\n\tval, err := strconv.ParseFloat(valunit[0], 64)\n\tif err != nil {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\tunit := valunit[1]\n\tswitch unit {\n\tcase \"KB\":\n\t\ts.size = int64(val * 1024)\n\tcase \"MB\":\n\t\ts.size = int64(val * 1024 * 1024)\n\tcase \"GB\":\n\t\ts.size = int64(val * 1024 * 1024 * 1024)\n\tdefault:\n\t\ts.size = int64(val)\n\t}\n\treturn s\n}\n\nfunc getSrc(html string) []*src {\n\tsrcs := []*src{}\n\td, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\td.Find(downloadclass).Each(func(i int, s *goquery.Selection) {\n\t\ts.Contents().Each(func(i int, s *goquery.Selection) {\n\t\t\tfor ns := range s.Nodes {\n\t\t\t\tn := s.Get(ns)\n\t\t\t\tif n.Data == \"a\" {\n\t\t\t\t\tvar sr *src\n\t\t\t\t\tif n.FirstChild != nil {\n\t\t\t\t\t\tsr = getSrcMeta(n.FirstChild.Data)\n\t\t\t\t\t}\n\t\t\t\t\tfor _, a := range n.Attr {\n\t\t\t\t\t\tif a.Key == \"href\" {\n\t\t\t\t\t\t\tsr.url = a.Val\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tsrcs = append(srcs, sr)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t})\n\n\treturn srcs\n}\n\ntype extractor struct{}\n\n// New returns a eporner extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(u, u, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `(.+?)`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"eporner\"\n\t}\n\tuu, err := url.Parse(u)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsrcs := getSrc(html)\n\tstreams := make(map[string]*extractors.Stream, len(srcs))\n\tfor _, src := range srcs {\n\t\tsrcurl := uu.Scheme + \"://\" + uu.Host + src.url\n\t\t// skipping an extra HEAD request to the URL.\n\t\t// size, err := request.Size(srcurl, u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL: srcurl,\n\t\t\tSize: src.size,\n\t\t\tExt: \"mp4\",\n\t\t}\n\t\tstreams[src.quality] = &extractors.Stream{\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: src.size,\n\t\t\tQuality: src.quality,\n\t\t}\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"EPORNER eporner.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: u,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/eporner/eporner_test.go", "content": "package eporner\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.eporner.com/video-mbubfvXYFip/dirtywivesclub-becky-bandini/\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize: 1525510307,\n\t\t\t\tTitle: \"DirtyWivesClub - Becky Bandini - EPORNER\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/errors.go", "content": "package extractors\n\nimport (\n\t\"errors\"\n)\n\nvar (\n\t// ErrURLParseFailed defines url parse failed error.\n\tErrURLParseFailed = errors.New(\"url parse failed\")\n\tErrInvalidRegularExpression = errors.New(\"invalid regular expression\")\n\tErrURLQueryParamsParseFailed = errors.New(\"url query params parse failed\")\n\tErrBodyParseFailed = errors.New(\"body parse failed\")\n)\n" }, { "path": "extractors/extractors.go", "content": "package extractors\n\nimport (\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar lock sync.RWMutex\nvar extractorMap = make(map[string]Extractor)\n\n// Register registers an Extractor.\nfunc Register(domain string, e Extractor) {\n\tlock.Lock()\n\textractorMap[domain] = e\n\tlock.Unlock()\n}\n\n// Extract is the main function to extract the data.\nfunc Extract(u string, option Options) ([]*Data, error) {\n\tu = strings.TrimSpace(u)\n\tvar domain string\n\n\tbilibiliShortLink := utils.MatchOneOf(u, `^(av|BV|ep)\\w+`)\n\tif len(bilibiliShortLink) > 1 {\n\t\tbilibiliURL := map[string]string{\n\t\t\t\"av\": \"https://www.bilibili.com/video/\",\n\t\t\t\"BV\": \"https://www.bilibili.com/video/\",\n\t\t\t\"ep\": \"https://www.bilibili.com/bangumi/play/\",\n\t\t}\n\t\tdomain = \"bilibili\"\n\t\tu = bilibiliURL[bilibiliShortLink[1]] + u\n\t} else {\n\t\tu, err := url.ParseRequestURI(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif u.Host == \"haokan.baidu.com\" {\n\t\t\tdomain = \"haokan\"\n\t\t} else if u.Host == \"xhslink.com\" {\n\t\t\tdomain = \"xiaohongshu\"\n\t\t} else {\n\t\t\tdomain = utils.Domain(u.Host)\n\t\t}\n\t}\n\textractor := extractorMap[domain]\n\tif extractor == nil {\n\t\textractor = extractorMap[\"\"]\n\t}\n\tvideos, err := extractor.Extract(u, option)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor _, v := range videos {\n\t\tv.FillUpStreamsData()\n\t}\n\treturn videos, nil\n}\n" }, { "path": "extractors/facebook/facebook.go", "content": "package facebook\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"facebook\", New())\n}\n\ntype extractor struct{}\n\n// New returns a facebook extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `([^<]+)`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := strings.TrimSpace(titles[1])\n\n\ttitle = regexp.MustCompile(`\\n+`).ReplaceAllString(title, \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"playable_url\":\\s*\"([^\"]+)\"`),\n\t\t// \"hd\": regexp.MustCompile(`\"playable_url_quality_hd\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 2)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\n\t\tif len(matcher) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], \"\\\\\", \"\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL: u,\n\t\t\tSize: size,\n\t\t\tExt: \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Facebook facebook.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/facebook/facebook_test.go", "content": "package facebook\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.facebook.com/100058251872436/videos/424557726111987\",\n\t\t\t\tTitle: \"Роман Грищук - Підтримка з Японії 🇯🇵 Гурт Yokohama Sisters 👏\",\n\t\t\t\tSize: 1441128,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/geekbang/geekbang.go", "content": "package geekbang\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"geekbang\", New())\n}\n\ntype geekData struct {\n\tCode int `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData struct {\n\t\tVideoID string `json:\"video_id\"`\n\t\tTitle string `json:\"article_sharetitle\"`\n\t\tColumnHadSub bool `json:\"column_had_sub\"`\n\t} `json:\"data\"`\n}\n\ntype videoPlayAuth struct {\n\tCode int `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData struct {\n\t\tPlayAuth string `json:\"play_auth\"`\n\t} `json:\"data\"`\n}\n\ntype playInfo struct {\n\tVideoBase struct {\n\t\tVideoID string `json:\"VideoId\"`\n\t\tTitle string `json:\"Title\"`\n\t\tCoverURL string `josn:\"CoverURL\"`\n\t} `json:\"VideoBase\"`\n\tPlayInfoList struct {\n\t\tPlayInfo []struct {\n\t\t\tURL string `json:\"PlayURL\"`\n\t\t\tSize int64 `json:\"Size\"`\n\t\t\tDefinition string `json:\"Definition\"`\n\t\t} `json:\"PlayInfo\"`\n\t} `json:\"PlayInfoList\"`\n}\n\ntype geekURLInfo struct {\n\tURL string\n\tSize int64\n}\n\nfunc geekM3u8(url string) ([]geekURLInfo, error) {\n\tvar (\n\t\tdata []geekURLInfo\n\t\ttemp geekURLInfo\n\t\tsize int64\n\t\terr error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor _, u := range urls {\n\t\ttemp = geekURLInfo{\n\t\t\tURL: u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct{}\n\n// New returns a geekbang extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tmatches := utils.MatchOneOf(url, `https?://time.geekbang.org/course/detail/(\\d+)-(\\d+)`)\n\tif matches == nil || len(matches) < 3 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get video information\n\theanders := map[string]string{\"Origin\": \"https://time.geekbang.org\", \"Content-Type\": \"application/json\", \"Referer\": url}\n\tparams := strings.NewReader(fmt.Sprintf(`{\"id\": %q}`, matches[2]))\n\tres, err := request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v1/article\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar data geekData\n\tif err = json.NewDecoder(res.Body).Decode(&data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif data.Code < 0 {\n\t\treturn nil, errors.New(string(data.Error))\n\t}\n\n\tif data.Data.VideoID == \"\" && !data.Data.ColumnHadSub {\n\t\treturn nil, errors.New(\"请先购买课程，或使用Cookie登录。\")\n\t}\n\n\t// Get video license token information\n\tparams = strings.NewReader(\"{\\\"source_type\\\":1,\\\"aid\\\":\" + matches[2] + \",\\\"video_id\\\":\\\"\" + data.Data.VideoID + \"\\\"}\")\n\tres, err = request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v3/source_auth/video_play_auth\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playAuth videoPlayAuth\n\tif err = json.NewDecoder(res.Body).Decode(&playAuth); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif playAuth.Code < 0 {\n\t\treturn nil, errors.New(string(playAuth.Error))\n\t}\n\n\t// Get video playback information\n\theanders = map[string]string{\"Accept-Encoding\": \"\"}\n\tres, err = request.Request(http.MethodGet, \"http://ali.mantv.top/play/info?playAuth=\"+playAuth.Data.PlayAuth, nil, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playInfo playInfo\n\tif err = json.NewDecoder(res.Body).Decode(&playInfo); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.Data.Title\n\n\tstreams := make(map[string]*extractors.Stream, len(playInfo.PlayInfoList.PlayInfo))\n\n\tfor _, media := range playInfo.PlayInfoList.PlayInfo {\n\t\tm3u8URLs, err := geekM3u8(media.URL)\n\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL: u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt: \"ts\",\n\t\t\t}\n\t\t}\n\n\t\tstreams[media.Definition] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize: media.Size,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"极客时间 geekbang.org\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/geekbang/geekbang_test.go", "content": "package geekbang\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://time.geekbang.org/course/detail/190-97203\",\n\t\t\t\tTitle: \"02 | 内容综述 - 玩转webpack\",\n\t\t\t\tSize: 10752472,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/haokan/haokan.go", "content": "package haokan\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"haokan\", New())\n}\n\ntype extractor struct{}\n\n// New returns a haokan extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `property=\"og:title\"\\s+content=\"(.+?)\"`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// 之前的好看网页中，视频地址是放在 video 标签下\n\turls := utils.MatchOneOf(html, ``)\n\n\tif urls == nil || len(urls) < 2 {\n\t\t// fallbak: 新的好看网页中，视频地址在 json 数据里\n\t\turls = utils.MatchOneOf(html, `\"playurl\":\"(http.+?)\"`)\n\t}\n\n\tif urls == nil || len(urls) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tplayurl := strings.Replace(urls[1], `\\/`, `/`, -1)\n\n\tsize, err := request.Size(playurl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t_, ext, err := utils.GetNameAndExt(playurl)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL: playurl,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt: ext,\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"好看视频 haokan.baidu.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/haokan/haokan_test.go", "content": "package haokan\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://haokan.baidu.com/v?vid=10057409468467026969\",\n\t\t\t\tTitle: \"听歌学英语小学篇（6）：my new pen pal\",\n\t\t\t\tSize: 2027354,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/hupu/hupu.go", "content": "package hupu\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"hupu\", New())\n}\n\ntype extractor struct{}\n\n// New returns a hupu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `(.+?)`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"hupu video\"\n\t}\n\n\tvar videoUrl string\n\turlDesc := utils.MatchOneOf(html, ``)\n\tif len(urlDesc) > 1 {\n\t\tvideoUrl = urlDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL: videoUrl,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"虎扑 hupu.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/hupu/hupu_test.go", "content": "package hupu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHupu(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://bbs.hupu.com/47401018.html?is_reflow=1&cid=84752419&bddid=56KXU5QUJH4VGM26SFPTYTKNI5CFNJMX736TIZ52DXLGUAAMBJVA01&puid=16522089&client=8577E496-4D9B-4E5C-A9DB-A8EF5C1956D2\",\n\t\t\t\tTitle: \"结局引起舒适\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n" }, { "path": "extractors/huya/huya.go", "content": "package huya\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"huya\", New())\n}\n\ntype extractor struct{}\n\nconst huyaVideoHost = \"https://videotx-platform.cdn.huya.com/\"\n\n// New returns a huya extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `

(.+?)

`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"huya video\"\n\t}\n\n\tvar videoUrl string\n\tvideoDesc := utils.MatchOneOf(html, `//videotx-platform.cdn.huya.com/(.*)\" poster=(.+?)`)\n\tif len(videoDesc) > 1 {\n\t\tvideoUrl = huyaVideoHost + videoDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL: videoUrl,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"虎牙 huya.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/huya/huya_test.go", "content": "package huya\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHuya(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2\",\n\t\t\t\tTitle: \"12.28 集梦薛小谦【封号斗罗】直播名场面\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n" }, { "path": "extractors/instagram/instagram.go", "content": "package instagram\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar client *http.Client\n\nfunc init() {\n\textractors.Register(\"instagram\", New())\n\tclient = &http.Client{\n\t\tTimeout: 10 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t},\n\t}\n}\n\n// sliderItemNode contains information about the Instagram post\ntype sliderItemNode struct {\n\tDisplayURL string `json:\"display_url\"` // URL of the Media (resolution is dynamic)\n\n\tIsVideo bool `json:\"is_video\"` // Is type of the Media equals to video\n\tVideoURL string `json:\"video_url\"` // Direct URL to the Video\n}\n\nfunc (s sliderItemNode) extractMediaURL() string {\n\tif s.IsVideo {\n\t\treturn s.VideoURL\n\t}\n\n\treturn s.DisplayURL\n}\n\ntype instagramPayload struct {\n\tMedia struct {\n\t\tID string `json:\"id\"` // Unique ID of the Media\n\t\tSliderItems struct {\n\t\t\tEdges []struct {\n\t\t\t\tNode sliderItemNode `json:\"node\"`\n\t\t\t} `json:\"edges\"`\n\t\t} `json:\"edge_sidecar_to_children\"` // Children of the Media\n\t} `json:\"shortcode_media\"` // Media\n}\n\nfunc (s instagramPayload) isEmpty() bool {\n\treturn s.Media.ID == \"\"\n}\n\nfunc getPostWithCode(code string) ([]string, error) {\n\tURL := fmt.Sprintf(\"https://www.instagram.com/p/%v/embed/captioned/\", code)\n\n\tvar embeddedMediaImage string\n\tvar embedResponse = instagramPayload{}\n\tcollector := colly.NewCollector()\n\tcollector.SetClient(client)\n\tvar collectorErr error\n\n\tcollector.OnHTML(\"img.EmbeddedMediaImage\", func(e *colly.HTMLElement) {\n\t\tembeddedMediaImage = e.Attr(\"src\")\n\t})\n\n\tcollector.OnHTML(\"script\", func(e *colly.HTMLElement) {\n\t\tr := regexp.MustCompile(`\\\\\\\"gql_data\\\\\\\":([\\s\\S]*)\\}\\\"\\}\\]\\]\\,\\[\\\"NavigationMetrics`)\n\t\tmatch := r.FindStringSubmatch(e.Text)\n\n\t\tif len(match) < 2 {\n\t\t\treturn\n\t\t}\n\n\t\ts := strings.ReplaceAll(match[1], `\\\"`, `\"`)\n\t\ts = strings.ReplaceAll(s, `\\\\/`, `/`)\n\t\ts = strings.ReplaceAll(s, `\\\\`, `\\`)\n\n\t\terr := json.Unmarshal([]byte(s), &embedResponse)\n\t\tif err != nil {\n\t\t\tcollectorErr = err\n\t\t}\n\t})\n\n\tcollector.OnRequest(func(r *colly.Request) {\n\t\tr.Headers.Set(\"User-Agent\", browser.Chrome())\n\t})\n\n\tif err := collector.Visit(URL); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to send HTTP request to the Instagram: %v\", err)\n\t}\n\n\tif collectorErr != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse the Instagram response: %v\", collectorErr)\n\t}\n\n\t// If the method one which is JSON parsing didn't fail\n\tif !embedResponse.isEmpty() {\n\t\tresult := make([]string, 0, len(embedResponse.Media.SliderItems.Edges))\n\t\tfor _, item := range embedResponse.Media.SliderItems.Edges {\n\t\t\tresult = append(result, item.Node.extractMediaURL())\n\t\t}\n\n\t\treturn result, nil\n\t}\n\n\tif embeddedMediaImage != \"\" {\n\t\treturn []string{embeddedMediaImage}, nil\n\t}\n\n\t// If every two methods have failed, then return an error\n\treturn nil, errors.New(\"failed to fetch the post, the page might be \\\"private\\\", or the link is completely wrong\")\n}\n\nfunc extractShortCodeFromLink(link string) (string, error) {\n\tvalues := regexp.MustCompile(`(p|tv|reel|reels\\/videos)\\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link)\n\tif len(values) != 3 {\n\t\treturn \"\", errors.New(\"couldn't extract the media short code from the link\")\n\t}\n\n\treturn values[2], nil\n}\n\ntype extractor struct{}\n\n// New returns a instagram extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tu, err := netURL.Parse(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tshortCode, err := extractShortCodeFromLink(u.String())\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turls, err := getPostWithCode(shortCode)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar totalSize int64\n\tvar parts []*extractors.Part\n\n\tfor _, u := range urls {\n\t\t_, ext, err := utils.GetNameAndExt(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfileSize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tpart := &extractors.Part{\n\t\t\tURL: u,\n\t\t\tSize: fileSize,\n\t\t\tExt: ext,\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tfor _, part := range parts {\n\t\ttotalSize += part.Size\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Instagram instagram.com\",\n\t\t\tTitle: \"Instagram \" + shortCode,\n\t\t\tType: extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/instagram/instagram_test.go", "content": "package instagram\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.instagram.com/p/BlIka1ZFCNr\",\n\t\t\t\tTitle: \"Instagram BlIka1ZFCNr\",\n\t\t\t\tSize: 992330,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.instagram.com/p/Bl5oVUyl9Yx\",\n\t\t\t\tTitle: \"Instagram Bl5oVUyl9Yx\",\n\t\t\t\tSize: 250596,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image album test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.instagram.com/p/Bjyr-gxF4Rb\",\n\t\t\t\tTitle: \"Instagram Bjyr-gxF4Rb\",\n\t\t\t\tSize: 656476,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/iqiyi/iqiyi.go", "content": "package iqiyi\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"iqiyi\", New(SiteTypeIqiyi))\n\textractors.Register(\"iq\", New(SiteTypeIQ))\n}\n\ntype iqiyi struct {\n\tCode string `json:\"code\"`\n\tData struct {\n\t\tVP struct {\n\t\t\tDu string `json:\"du\"`\n\t\t\tTkl []struct {\n\t\t\t\tVs []struct {\n\t\t\t\t\tBid int `json:\"bid\"`\n\t\t\t\t\tScrsz string `json:\"scrsz\"`\n\t\t\t\t\tVsize int64 `json:\"vsize\"`\n\t\t\t\t\tFs []struct {\n\t\t\t\t\t\tL string `json:\"l\"`\n\t\t\t\t\t\tB int64 `json:\"b\"`\n\t\t\t\t\t} `json:\"fs\"`\n\t\t\t\t} `json:\"vs\"`\n\t\t\t} `json:\"tkl\"`\n\t\t} `json:\"vp\"`\n\t} `json:\"data\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype iqiyiURL struct {\n\tL string `json:\"l\"`\n}\n\n// SiteType indicates the site type of iqiyi\ntype SiteType int\n\nconst (\n\t// SiteTypeIQ indicates the site is iq.com\n\tSiteTypeIQ SiteType = iota\n\t// SiteTypeIqiyi indicates the site is iqiyi.com\n\tSiteTypeIqiyi\n\tiqReferer = \"https://www.iq.com\"\n\tiqiyiReferer = \"https://www.iqiyi.com\"\n)\n\nfunc getMacID() string {\n\tvar macID string\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tsize := len(chars)\n\tfor i := 0; i < 32; i++ {\n\t\tmacID += chars[rand.Intn(size)]\n\t}\n\treturn macID\n}\n\nfunc getVF(params string) string {\n\tvar suffix string\n\tfor j := 0; j < 8; j++ {\n\t\tfor k := 0; k < 4; k++ {\n\t\t\tvar v8 int\n\t\t\tv4 := 13 * (66*k + 27*j) % 35\n\t\t\tif v4 >= 10 {\n\t\t\t\tv8 = v4 + 88\n\t\t\t} else {\n\t\t\t\tv8 = v4 + 49\n\t\t\t}\n\t\t\tsuffix += string(rune(v8)) // string(97) -> \"a\"\n\t\t}\n\t}\n\tparams += suffix\n\n\treturn utils.Md5(params)\n}\n\nfunc getVPS(tvid, vid, refer string) (*iqiyi, error) {\n\tt := time.Now().Unix() * 1000\n\thost := \"http://cache.video.qiyi.com\"\n\tparams := fmt.Sprintf(\n\t\t\"/vps?tvid=%s&vid=%s&v=0&qypid=%s_12&src=01012001010000000000&t=%d&k_tag=1&k_uid=%s&rs=1\",\n\t\ttvid, vid, tvid, t, getMacID(),\n\t)\n\tvf := getVF(params)\n\tapiURL := fmt.Sprintf(\"%s%s&vf=%s\", host, params, vf)\n\tinfo, err := request.Get(apiURL, refer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdata := new(iqiyi)\n\tif err := json.Unmarshal([]byte(info), data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct {\n\tsiteType SiteType\n}\n\n// New returns a iqiyi extractor.\nfunc New(siteType SiteType) extractors.Extractor {\n\treturn &extractor{\n\t\tsiteType: siteType,\n\t}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\trefer := iqiyiReferer\n\theaders := make(map[string]string)\n\tif e.siteType == SiteTypeIQ {\n\t\theaders = map[string]string{\n\t\t\t\"Accept-Language\": \"zh-TW\",\n\t\t}\n\t\trefer = iqReferer\n\t}\n\thtml, err := request.Get(url, refer, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=(.+)_`,\n\t\t`tvid=([^&]+)`,\n\t)\n\tif tvid == nil {\n\t\ttvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-tvid=\"([^\"]+)\"`,\n\t\t\t`param\\['tvid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"tvid\":\"(\\d+)\"`,\n\t\t\t`\"tvId\":(\\d+)`,\n\t\t)\n\t}\n\tif tvid == nil || len(tvid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=.+_(.*)$`,\n\t\t`vid=([^&]+)`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-videoid=\"([^\"]+)\"`,\n\t\t\t`param\\['vid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"vid\":\"(\\w+)\"`,\n\t\t)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tif e.siteType == SiteTypeIqiyi {\n\t\ttitle = strings.TrimSpace(doc.Find(\"h1>a\").First().Text())\n\t\tvar sub string\n\t\tfor _, k := range []string{\"span\", \"em\"} {\n\t\t\tif sub != \"\" {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tsub = strings.TrimSpace(doc.Find(\"h1>\" + k).First().Text())\n\t\t}\n\t\ttitle += sub\n\t} else {\n\t\ttitle = strings.TrimSpace(doc.Find(\"span#pageMetaTitle\").First().Text())\n\t\tsub := utils.MatchOneOf(html, `\"subTitle\":\"([^\"]+)\",\"isoDuration\":`)\n\t\tif len(sub) > 1 {\n\t\t\ttitle += fmt.Sprintf(\" %s\", sub[1])\n\t\t}\n\t}\n\tif title == \"\" {\n\t\ttitle = doc.Find(\"title\").Text()\n\t}\n\tvideoDatas, err := getVPS(tvid[1], vid[1], refer)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif videoDatas.Code != \"A00000\" {\n\t\treturn nil, errors.Errorf(\"can't play this video: %s\", videoDatas.Msg)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\turlPrefix := videoDatas.Data.VP.Du\n\tfor _, video := range videoDatas.Data.VP.Tkl[0].Vs {\n\t\turls := make([]*extractors.Part, len(video.Fs))\n\t\tfor index, v := range video.Fs {\n\t\t\trealURLData, err := request.Get(urlPrefix+v.L, refer, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tvar realURL iqiyiURL\n\t\t\tif err = json.Unmarshal([]byte(realURLData), &realURL); err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL.L)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL: realURL.L,\n\t\t\t\tSize: v.B,\n\t\t\t\tExt: ext,\n\t\t\t}\n\t\t}\n\t\tstreams[strconv.Itoa(video.Bid)] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize: video.Vsize,\n\t\t\tQuality: video.Scrsz,\n\t\t}\n\t}\n\n\tsiteName := \"爱奇艺 iqiyi.com\"\n\tif e.siteType == SiteTypeIQ {\n\t\tsiteName = \"爱奇艺 iq.com\"\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: siteName,\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/iqiyi/iqiyi_test.go", "content": "package iqiyi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"http://www.iqiyi.com/v_19rrbdmaj0.html\",\n\t\t\t\tTitle: \"新一轮降水将至 冷空气影响中东部地区\",\n\t\t\t\tSize: 2952228,\n\t\t\t\tQuality: \"896x504\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"title test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"http://www.iqiyi.com/v_19rqy2z83w.html\",\n\t\t\t\tTitle: \"收了创意视频2018 :58天环球飞行记\",\n\t\t\t\tSize: 76186786,\n\t\t\t\tQuality: \"1920x1080\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"curid test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.iqiyi.com/v_19rro0jdls.html#curid=350289100_6e6601aae889d0b1004586a52027c321\",\n\t\t\t\tTitle: \"Shawn Mendes - Never Be Alone\",\n\t\t\t\tSize: 79921894,\n\t\t\t\tQuality: \"1920x800\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New(SiteTypeIqiyi).Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/ixigua/ixigua.go", "content": "package ixigua\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"ixigua\", New())\n\textractors.Register(\"toutiao\", New())\n}\n\ntype extractor struct{}\n\ntype Video struct {\n\tTitle string `json:\"title\"`\n\tQualities []struct {\n\t\tQuality string `json:\"quality\"`\n\t\tSize int64 `json:\"size\"`\n\t\tURL string `json:\"url\"`\n\t\tExt string `json:\"ext\"`\n\t} `json:\"qualities\"`\n}\n\n// New returns a ixigua extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": browser.Chrome(),\n\t\t\"Cookie\": option.Cookie,\n\t}\n\n\t// ixigua 有三种格式的 URL\n\t// 格式一 https://www.ixigua.com/7053389963487871502\n\t// 格式二 https://v.ixigua.com/RedcbWM/\n\t// 格式三 https://m.toutiao.com/is/dtj1pND/\n\t// 格式二会跳转到格式一\n\t// 格式三会跳转到 https://www.toutiao.com/a7053389963487871502\n\n\tvar finalURL string\n\tif strings.HasPrefix(url, \"https://www.ixigua.com/\") {\n\t\tfinalURL = url\n\t}\n\n\tif strings.HasPrefix(url, \"https://v.ixigua.com/\") || strings.HasPrefix(url, \"https://m.toutiao.com/\") {\n\t\tresp, err := http.Get(url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\t// follow redirects, https://stackoverflow.com/a/16785343\n\t\tfinalURL = resp.Request.URL.String()\n\t}\n\n\tfinalURL = strings.ReplaceAll(finalURL, \"https://www.toutiao.com/video/\", \"https://www.ixigua.com/\")\n\n\tr := regexp.MustCompile(`(ixigua.com/)(\\w+)?`)\n\tid := r.FindSubmatch([]byte(finalURL))[2]\n\turl2 := fmt.Sprintf(\"https://www.ixigua.com/%s\", string(id))\n\n\tbody, err := request.Get(url2, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\\{.*?\\})\\<\\/script\\>`)\n\tif videoListJson == nil || len(videoListJson) != 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrBodyParseFailed)\n\t}\n\n\tvideoUrl := videoListJson[1]\n\tvideoUrl = strings.Replace(videoUrl, \":undefined\", \":\\\"undefined\\\"\", -1)\n\n\tvar data xiguanData\n\tif err = json.Unmarshal([]byte(videoUrl), &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.AnyVideo.GidInformation.PackerData.Video.Title\n\tvideoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList\n\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, v := range videoList {\n\t\tstreams[v.Definition] = &extractors.Stream{\n\t\t\tQuality: v.Definition,\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL: base64Decode(v.MainUrl),\n\t\t\t\t\tSize: v.Size,\n\t\t\t\t\tExt: v.Vtype,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"西瓜视频 ixigua.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n\nfunc base64Decode(t string) string {\n\td, _ := base64.StdEncoding.DecodeString(t)\n\treturn string(d)\n}\n" }, { "path": "extractors/ixigua/ixigua_test.go", "content": "package ixigua\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.ixigua.com/7053389963487871502\",\n\t\t\t\tTitle: \"漫威斥巨资拍的《永恒族》，刚上架就被多国禁播，究竟拍了什么？\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize: 313091514,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://v.ixigua.com/RedcbWM/\",\n\t\t\t\tTitle: \"为长生不老，竟然连小鲛人都杀@中视频伙伴计划官号\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize: 64980732,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://m.toutiao.com/is/dtj1pND/\",\n\t\t\t\tTitle: \"卡尔：59杀4200法强小法师，点塔只需一下，W技能瞬秒对方\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize: 468324298,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/ixigua/types.go", "content": "package ixigua\n\ntype xiguanData struct {\n\tAnyVideo struct {\n\t\tGidInformation struct {\n\t\t\tGid string `json:\"gid\"`\n\t\t\tPackerData struct {\n\t\t\t\tVideo struct {\n\t\t\t\t\tTitle string `json:\"title\"`\n\t\t\t\t\tPosterUrl string `json:\"poster_url\"`\n\t\t\t\t\tVideoResource struct {\n\t\t\t\t\t\tVid string `json:\"vid\"`\n\t\t\t\t\t\tNormal struct {\n\t\t\t\t\t\t\tVideoId string `json:\"video_id\"`\n\t\t\t\t\t\t\tVideoList map[string]struct {\n\t\t\t\t\t\t\t\tDefinition string `json:\"definition\"`\n\t\t\t\t\t\t\t\tQuality string `json:\"quality\"`\n\t\t\t\t\t\t\t\tVtype string `json:\"vtype\"`\n\t\t\t\t\t\t\t\tVwidth int `json:\"vwidth\"`\n\t\t\t\t\t\t\t\tVheight int `json:\"vheight\"`\n\t\t\t\t\t\t\t\tBitrate int64 `json:\"bitrate\"`\n\t\t\t\t\t\t\t\tRealBitrate int64 `json:\"real_bitrate\"`\n\t\t\t\t\t\t\t\tFps int `json:\"fps\"`\n\t\t\t\t\t\t\t\tCodecType string `json:\"codec_type\"`\n\t\t\t\t\t\t\t\tSize int64 `json:\"size\"`\n\t\t\t\t\t\t\t\tMainUrl string `json:\"main_url\"`\n\t\t\t\t\t\t\t\tBackupUrl1 string `json:\"backup_url_1\"`\n\t\t\t\t\t\t\t} `json:\"video_list\"`\n\t\t\t\t\t\t} `json:\"normal\"`\n\t\t\t\t\t} `json:\"videoResource\"`\n\t\t\t\t} `json:\"video\"`\n\t\t\t\tKey string `json:\"key\"`\n\t\t\t} `json:\"packerData\"`\n\t\t} `json:\"gidInformation\"`\n\t} `json:\"anyVideo\"`\n}\n" }, { "path": "extractors/kuaishou/kuaishou.go", "content": "package kuaishou\n\nimport (\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"kuaishou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a kuaishou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// fetch url and get the cookie that write by server\nfunc fetchCookies(url string, headers map[string]string) (string, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, headers)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tcookiesArr := make([]string, 0)\n\tcookies := res.Cookies()\n\n\tfor _, c := range cookies {\n\t\tcookiesArr = append(cookiesArr, c.Name+\"=\"+c.Value)\n\t}\n\n\treturn strings.Join(cookiesArr, \"; \"), nil\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t}\n\n\tcookies, err := fetchCookies(url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\theaders[\"Cookie\"] = cookies\n\n\thtml, err := request.Get(url, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `([^<]+)`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.New(\"can not found title\")\n\t}\n\n\ttitle := regexp.MustCompile(`\\n+`).ReplaceAllString(strings.TrimSpace(titles[1]), \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"photoUrl\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 1)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\t\tif len(matcher) != 2 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], `\\u002F`, \"/\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL: u,\n\t\t\tSize: size,\n\t\t\tExt: \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"快手 kuaishou.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/kuaishou/kuaishou_test.go", "content": "package kuaishou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.kuaishou.com/short-video/3x43cyvcyph57i4?authorId=3xtq3uqyjmhbimq&streamSource=find&area=homexxbrilliant\",\n\t\t\t\tTitle: \"现在连戴口罩都开始内卷了吗？！快get口罩心机戴法，直接戴出小V脸啊 ！ #口罩 #显脸小-快手\",\n\t\t\t\tSize: 1077774,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/mgtv/mgtv.go", "content": "package mgtv\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"mgtv\", New())\n}\n\ntype mgtvVideoStream struct {\n\tName string `json:\"name\"`\n\tURL string `json:\"url\"`\n\tDef string `json:\"def\"`\n}\n\ntype mgtvVideoInfo struct {\n\tTitle string `json:\"title\"`\n\tDesc string `json:\"desc\"`\n}\n\ntype mgtvVideoData struct {\n\tStream []mgtvVideoStream `json:\"stream\"`\n\tStreamDomain []string `json:\"stream_domain\"`\n\tInfo mgtvVideoInfo `json:\"info\"`\n}\n\ntype mgtv struct {\n\tData mgtvVideoData `json:\"data\"`\n}\n\ntype mgtvVideoAddr struct {\n\tInfo string `json:\"info\"`\n}\n\ntype mgtvURLInfo struct {\n\tURL string\n\tSize int64\n}\n\ntype mgtvPm2Data struct {\n\tData struct {\n\t\tAtc struct {\n\t\t\tPm2 string `json:\"pm2\"`\n\t\t} `json:\"atc\"`\n\t\tInfo mgtvVideoInfo `json:\"info\"`\n\t} `json:\"data\"`\n}\n\nfunc mgtvM3u8(url string) ([]mgtvURLInfo, int64, error) {\n\tvar data []mgtvURLInfo\n\tvar temp mgtvURLInfo\n\tvar size, totalSize int64\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tm3u8String, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tsizes := utils.MatchAll(m3u8String, `#EXT-MGTV-File-SIZE:(\\d+)`)\n\t// sizes: [[#EXT-MGTV-File-SIZE:1893724, 1893724]]\n\tfor index, u := range urls {\n\t\tsize, err = strconv.ParseInt(sizes[index][1], 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = mgtvURLInfo{\n\t\t\tURL: u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\nfunc encodeTk2(str string) string {\n\tencodeString := base64.StdEncoding.EncodeToString([]byte(str))\n\tr1 := regexp.MustCompile(`/\\+/g`)\n\tr2 := regexp.MustCompile(`///g`)\n\tr3 := regexp.MustCompile(`/=/g`)\n\tr1.ReplaceAllString(encodeString, \"_\")\n\tr2.ReplaceAllString(encodeString, \"~\")\n\tr3.ReplaceAllString(encodeString, \"-\")\n\tencodeString = utils.Reverse(encodeString)\n\treturn encodeString\n}\n\ntype extractor struct{}\n\n// New returns a mgtv extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`https?://www.mgtv.com/(?:b|l)/\\d+/(\\d+).html`,\n\t\t`https?://www.mgtv.com/hz/bdpz/\\d+/(\\d+).html`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(html, `vid: (\\d+),`)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// API extract from https://js.mgtv.com/imgotv-miniv6/global/page/play-tv.js\n\t// getSource and getPlayInfo function\n\t// Chrome Network JS panel\n\theaders := map[string]string{\n\t\t\"Cookie\": \"PM_CHKID=1\",\n\t}\n\tclit := fmt.Sprintf(\"clit=%d\", time.Now().Unix()/1000)\n\tpm2DataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/video?video_id=%s&tk2=%s\",\n\t\t\tvid[1],\n\t\t\tencodeTk2(fmt.Sprintf(\n\t\t\t\t\"did=f11dee65-4e0d-4d25-bfce-719ad9dc991d|pno=1030|ver=5.5.1|%s\", clit,\n\t\t\t)),\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar pm2 mgtvPm2Data\n\tif err = json.Unmarshal([]byte(pm2DataString), &pm2); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/getSource?video_id=%s&tk2=%s&pm2=%s\",\n\t\t\tvid[1], encodeTk2(clit), pm2.Data.Atc.Pm2,\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar mgtvData mgtv\n\tif err = json.Unmarshal([]byte(dataString), &mgtvData); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := strings.TrimSpace(\n\t\tpm2.Data.Info.Title + \" \" + pm2.Data.Info.Desc,\n\t)\n\tmgtvStreams := mgtvData.Data.Stream\n\tvar addr mgtvVideoAddr\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, stream := range mgtvStreams {\n\t\tif stream.URL == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\t// real download address\n\t\taddr = mgtvVideoAddr{}\n\t\taddrInfo, err := request.GetByte(mgtvData.Data.StreamDomain[0]+stream.URL, url, headers)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif err = json.Unmarshal(addrInfo, &addr); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tm3u8URLs, totalSize, err := mgtvM3u8(addr.Info)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL: u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt: \"ts\",\n\t\t\t}\n\t\t}\n\t\tstreams[stream.Def] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize: totalSize,\n\t\t\tQuality: stream.Name,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"芒果TV mgtv.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/mgtv/mgtv_test.go", "content": "package mgtv\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.mgtv.com/b/322712/4317248.html\",\n\t\t\t\tTitle: \"我是大侦探 先导片：何炅吴磊邓伦穿越破案\",\n\t\t\t\tSize: 86169236,\n\t\t\t\tQuality: \"超清\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.mgtv.com/b/308703/4197072.html\",\n\t\t\t\tTitle: \"芒果捞星闻 2017 诺一为爷爷和姥爷做翻译超萌\",\n\t\t\t\tSize: 6486376,\n\t\t\t\tQuality: \"超清\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"vip test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.mgtv.com/b/322865/4352046.html\",\n\t\t\t\tTitle: \"向往的生活 第二季 先导片：何炅黄磊回归质朴生活\",\n\t\t\t\tSize: 453246944,\n\t\t\t\tQuality: \"超清\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n" }, { "path": "extractors/miaopai/miaopai.go", "content": "package miaopai\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"miaopai\", New())\n}\n\ntype miaopaiData struct {\n\tData struct {\n\t\tDescription string `json:\"description\"`\n\t\tMetaData []struct {\n\t\t\tURLs struct {\n\t\t\t\tM string `json:\"m\"`\n\t\t\t} `json:\"play_urls\"`\n\t\t} `json:\"meta_data\"`\n\t} `json:\"data\"`\n}\n\nfunc getRandomString(l int) string {\n\ts := make([]string, 0)\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tfor i := 0; i < l; i++ {\n\t\ts = append(s, chars[rand.Intn(len(chars)-1)])\n\t}\n\treturn strings.Join(s, \"\")\n}\n\ntype extractor struct{}\n\n// New returns a miaopai extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tids := utils.MatchOneOf(url, `/media/([^\\./]+)`, `/show(?:/channel)?/([^\\./]+)`)\n\tif ids == nil || len(ids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tid := ids[1]\n\n\trandomString := getRandomString(10)\n\n\tvar data miaopaiData\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"https://n.miaopai.com/api/aj_media/info.json?smid=%s&appid=530&_cb=_jsonp%s\", id, randomString),\n\t\turl, nil,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tmatch := utils.MatchOneOf(jsonString, randomString+`\\((.*)\\);$`)\n\tif match == nil || len(match) < 2 {\n\t\treturn nil, errors.New(\"获取视频信息失败。\")\n\t}\n\n\terr = json.Unmarshal([]byte(match[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURL := data.Data.MetaData[0].URLs.M\n\tsize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL: realURL,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"秒拍 miaopai.com\",\n\t\t\tTitle: data.Data.Description,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/miaopai/miaopai_test.go", "content": "package miaopai\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"http://n.miaopai.com/media/Dqg5Pmb~I6lChdvOb-~r1BpKzzDu~MPr\",\n\t\t\t\tTitle: \"小学霸6点半起床学习:想赢在起跑线\",\n\t\t\t\tSize: 6743958,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/netease/netease.go", "content": "package netease\n\nimport (\n\tnetURL \"net/url\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"163\", New())\n}\n\ntype extractor struct{}\n\n// New returns a netease extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\turl = strings.Replace(url, \"/#/\", \"/\", 1)\n\tvid := utils.MatchOneOf(url, `/(mv|video)\\?id=(\\w+)`)\n\tif vid == nil {\n\t\treturn nil, errors.New(\"invalid url for netease music\")\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif strings.Contains(html, \"u-errlg-404\") {\n\t\treturn nil, errors.New(\"404 music not found\")\n\t}\n\n\ttitles := utils.MatchOneOf(html, ``)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\trealURLs := utils.MatchOneOf(html, ``)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\trealURL, _ := netURL.QueryUnescape(realURLs[1])\n\n\tsize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL: realURL,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"网易云音乐 music.163.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/netease/netease_test.go", "content": "package netease\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"mv test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://music.163.com/#/mv?id=5547010\",\n\t\t\t\tTitle: \"There For You\",\n\t\t\t\tSize: 24249078,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://music.163.com/#/video?id=C8C9D11629798595BD28451DE3AC9FF4\",\n\t\t\t\tTitle: \"＃金曜日の新垣结衣 总集編〈全9編〉\",\n\t\t\t\tSize: 37408123,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/odysee/odysee.go", "content": "package odysee\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"net/http\"\n\t\"regexp\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc init() {\n\textractors.Register(\"odysee\", New())\n}\n\ntype extractor struct{}\n\ntype odyseePayload struct {\n\tContentURL string `json:\"contentUrl\"`\n\tDescription string `json:\"description\"`\n\tName string `json:\"name\"`\n\tThumbnailURL string `json:\"thumbnailUrl\"`\n\tURL string `json:\"url\"`\n}\n\n// New returns an odysee extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) {\n\tres, err := request.Request(http.MethodGet, u, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tregScript := regexp.MustCompile(`(?im)\\`)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrInvalidRegularExpression)\n\t}\n\n\tmatchers := reg.FindAllStringSubmatch(html, -1)\n\tvar encryptedScript string\n\n\tfor _, scripts := range matchers {\n\t\tscript := scripts[1]\n\t\tif !strings.Contains(script, \"flashvars_\") {\n\t\t\tcontinue\n\t\t} else {\n\t\t\tencryptedScript = script\n\t\t\tbreak\n\t\t}\n\t}\n\n\tflashId := regexp.MustCompile(`flashvars_\\d+`).FindString(encryptedScript)\n\n\tvm := otto.New()\n\t_, err = vm.Run(`var playerObjList = {};` + encryptedScript + fmt.Sprintf(`;var __VM__OUTPUT = JSON.stringify(%s.mediaDefinitions)`, flashId))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttype MediaDefinition struct {\n\t\tFormat string `json:\"format\"`\n\t\tVideoURL string `json:\"videoUrl\"`\n\t}\n\n\tmediaDefinitions := make([]MediaDefinition, 0)\n\n\tif str, err := value.ToString(); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t} else {\n\t\tif err := json.Unmarshal([]byte(str), &mediaDefinitions); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t}\n\n\tvar mp4MediaDefinition *MediaDefinition\n\n\tfor _, mediaDefinition := range mediaDefinitions {\n\t\tif mediaDefinition.Format == \"mp4\" {\n\t\t\tmp4MediaDefinition = &mediaDefinition\n\t\t}\n\t}\n\n\tif mp4MediaDefinition == nil {\n\t\treturn nil, errors.New(\"can not found media\")\n\t}\n\n\tresApi, err := request.Get(mp4MediaDefinition.VideoURL, mp4MediaDefinition.VideoURL, map[string]string{\n\t\t\"Cookie\": strings.Join(cookiesArr, \"; \"),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tpornhubs := make([]pornhubData, 0)\n\n\tif err := json.Unmarshal([]byte(resApi), &pornhubs); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(pornhubs))\n\n\tfor _, data := range pornhubs {\n\t\tsize, err := request.Size(data.VideoURL, data.VideoURL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL: data.VideoURL,\n\t\t\tSize: size,\n\t\t\tExt: data.Format,\n\t\t}\n\n\t\tstreams[data.Quality] = &extractors.Stream{\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize: size,\n\t\t\tQuality: data.Quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Pornhub pornhub.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/pornhub/pornhub_test.go", "content": "package pornhub\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestPornhub(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.pornhub.com/view_video.php?viewkey=ph5cb5fc41c6ebd\",\n\t\t\t\tTitle: \"Must watch Milf drilled by the fireplace\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n" }, { "path": "extractors/qq/qq.go", "content": "package qq\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"qq\", New())\n}\n\ntype qqVideoInfo struct {\n\tFl struct {\n\t\tFi []struct {\n\t\t\tID int `json:\"id\"`\n\t\t\tName string `json:\"name\"`\n\t\t\tCname string `json:\"cname\"`\n\t\t\tFs int64 `json:\"fs\"`\n\t\t} `json:\"fi\"`\n\t} `json:\"fl\"`\n\tVl struct {\n\t\tVi []struct {\n\t\t\tFn string `json:\"fn\"`\n\t\t\tTi string `json:\"ti\"`\n\t\t\tFvkey string `json:\"fvkey\"`\n\t\t\tCl struct {\n\t\t\t\tFc int `json:\"fc\"`\n\t\t\t\tCi []struct {\n\t\t\t\t\tIdx int `json:\"idx\"`\n\t\t\t\t} `json:\"ci\"`\n\t\t\t} `json:\"cl\"`\n\t\t\tUl struct {\n\t\t\t\tUI []struct {\n\t\t\t\t\tURL string `json:\"url\"`\n\t\t\t\t} `json:\"ui\"`\n\t\t\t} `json:\"ul\"`\n\t\t} `json:\"vi\"`\n\t} `json:\"vl\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype qqKeyInfo struct {\n\tKey string `json:\"key\"`\n}\n\nconst qqPlayerVersion string = \"3.2.19.333\"\n\nfunc getVinfo(vid, defn, refer string) (qqVideoInfo, error) {\n\thtml, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"http://vv.video.qq.com/getinfo?otype=json&platform=11&defnpayver=1&appver=%s&defn=%s&vid=%s\",\n\t\t\tqqPlayerVersion, defn, vid,\n\t\t), refer, nil,\n\t)\n\tif err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\tjsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn qqVideoInfo{}, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\tvar data qqVideoInfo\n\tif err = json.Unmarshal([]byte(jsonString), &data); err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\treturn data, nil\n}\n\nfunc genStreams(vid, cdn string, data qqVideoInfo) (map[string]*extractors.Stream, error) {\n\tstreams := make(map[string]*extractors.Stream)\n\tvar vkey string\n\t// number of fragments\n\tvar clips int\n\n\tfor _, fi := range data.Fl.Fi {\n\t\tvar fmtIDPrefix string\n\t\tvar fns []string\n\t\tif slices.Contains([]string{\"shd\", \"fhd\"}, fi.Name) {\n\t\t\tfmtIDPrefix = \"p\"\n\t\t\tfmtIDName := fmt.Sprintf(\"%s%d\", fmtIDPrefix, fi.ID%10000)\n\t\t\tfns = []string{strings.Split(data.Vl.Vi[0].Fn, \".\")[0], fmtIDName, \"mp4\"}\n\t\t\tif len(fns) > 3 {\n\t\t\t\t// delete ID part\n\t\t\t\t// e0765r4mwcr.2.mp4 -> e0765r4mwcr.mp4\n\t\t\t\tfns = append(fns[:1], fns[2:]...)\n\t\t\t}\n\t\t\tclips = data.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t} else {\n\t\t\ttmpData, err := getVinfo(vid, fi.Name, cdn)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tfns = strings.Split(tmpData.Vl.Vi[0].Fn, \".\")\n\t\t\tif len(fns) >= 3 && utils.MatchOneOf(fns[1], `^p(\\d{3})$`) != nil {\n\t\t\t\tfmtIDPrefix = \"p\"\n\t\t\t}\n\t\t\tclips = tmpData.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t}\n\n\t\tvar urls []*extractors.Part\n\t\tvar totalSize int64\n\t\tvar filename string\n\t\tfor part := 1; part < clips+1; part++ {\n\t\t\t// Multiple fragments per streams\n\t\t\tif fmtIDPrefix == \"p\" {\n\t\t\t\tif len(fns) < 4 {\n\t\t\t\t\t// If the number of fragments > 0, the filename needs to add the number of fragments\n\t\t\t\t\t// n0687peq62x.p709.mp4 -> n0687peq62x.p709.1.mp4\n\t\t\t\t\tfns = append(fns[:2], append([]string{strconv.Itoa(part)}, fns[2:]...)...)\n\t\t\t\t} else {\n\t\t\t\t\tfns[2] = strconv.Itoa(part)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfilename = strings.Join(fns, \".\")\n\t\t\thtml, err := request.Get(\n\t\t\t\tfmt.Sprintf(\n\t\t\t\t\t\"http://vv.video.qq.com/getkey?otype=json&platform=11&appver=%s&filename=%s&format=%d&vid=%s\",\n\t\t\t\t\tqqPlayerVersion, filename, fi.ID, vid,\n\t\t\t\t), \"\", nil,\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tjsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`)\n\t\t\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t\t}\n\t\t\tjsonString := jsonStrings[1]\n\n\t\t\tvar keyData qqKeyInfo\n\t\t\tif err = json.Unmarshal([]byte(jsonString), &keyData); err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\n\t\t\tvkey = keyData.Key\n\t\t\tif vkey == \"\" {\n\t\t\t\tvkey = data.Vl.Vi[0].Fvkey\n\t\t\t}\n\t\t\trealURL := fmt.Sprintf(\"%s%s?vkey=%s\", cdn, filename, vkey)\n\t\t\tsize, err := request.Size(realURL, cdn)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turlData := &extractors.Part{\n\t\t\t\tURL: realURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt: \"mp4\",\n\t\t\t}\n\t\t\turls = append(urls, urlData)\n\t\t\ttotalSize += size\n\t\t}\n\t\tstreams[fi.Name] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize: totalSize,\n\t\t\tQuality: fi.Cname,\n\t\t}\n\t}\n\treturn streams, nil\n}\n\ntype extractor struct{}\n\n// New returns a qq extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvids := utils.MatchOneOf(url, `vid=(\\w+)`, `/(\\w+)\\.html`)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tif len(vid) != 11 {\n\t\tu, err := request.Get(url, url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tvids = utils.MatchOneOf(\n\t\t\tu, `vid=(\\w+)`, `vid:\\s*[\"'](\\w+)`, `vid\\s*=\\s*[\"']\\s*(\\w+)`,\n\t\t)\n\t\tif vids == nil || len(vids) < 2 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\t\tvid = vids[1]\n\t}\n\n\tdata, err := getVinfo(vid, \"shd\", url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// API request error\n\tif data.Msg != \"\" {\n\t\treturn nil, errors.New(data.Msg)\n\t}\n\tcdn := data.Vl.Vi[0].Ul.UI[0].URL\n\tstreams, err := genStreams(vid, cdn, data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"腾讯视频 v.qq.com\",\n\t\t\tTitle: data.Vl.Vi[0].Ti,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/qq/qq_test.go", "content": "package qq\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://v.qq.com/x/page/n0687peq62x.html\",\n\t\t\t\tTitle: \"世界杯第一期：100秒速成！“伪球迷”世界杯生存指南\",\n\t\t\t\tSize: 23759683,\n\t\t\t\tQuality: \"蓝光;(1080P)\",\n\t\t\t},\n\t\t},\n\t\t// {\n\t\t// \tname: \"movie and vid test\",\n\t\t// \targs: test.Args{\n\t\t// \t\tURL: \"https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html\",\n\t\t// \t\tTitle: \"赌侠（粤语版）\",\n\t\t// \t\tSize: 1046910811,\n\t\t// \t\tQuality: \"超清;(720P)\",\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tname: \"fmt ID test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html\",\n\t\t\t\tTitle: \"《卡路里》出圈！妖娆男子教学广场舞版，大妈表情亮了！\",\n\t\t\t\tSize: 14112979,\n\t\t\t\tQuality: \"超清;(720P)\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/reddit/reddit.go", "content": "package reddit\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"reddit\", New())\n}\n\nconst (\n\treferer = \"https://www.reddit.com\"\n\tsiteName = \"Reddit reddit.com\"\n\n\tredditMP4API = \"https://v.redd.it/\"\n\tredditIMGAPI = \"https://i.redd.it/\"\n\taudioURLPart = \"/DASH_audio.mp4\"\n)\n\nvar resMap = map[string]string{\n\t\"720p\": \"/DASH_720.mp4\",\n\t\"480p\": \"/DASH_480.mp4\",\n\t\"360p\": \"/DASH_360.mp4\",\n\t\"240p\": \"/DASH_240.mp4\",\n\t\"220p\": \"/DASH_220.mp4\",\n}\n\ntype extractor struct{}\n\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\ttitle := utils.MatchOneOf(html, `(.+?)<\\/title>`)[1]\n\n\tif utils.MatchOneOf(html, `meta property=\"og:video\" content=.*HLSPlaylist`) != nil {\n\t\tmp4URL := utils.MatchOneOf(html, `https://v.redd.it/(.+?)/HLSPlaylist`)[1]\n\t\tif mp4URL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match mp4 content downloadable url\")\n\t\t}\n\n\t\taudioURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, audioURLPart)\n\t\tsize, err := request.Size(audioURL, referer)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\taudioPart := &extractors.Part{\n\t\t\tURL: audioURL,\n\t\t\tSize: size,\n\t\t\tExt: \"mp3\",\n\t\t}\n\n\t\tstreams := make(map[string]*extractors.Stream, len(resMap))\n\t\tfor res, urlParts := range resMap {\n\t\t\tresURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, urlParts)\n\t\t\tsize, err := request.Size(resURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tstreams[res] = &extractors.Stream{\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL: resURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t\taudioPart,\n\t\t\t\t},\n\t\t\t\tSize: size + audioPart.Size,\n\t\t\t\tQuality: res,\n\t\t\t\tNeedMux: true,\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite: siteName,\n\t\t\t\tTitle: title,\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL: url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `<meta property=\"og:type\" content=\"image\"/>`) != nil {\n\t\tvar imgURL string\n\t\tvar size int64\n\t\tif utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`) != nil {\n\t\t\timgURL = redditIMGAPI + utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`)[1]\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t} else {\n\t\t\timgURL = utils.MatchOneOf(html, `content\":\"(.+?)\",\"type\":\"image\"`)[1]\n\t\t\timgURL = strings.ReplaceAll(imgURL, \"auto=webp\\\\u0026s\", \"auto=webp&s\")\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite: siteName,\n\t\t\t\tTitle: title,\n\t\t\t\tType: extractors.DataTypeImage,\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"default\": {\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL: imgURL,\n\t\t\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\t\t\tExt: \"jpg\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tURL: url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*gif`) != nil {\n\t\tgifURL := utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*?\\.gif\\?format=mp4.*?\"`)[0]\n\t\tif gifURL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match gif content downloadable url\")\n\t\t}\n\n\t\tgifURL = strings.ReplaceAll(gifURL, \"&\", \"&\")\n\t\tgifURL = strings.ReplaceAll(gifURL, \"\\\"\", \"\")\n\n\t\tsize, err := request.Size(gifURL, \"reddit.com\")\n\t\tif err != nil {\n\t\t\treturn nil, errors.New(\"can't get video size\")\n\t\t}\n\n\t\tstreams := map[string]*extractors.Stream{\n\t\t\t\"default\": {\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL: gifURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSize: size,\n\t\t\t},\n\t\t}\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite: siteName,\n\t\t\t\tTitle: title,\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL: url,\n\t\t\t},\n\t\t}, nil\n\t}\n\n\treturn nil, fmt.Errorf(\"unable to handle url: %s\", url)\n}\n" }, { "path": "extractors/reddit/reddit_test.go", "content": "package reddit\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestReddit(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 0\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/space/comments/uj8sod/a_couple_of_days_ago_i_visited_this_place_an/\",\n\t\t\t\tTitle: \"A couple of days ago I visited this place. An abandoned space shuttle : space\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/DotA2/comments/uq012r/til_how_useful_hurricane_bird_is/\",\n\t\t\t\tTitle: \"TIL how useful hurricane bird is : DotA2\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/ProgrammerHumor/comments/uqovco/my_code_works/\",\n\t\t\t\tTitle: \"My code works : ProgrammerHumor\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/AnimatedPixelArt/comments/uomu32/animation_for_astral_ascent/\",\n\t\t\t\tTitle: \"Animation for Astral Ascent : AnimatedPixelArt\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 4\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/linuxmemes/comments/v1a4wh/please_olive_do_something/\",\n\t\t\t\tTitle: \"Please Olive, do something... : linuxmemes\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 5\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.reddit.com/r/gaming/comments/v27m79/skyrim_probably/\",\n\t\t\t\tTitle: \"Skyrim, probably : gaming\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/rumble/rumble.go", "content": "package rumble\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"rumble\", New())\n}\n\ntype extractor struct{}\n\n// New returns a rumble extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\ntype rumbleData struct {\n\tFormat string `json:\"format\"`\n\tName string `json:\"name\"`\n\tEmbedURL string `json:\"embedUrl\"`\n\tThumbnailURL string `json:\"thumbnailUrl\"`\n\tType string `json:\"@type\"`\n\tVideoURL string `json:\"videoUrl\"`\n\tQuality string `json:\"quality\"`\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\thtml := string(b)\n\tvar title string\n\tmatchTitle := utils.MatchOneOf(html, `<title>(.+?)`)\n\tif len(matchTitle) > 1 {\n\t\ttitle = matchTitle[1]\n\t} else {\n\t\ttitle = \"rumble video\"\n\t}\n\n\tpayload, err := readPayload(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoID, err := getVideoID(payload.EmbedURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams, err := fetchVideoQuality(videoID)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Rumble rumble.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n\n// Read JSON object from the video webpage\nfunc readPayload(html string) (*rumbleData, error) {\n\tmatchPayload := utils.MatchOneOf(html, `\\(.+?)\\<\\/script>`)\n\tif len(matchPayload) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\n\trumbles := make([]rumbleData, 0)\n\tif err := json.Unmarshal([]byte(matchPayload[1]), &rumbles); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor _, it := range rumbles {\n\t\tif it.Type == \"VideoObject\" {\n\t\t\treturn &it, nil\n\t\t}\n\t}\n\n\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n}\n\nfunc getVideoID(embedURL string) (string, error) {\n\tu, err := url.Parse(embedURL)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\treturn path.Base(u.Path), nil\n}\n\n// Rumble response contains the streams in `rumbleStreams`\ntype rumbleResponse struct {\n\tStreams *json.RawMessage `json:\"ua\"`\n}\n\n// Common video meta data\ntype streamInfo struct {\n\tURL string `json:\"url\"`\n\tMeta struct {\n\t\tBitrate uint16 `json:\"bitrate\"`\n\t\tSize int64 `json:\"size\"`\n\t\tWidth uint16 `json:\"w\"`\n\t\tHeight uint16 `json:\"h\"`\n\t} `json:\"meta\"`\n}\n\n// common video qualities for `mp4`, `webm`\ntype videoQualities struct {\n\tQ240 struct{ streamInfo } `json:\"240\"`\n\tQ360 struct{ streamInfo } `json:\"360\"`\n\tQ480 struct{ streamInfo } `json:\"480\"`\n\tQ720 struct{ streamInfo } `json:\"720\"`\n\tQ1080 struct{ streamInfo } `json:\"1080\"`\n\tQ1440 struct{ streamInfo } `json:\"1440\"`\n\tQ2160 struct{ streamInfo } `json:\"2160\"`\n\tQ2161 struct{ streamInfo } `json:\"2161\"`\n}\n\n// Video payload for adaptive stream and different qualities\ntype rumbleStreams struct {\n\tFMp4 struct {\n\t\tvideoQualities\n\t} `json:\"mp4\"`\n\tFWebm struct {\n\t\tvideoQualities\n\t} `json:\"webm\"`\n\tFHLS struct {\n\t\tQAuto struct{ streamInfo } `json:\"auto\"`\n\t} `json:\"hls\"`\n\tFTAR map[string]streamInfo `json:\"tar\"`\n}\n\n// Unmarshall the video response\n// Some properties like `mp4`, `webm` are either array or an object\nfunc (r *rumbleStreams) UnmarshalJSON(b []byte) error {\n\tvar resp *rumbleResponse\n\tif err := json.Unmarshal(b, &resp); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get individual stream from the response\n\tvar obj map[string]*json.RawMessage\n\tif err := json.Unmarshal(*resp.Streams, &obj); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tif v, ok := obj[\"mp4\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FMp4)\n\t}\n\tif v, ok := obj[\"webm\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FWebm)\n\t}\n\tif v, ok := obj[\"hls\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FHLS)\n\t}\n\tif v, ok := obj[\"tar\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FTAR)\n\t}\n\treturn nil\n}\n\n// Use this to create all the streams for `mp4`, `webm`\nfunc (rs *rumbleStreams) makeAllVODStreams(m map[string]*extractors.Stream) {\n\tm[\"webm\"] = makeStreamMeta(\"480\", \"webm\", &rs.FWebm.Q480.streamInfo)\n\tm[\"240\"] = makeStreamMeta(\"240\", \"mp4\", &rs.FMp4.Q240.streamInfo)\n\tm[\"360\"] = makeStreamMeta(\"360\", \"mp4\", &rs.FMp4.Q360.streamInfo)\n\tm[\"480\"] = makeStreamMeta(\"480\", \"mp4\", &rs.FMp4.Q480.streamInfo)\n\tm[\"720\"] = makeStreamMeta(\"720\", \"mp4\", &rs.FMp4.Q720.streamInfo)\n\tm[\"1080\"] = makeStreamMeta(\"1080\", \"mp4\", &rs.FMp4.Q1080.streamInfo)\n\tm[\"1440\"] = makeStreamMeta(\"1440\", \"mp4\", &rs.FMp4.Q1440.streamInfo)\n\tm[\"2160\"] = makeStreamMeta(\"2160\", \"mp4\", &rs.FMp4.Q2160.streamInfo)\n\tm[\"2161\"] = makeStreamMeta(\"2161\", \"mp4\", &rs.FMp4.Q2161.streamInfo)\n}\n\nvar reResolution = regexp.MustCompile(`_(\\d{3,4})p\\/`) // ex. _720p/\n\n// Use this to create all the streams for live videos\nfunc (rs *rumbleStreams) makeAllLiveStreams(m map[string]*extractors.Stream) error {\n\tplaylists, err := utils.M3u8URLs(rs.FHLS.QAuto.URL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tif len(playlists) == 0 {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Find the highest resolution\n\tplaylistURL := playlists[0]\n\tmaxRes := 0\n\tfor _, x := range playlists {\n\t\tmatched := reResolution.FindStringSubmatch(x)\n\t\tif len(matched) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tres, err := strconv.Atoi(matched[1])\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif maxRes < res {\n\t\t\tmaxRes = res\n\t\t\tplaylistURL = x\n\t\t}\n\t}\n\n\ttsURLs, err := utils.M3u8URLs(playlistURL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tvar parts []*extractors.Part\n\tfor _, x := range tsURLs {\n\t\tpart := &extractors.Part{\n\t\t\tURL: x,\n\t\t\tSize: rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\t\tExt: \"ts\",\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tm[\"hls\"] = &extractors.Stream{\n\t\tParts: parts,\n\t\tSize: rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\tQuality: strconv.Itoa(maxRes),\n\t}\n\n\treturn nil\n}\n\nfunc (rs *rumbleStreams) makeAllNewVodStreams(m map[string]*extractors.Stream) error {\n\tfor size, details := range rs.FTAR {\n\t\tplaylists, err := utils.M3u8URLs(details.URL)\n\t\tif err != nil {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\tif len(playlists) == 0 {\n\t\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tvar parts []*extractors.Part\n\t\tfor _, x := range playlists {\n\t\t\tpart := &extractors.Part{\n\t\t\t\tURL: x,\n\t\t\t\tSize: details.Meta.Size,\n\t\t\t\tExt: \"ts\",\n\t\t\t}\n\t\t\tparts = append(parts, part)\n\t\t}\n\n\t\tm[size] = &extractors.Stream{\n\t\t\tParts: parts,\n\t\t\tSize: details.Meta.Size,\n\t\t\tQuality: strconv.Itoa(int(details.Meta.Height)),\n\t\t}\n\t}\n\treturn nil\n}\n\n// Request video formats and qualities\nfunc fetchVideoQuality(videoID string) (map[string]*extractors.Stream, error) {\n\treqURL := fmt.Sprintf(`https://rumble.com/embedJS/u3/?request=video&ver=2&v=%s&ext={\"ad_count\":null}&ad_wt=0`, videoID)\n\n\tres, err := request.Request(http.MethodGet, reqURL, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar rs rumbleStreams\n\tif err := json.Unmarshal(b, &rs); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 9)\n\trs.makeAllVODStreams(streams)\n\t_ = rs.makeAllLiveStreams(streams)\n\t_ = rs.makeAllNewVodStreams(streams)\n\treturn streams, nil\n}\n\nfunc makeStreamMeta(q, ext string, info *streamInfo) *extractors.Stream {\n\turlMeta := &extractors.Part{\n\t\tURL: info.URL,\n\t\tSize: info.Meta.Size,\n\t\tExt: ext,\n\t}\n\n\treturn &extractors.Stream{\n\t\tParts: []*extractors.Part{urlMeta},\n\t\tSize: info.Meta.Size,\n\t\tQuality: q,\n\t}\n}\n" }, { "path": "extractors/rumble/rumble_test.go", "content": "package rumble\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestRumble(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html\",\n\t\t\t\tTitle: \"Just Say YES to Climate Lockdowns!\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://rumble.com/v6rmfm1-monday-full-show-33125-hhs-head-rfk-jr.-pledges-to-stopv.html\",\n\t\t\t\tTitle: \"MONDAY FULL SHOW 3/31/25 — HHS Head RFK Jr. Pledges To Stopv\",\n\t\t\t},\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tfor _, d := range data {\n\t\t\t\tfound := false\n\t\t\t\tfor _, s := range d.Streams {\n\t\t\t\t\tif s.Size > 0 {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tt.Errorf(\"no streams found in test %d\", i)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "extractors/streamtape/streamtape.go", "content": "package streamtape\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/robertkrimen/otto\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"streamtape\", e)\n\textractors.Register(\"streamta\", e) // streamta.pe\n}\n\ntype extractor struct{}\n\n// New returns a StreamTape extractor\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tscripts := utils.MatchOneOf(html, `document.getElementById\\('norobotlink'\\).innerHTML = (.+?);`)\n\tif len(scripts) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvm := otto.New()\n\t_, err = vm.Run(fmt.Sprintf(\"var __VM__OUTPUT = %s\", scripts[1]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu, err := value.ToString() // //streamtape.com/get_video?id=xx&expires=xx&ip=xx&token=xx\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu = fmt.Sprintf(\"https:%s&stream=1\", u)\n\n\t// get title\n\tvar title = \"StreamTape Video\"\n\ttitleMatch := utils.MatchOneOf(html,\n\t\t`\\`)\n\tif len(titleMatch) >= 2 {\n\t\ttitle = titleMatch[1]\n\t}\n\n\tsize, err := request.Size(u, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := &extractors.Part{\n\t\tURL: u,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize: size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tURL: u,\n\t\t\tSite: \"StreamTape streamtape.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/streamtape/streamtape_test.go", "content": "package streamtape\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestStreamtape(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://streamtape.com/v/YKLDrr4X9gSvm9q/00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t\tTitle: \"00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/tangdou/tangdou.go", "content": "package tangdou\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tangdou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tangdou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nvar defaultHeader = map[string]string{\n\t\"Sec-Fetch-Dest\": \"document\",\n\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\"Sec-Fetch-Site\": \"cross-site\",\n\t\"Sec-GPC\": \"1\",\n\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\treturn []*extractors.Data{tangdouDownload(url)}, nil\n}\n\n// tangdouDownload download function for single url\nfunc tangdouDownload(uri string) *extractors.Data {\n\thtml, err := request.Get(uri, uri, defaultHeader)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\ttitles := utils.MatchOneOf(\n\t\thtml, `
(.+?)
`, `(.+?)`,\n\t)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\ttitle := titles[1]\n\n\tvideoURLs := utils.MatchOneOf(\n\t\thtml, `video:'(.+?)'`, `video:\"(.+?)\"`, `]*src=\"(.+?)\"`, `play_url:\\s*\"(.+?)\",`,\n\t)\n\n\tif len(videoURLs) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\n\trealURL := strings.ReplaceAll(videoURLs[1], `\\u002F`, \"/\")\n\n\tsize, err := request.Size(realURL, uri)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL: realURL,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt: \"mp4\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn &extractors.Data{\n\t\tSite: \"糖豆广场舞 tangdou.com\",\n\t\tTitle: title,\n\t\tType: extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tURL: uri,\n\t}\n}\n" }, { "path": "extractors/tangdou/tangdou_test.go", "content": "package tangdou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestTangDou(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"need call share url first and get the signed video URL test and can get title from head's title tag\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://m.tangdou.com/play/1500676338077\",\n\t\t\t\tTitle: \"暴瘦减肚子，不用跑不用跳，8天瘦了16斤 正面演示 背面演示 分解教学__广场舞_糖豆广场舞-糖豆视频\",\n\t\t\t\tSize: 62258444,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// playlist mode\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist: true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "extractors/threads/threads.go", "content": "package threads\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"threads\", New())\n}\n\ntype extractor struct {\n\tclient *http.Client\n}\n\n// New returns a instagram extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{\n\t\tclient: &http.Client{\n\t\t\tTimeout: 10 * time.Second,\n\t\t\tTransport: &http.Transport{\n\t\t\t\tDial: (&net.Dialer{\n\t\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t\t}).Dial,\n\t\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t\t},\n\t\t},\n\t}\n}\n\ntype media struct {\n\tURL string\n\tType extractors.DataType\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tURL, err := netURL.Parse(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tpaths := strings.Split(URL.Path, \"/\")\n\tif len(paths) < 3 {\n\t\treturn nil, errors.New(\"invalid URL format\")\n\t}\n\n\tposter := paths[1]\n\tshortCode := paths[3]\n\n\tmedias := make([]media, 0)\n\n\ttitle := fmt.Sprintf(\"Threads %s - %s\", poster, shortCode)\n\n\tcollector := colly.NewCollector()\n\tcollector.SetClient(e.client)\n\n\t// case single image or video\n\tcollector.OnHTML(\"div.SingleInnerMediaContainer\", func(e *colly.HTMLElement) {\n\t\tif src := e.ChildAttr(\"img\", \"src\"); src != \"\" {\n\t\t\tmedias = append(medias, media{\n\t\t\t\tURL: src,\n\t\t\t\tType: extractors.DataTypeImage,\n\t\t\t})\n\t\t}\n\t\tif src := e.ChildAttr(\"video > source\", \"src\"); src != \"\" {\n\t\t\tmedias = append(medias, media{\n\t\t\t\tURL: src,\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t})\n\t\t}\n\t})\n\n\t// case multiple image or video\n\tcollector.OnHTML(\"div.MediaScrollImageContainer\", func(e *colly.HTMLElement) {\n\t\tif src := e.ChildAttr(\"img\", \"src\"); src != \"\" {\n\t\t\tmedias = append(medias, media{\n\t\t\t\tURL: src,\n\t\t\t\tType: extractors.DataTypeImage,\n\t\t\t})\n\t\t}\n\t\tif src := e.ChildAttr(\"video > source\", \"src\"); src != \"\" {\n\t\t\tmedias = append(medias, media{\n\t\t\t\tURL: src,\n\t\t\t\tType: extractors.DataTypeVideo,\n\t\t\t})\n\t\t}\n\t})\n\n\t// title with caption\n\t// collector.OnHTML(\"span.BodyTextContainer\", func(e *colly.HTMLElement) {\n\t// \ttitle = e.Text\n\t// })\n\n\tif err := collector.Visit(URL.JoinPath(\"embed\").String()); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to send HTTP request to the Threads: %w\", errors.WithStack(err))\n\t}\n\n\tvar totalSize int64\n\tvar parts []*extractors.Part\n\n\tfor _, m := range medias {\n\t\t_, ext, err := utils.GetNameAndExt(m.URL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfileSize, err := request.Size(m.URL, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tpart := &extractors.Part{\n\t\t\tURL: m.URL,\n\t\t\tSize: fileSize,\n\t\t\tExt: ext,\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tfor _, part := range parts {\n\t\ttotalSize += part.Size\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Threads www.threads.net\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/threads/threads_test.go", "content": "package threads_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/extractors/threads\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.threads.net/@rowancheung/post/C9xPmHcpfiN\",\n\t\t\t\tTitle: `Threads @rowancheung - C9xPmHcpfiN`,\n\t\t\t\tSize: 5740684,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video shared test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.threads.net/@zuck/post/C9xRqbNPbx2\",\n\t\t\t\tTitle: `Threads @zuck - C9xRqbNPbx2`,\n\t\t\t\tSize: 5740684,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.threads.net/@zuck/post/C-BoS7lM8sH\",\n\t\t\t\tTitle: `Threads @zuck - C-BoS7lM8sH`,\n\t\t\t\tSize: 159331,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"hybrid album test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.threads.net/@meta/post/C95Z1DrPNhi\",\n\t\t\t\tTitle: `Threads @meta - C95Z1DrPNhi`,\n\t\t\t\tSize: 1131229,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := threads.New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/tiktok/tiktok.go", "content": "package tiktok\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"tiktok\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tiktok extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, map[string]string{\n\t\t// tiktok require a user agent\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlMatcherRegExp := regexp.MustCompile(`\"downloadAddr\":\\s*\"([^\"]+)\"`)\n\n\tdownloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(downloadURLMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvideoURL := strings.ReplaceAll(downloadURLMatcher[1], `\\u002F`, \"/\")\n\n\ttitleMatcherRegExp := regexp.MustCompile(`]*>([^<]+)`)\n\n\ttitleMatcherRegExpOpt := regexp.MustCompile(`\"desc\":\"([^\"]*)\"`)\n\n\ttitleMatcher := titleMatcherRegExp.FindStringSubmatch(html)\n\n\ttitleMatcherOpt := titleMatcherRegExpOpt.FindStringSubmatch(html)\n\n\tif len(titleMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := titleMatcher[1]\n\n\tif title == \"TikTok - Make Your Day\" {\n\t\tif len(titleMatcherOpt[1]) > 64 {\n\t\t\tcutoff := titleMatcherOpt[1][:64]\n\t\t\tlastSpace := strings.LastIndex(cutoff, \" \")\n\t\t\ttitle = titleMatcherOpt[1][:lastSpace]\n\t\t} else {\n\t\t\ttitle = titleMatcherOpt[1]\n\t\t}\n\t}\n\n\ttitleArr := strings.Split(title, \"|\")\n\n\tif len(titleArr) == 1 {\n\t\ttitle = titleArr[0]\n\t} else {\n\t\ttitle = strings.TrimSpace(strings.Join(titleArr[:len(titleArr)-1], \"|\"))\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\n\tsize, err := request.Size(videoURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL: videoURL,\n\t\tSize: size,\n\t\tExt: \"mp4\",\n\t}\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize: size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"TikTok tiktok.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n" }, { "path": "extractors/tiktok/tiktok_test.go", "content": "package tiktok\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.tiktok.com/@ginjiro_koyama/video/7164293510617763073?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"イケすぎたXOXO#xoxo #repezenfoxx #背中男 #kfam #yoshikiさんを泣かせたチーム @K fam @【Repezen Foxx】🦊\",\n\t\t\t\tSize: 4356253,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL: \"https://www.tiktok.com/@enhypen/video/7165445991238356225?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"깜짝 퇴장 👋 #ENHYPEN #SUNGHOON #NI_KI #Make_the_change\",\n\t\t\t\tSize: 3848307,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n" }, { "path": "extractors/tumblr/tumblr.go", "content": "package tumblr\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tumblr\", New())\n}\n\ntype imageList struct {\n\tList []string `json:\"@list\"`\n}\n\ntype tumblrImageList struct {\n\tImage imageList `json:\"image\"`\n}\n\ntype tumblrImage struct {\n\tImage string `json:\"image\"`\n}\n\nfunc genURLData(url, referer string) (*extractors.Part, int64, error) {\n\tsize, err := request.Size(url, referer)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\t_, ext, err := utils.GetNameAndExt(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\treturn &extractors.Part{\n\t\tURL: url,\n\t\tSize: size,\n\t\tExt: ext,\n\t}, size, nil\n}\n\nfunc tumblrImageDownload(url, html, title string) ([]*extractors.Data, error) {\n\tjsonStrings := utils.MatchOneOf(\n\t\thtml, ``,\n\t)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\n\tvar totalSize int64\n\turls := make([]*extractors.Part, 0, 1)\n\tif strings.Contains(jsonString, `\"image\":{\"@list\"`) {\n\t\t// there are two data structures in the same field(image)\n\t\tvar imageList tumblrImageList\n\t\tif err := json.Unmarshal([]byte(jsonString), &imageList); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfor _, u := range imageList.Image.List {\n\t\t\turlData, size, err := genURLData(u, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\turls = append(urls, urlData)\n\t\t}\n\t} else {\n\t\tvar image tumblrImage\n\t\tif err := json.Unmarshal([]byte(jsonString), &image); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData, size, err := genURLData(image.Image, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\ttotalSize = size\n\t\turls = append(urls, urlData)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize: totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite: \"Tumblr tumblr.com\",\n\t\t\tTitle: title,\n\t\t\tType: extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL: url,\n\t\t},\n\t}, nil\n}\n\nfunc tumblrVideoDownload(url, html, title string) ([]*extractors.Data, error) {\n\tvideoURLs := utils.MatchOneOf(html, `