Repository: iawia002/lux Branch: master Commit: dd00f6d258d8 Files: 186 Total size: 449.2 KB Directory structure: gitextract_1wxrt7hr/ ├── .gitattributes ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── can-not-download-video.md │ │ └── support-a-new-website.md │ └── workflows/ │ ├── builder.yml │ ├── ci.yml │ ├── goreleaser.yml │ ├── stream_acfun.yml │ ├── stream_bcy.yml │ ├── stream_bilibili.yml │ ├── stream_bitchute.yml │ ├── stream_douyin.yml │ ├── stream_douyu.yml │ ├── stream_eporner.yml │ ├── stream_facebook.yml │ ├── stream_geekbang.yml │ ├── stream_haokan.yml │ ├── stream_hupu.yml │ ├── stream_huya.yml │ ├── stream_instagram.yml │ ├── stream_iqiyi.yml │ ├── stream_ixigua.yml │ ├── stream_kuaishou.yml │ ├── stream_mgtv.yml │ ├── stream_miaopai.yml │ ├── stream_netease.yml │ ├── stream_odysee.yml │ ├── stream_pinterest.yml │ ├── stream_pixivision.yml │ ├── stream_pornhub.yml │ ├── stream_qq.yml │ ├── stream_reddit.yml │ ├── stream_rumble.yml │ ├── stream_streamtape.yml │ ├── stream_tangdou.yml │ ├── stream_threads.yml │ ├── stream_tiktok.yml │ ├── stream_tumblr.yml │ ├── stream_twitter.yml │ ├── stream_udn.yml │ ├── stream_vimeo.yml │ ├── stream_vk.yml │ ├── stream_weibo.yml │ ├── stream_xiaohongshu.yml │ ├── stream_ximalaya.yml │ ├── stream_xinpianchang.yml │ ├── stream_xvideos.yml │ ├── stream_yinyuetai.yml │ ├── stream_youku.yml │ ├── stream_youtube.yml │ ├── stream_zhihu.yml │ └── stream_zingmp3.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CONTRIBUTING.md ├── Cask.toml ├── LICENSE ├── README.md ├── app/ │ ├── app.go │ └── register.go ├── codecov.yml ├── config/ │ └── config.go ├── downloader/ │ ├── downloader.go │ ├── downloader_test.go │ ├── types.go │ └── utils.go ├── extractors/ │ ├── acfun/ │ │ ├── acfun.go │ │ ├── acfun_test.go │ │ └── types.go │ ├── bcy/ │ │ ├── bcy.go │ │ └── bcy_test.go │ ├── bilibili/ │ │ ├── bilibili.go │ │ ├── bilibili_test.go │ │ └── types.go │ ├── bitchute/ │ │ ├── bitchute.go │ │ └── bitchute_test.go │ ├── douyin/ │ │ ├── douyin.go │ │ ├── douyin_test.go │ │ ├── sign.js │ │ └── types.go │ ├── douyu/ │ │ ├── douyu.go │ │ └── douyu_test.go │ ├── eporner/ │ │ ├── eporner.go │ │ └── eporner_test.go │ ├── errors.go │ ├── extractors.go │ ├── facebook/ │ │ ├── facebook.go │ │ └── facebook_test.go │ ├── geekbang/ │ │ ├── geekbang.go │ │ └── geekbang_test.go │ ├── haokan/ │ │ ├── haokan.go │ │ └── haokan_test.go │ ├── hupu/ │ │ ├── hupu.go │ │ └── hupu_test.go │ ├── huya/ │ │ ├── huya.go │ │ └── huya_test.go │ ├── instagram/ │ │ ├── instagram.go │ │ └── instagram_test.go │ ├── iqiyi/ │ │ ├── iqiyi.go │ │ └── iqiyi_test.go │ ├── ixigua/ │ │ ├── ixigua.go │ │ ├── ixigua_test.go │ │ └── types.go │ ├── kuaishou/ │ │ ├── kuaishou.go │ │ └── kuaishou_test.go │ ├── mgtv/ │ │ ├── mgtv.go │ │ └── mgtv_test.go │ ├── miaopai/ │ │ ├── miaopai.go │ │ └── miaopai_test.go │ ├── netease/ │ │ ├── netease.go │ │ └── netease_test.go │ ├── odysee/ │ │ ├── odysee.go │ │ └── odysee_test.go │ ├── pinterest/ │ │ ├── pinterest.go │ │ └── pinterest_test.go │ ├── pixivision/ │ │ ├── pixivision.go │ │ └── pixivision_test.go │ ├── pornhub/ │ │ ├── pornhub.go │ │ └── pornhub_test.go │ ├── qq/ │ │ ├── qq.go │ │ └── qq_test.go │ ├── reddit/ │ │ ├── reddit.go │ │ └── reddit_test.go │ ├── rumble/ │ │ ├── rumble.go │ │ └── rumble_test.go │ ├── streamtape/ │ │ ├── streamtape.go │ │ └── streamtape_test.go │ ├── tangdou/ │ │ ├── tangdou.go │ │ └── tangdou_test.go │ ├── threads/ │ │ ├── threads.go │ │ └── threads_test.go │ ├── tiktok/ │ │ ├── tiktok.go │ │ └── tiktok_test.go │ ├── tumblr/ │ │ ├── tumblr.go │ │ └── tumblr_test.go │ ├── twitter/ │ │ ├── twitter.go │ │ └── twitter_test.go │ ├── types.go │ ├── udn/ │ │ ├── udn.go │ │ └── udn_test.go │ ├── universal/ │ │ ├── universal.go │ │ └── universal_test.go │ ├── vimeo/ │ │ ├── vimeo.go │ │ └── vimeo_test.go │ ├── vk/ │ │ ├── vk.go │ │ └── vk_test.go │ ├── weibo/ │ │ ├── weibo.go │ │ └── weibo_test.go │ ├── xiaohongshu/ │ │ ├── xiaohongshu.go │ │ └── xiaohongshu_test.go │ ├── ximalaya/ │ │ ├── types.go │ │ ├── ximalaya.go │ │ └── ximalaya_test.go │ ├── xinpianchang/ │ │ ├── xinpianchang.go │ │ └── xinpianchang_test.go │ ├── xvideos/ │ │ ├── xvideos.go │ │ └── xvideos_test.go │ ├── yinyuetai/ │ │ ├── types.go │ │ ├── yinyuetai.go │ │ └── yinyuetai_test.go │ ├── youku/ │ │ ├── youku.go │ │ └── youku_test.go │ ├── youtube/ │ │ ├── youtube.go │ │ └── youtube_test.go │ ├── zhihu/ │ │ ├── types.go │ │ ├── zhihu.go │ │ └── zhihu_test.go │ └── zingmp3/ │ ├── zingmp3.go │ └── zingmp3_test.go ├── go.mod ├── go.sum ├── main.go ├── parser/ │ ├── parser.go │ └── parser_test.go ├── request/ │ ├── request.go │ └── request_test.go ├── script/ │ ├── generate_github_action_template.js │ └── github_action_template.yml ├── test/ │ └── utils.go └── utils/ ├── download.go ├── download_test.go ├── ffmpeg.go ├── pool.go ├── pool_test.go ├── utils.go └── utils_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ *.go text eol=lf *.md text eol=lf ================================================ FILE: .github/ISSUE_TEMPLATE/can-not-download-video.md ================================================ --- name: Can not download video about: If you find that a website download is not working title: "[download fail]: the website name" labels: bug assignees: '' --- **Website name**: name **OS:**: Windows/Linux/macOS **Video URL:**: url **Stack overflow** ``` error message here ``` **Screenshots** none **Additional context** none ================================================ FILE: .github/ISSUE_TEMPLATE/support-a-new-website.md ================================================ --- name: Support a new website about: If you want to request support for a new website title: "[new website require]: the website name" labels: enhancement assignees: '' --- - **Website name**: name - **Stream link**: url ================================================ FILE: .github/workflows/builder.yml ================================================ name: Builder on: push: branches: "*" paths: - "**/*.go" - "go.mod" - "go.sum" - ".github/workflows/builder.yml" pull_request: branches: "*" paths: - "**/*.go" - "go.mod" - "go.sum" - ".github/workflows/builder.yml" workflow_dispatch: env: PRODUCT: lux CGO_ENABLED: 0 GO111MODULE: on jobs: build: name: Build strategy: matrix: os: [ linux, freebsd, openbsd, dragonfly, windows, darwin ] arch: [ amd64, 386 ] include: - os: linux arch: arm arm: 5 - os: linux arch: arm arm: 6 - os: linux arch: arm arm: 7 - os: linux arch: arm64 - os: linux arch: mips mips: softfloat - os: linux arch: mips mips: hardfloat - os: linux arch: mipsle mipsle: softfloat - os: linux arch: mipsle mipsle: hardfloat - os: linux arch: mips64 - os: linux arch: mips64le - os: linux arch: ppc64 - os: linux arch: ppc64le - os: linux arch: s390x - os: windows arch: arm - os: android arch: arm64 - os: darwin arch: arm64 - os: freebsd arch: arm64 exclude: - os: darwin arch: 386 - os: dragonfly arch: 386 fail-fast: false runs-on: ubuntu-latest continue-on-error: true env: GOOS: ${{ matrix.os }} GOARCH: ${{ matrix.arch }} GOARM: ${{ matrix.arm }} GOMIPS: ${{ matrix.mips }} GOMIPS64: ${{ matrix.mips64 }} GOMIPSLE: ${{ matrix.mipsle }} steps: - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.24 - name: Check out code base if: github.event_name == 'push' uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check out code base if: github.event_name == 'pull_request' uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Cache go module uses: actions/cache@v4 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-go- - name: Get dependencies run: | go get -v -t -d ./... - name: Build binary id: builder run: | ARGS="${GOOS}-${GOARCH}" if [[ -n "${GOARM}" ]]; then ARGS="${ARGS}v${GOARM}" elif [[ -n "${GOMIPS}" ]]; then ARGS="${ARGS}-${GOMIPS}" elif [[ -n "${GOMIPS64}" ]]; then ARGS="${ARGS}-${GOMIPS64}" elif [[ -n "${GOMIPSLE}" ]]; then ARGS="${ARGS}-${GOMIPSLE}" fi go build -trimpath --ldflags "-s -w -buildid=" -v -o ./bin/${{ env.PRODUCT }}-${ARGS} echo "::set-output name=filename::${{ env.PRODUCT }}-${ARGS}" - name: Upload binary artifacts uses: actions/upload-artifact@v4 with: name: ${{ steps.builder.outputs.filename }} path: ./bin/${{ env.PRODUCT }}* if-no-files-found: error ================================================ FILE: .github/workflows/ci.yml ================================================ name: ci on: push: pull_request: schedule: # run ci weekly - cron: "0 0 * * 0" jobs: ci: runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: matrix: go: ["1.24"] os: [ubuntu-latest, macOS-latest] name: Go ${{ matrix.go }} in ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Environment run: | go version go env - name: Lint uses: golangci/golangci-lint-action@v5 with: version: v1.64.7 only-new-issues: true - name: Test env: GOFLAGS: -mod=mod run: go test -race -coverpkg=./... -coverprofile=coverage.txt ./... - name: Send coverage run: bash <(curl -s https://codecov.io/bash) ================================================ FILE: .github/workflows/goreleaser.yml ================================================ name: goreleaser on: push: tags: - '*' permissions: contents: write jobs: goreleaser: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.24 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 with: version: latest args: release --clean env: GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} ================================================ FILE: .github/workflows/stream_acfun.yml ================================================ name: acfun on: push: paths: - "extractors/acfun/*.go" - ".github/workflows/stream_acfun.yml" pull_request: paths: - "extractors/acfun/*.go" - ".github/workflows/stream_acfun.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/acfun ================================================ FILE: .github/workflows/stream_bcy.yml ================================================ name: bcy on: push: paths: - "extractors/bcy/*.go" - ".github/workflows/stream_bcy.yml" pull_request: paths: - "extractors/bcy/*.go" - ".github/workflows/stream_bcy.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bcy ================================================ FILE: .github/workflows/stream_bilibili.yml ================================================ name: bilibili on: push: paths: - "extractors/bilibili/*.go" - ".github/workflows/stream_bilibili.yml" pull_request: paths: - "extractors/bilibili/*.go" - ".github/workflows/stream_bilibili.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bilibili ================================================ FILE: .github/workflows/stream_bitchute.yml ================================================ name: bitchute on: push: paths: - "extractors/bitchute/*.go" - ".github/workflows/stream_bitchute.yml" pull_request: paths: - "extractors/bitchute/*.go" - ".github/workflows/stream_bitchute.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/bitchute ================================================ FILE: .github/workflows/stream_douyin.yml ================================================ name: douyin on: push: paths: - "extractors/douyin/*.go" - ".github/workflows/stream_douyin.yml" pull_request: paths: - "extractors/douyin/*.go" - ".github/workflows/stream_douyin.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/douyin ================================================ FILE: .github/workflows/stream_douyu.yml ================================================ name: douyu on: push: paths: - "extractors/douyu/*.go" - ".github/workflows/stream_douyu.yml" pull_request: paths: - "extractors/douyu/*.go" - ".github/workflows/stream_douyu.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/douyu ================================================ FILE: .github/workflows/stream_eporner.yml ================================================ name: eporner on: push: paths: - "extractors/eporner/*.go" - ".github/workflows/stream_eporner.yml" pull_request: paths: - "extractors/eporner/*.go" - ".github/workflows/stream_eporner.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/eporner ================================================ FILE: .github/workflows/stream_facebook.yml ================================================ name: facebook on: push: paths: - "extractors/facebook/*.go" - ".github/workflows/stream_facebook.yml" pull_request: paths: - "extractors/facebook/*.go" - ".github/workflows/stream_facebook.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/facebook ================================================ FILE: .github/workflows/stream_geekbang.yml ================================================ name: geekbang on: push: paths: - "extractors/geekbang/*.go" - ".github/workflows/stream_geekbang.yml" pull_request: paths: - "extractors/geekbang/*.go" - ".github/workflows/stream_geekbang.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/geekbang ================================================ FILE: .github/workflows/stream_haokan.yml ================================================ name: haokan on: push: paths: - "extractors/haokan/*.go" - ".github/workflows/stream_haokan.yml" pull_request: paths: - "extractors/haokan/*.go" - ".github/workflows/stream_haokan.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/haokan ================================================ FILE: .github/workflows/stream_hupu.yml ================================================ name: hupu on: push: paths: - "extractors/hupu/*.go" - ".github/workflows/stream_hupu.yml" pull_request: paths: - "extractors/hupu/*.go" - ".github/workflows/stream_hupu.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/hupu ================================================ FILE: .github/workflows/stream_huya.yml ================================================ name: huya on: push: paths: - "extractors/huya/*.go" - ".github/workflows/stream_huya.yml" pull_request: paths: - "extractors/huya/*.go" - ".github/workflows/stream_huya.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/huya ================================================ FILE: .github/workflows/stream_instagram.yml ================================================ name: instagram on: push: paths: - "extractors/instagram/*.go" - ".github/workflows/stream_instagram.yml" pull_request: paths: - "extractors/instagram/*.go" - ".github/workflows/stream_instagram.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/instagram ================================================ FILE: .github/workflows/stream_iqiyi.yml ================================================ name: iqiyi on: push: paths: - "extractors/iqiyi/*.go" - ".github/workflows/stream_iqiyi.yml" pull_request: paths: - "extractors/iqiyi/*.go" - ".github/workflows/stream_iqiyi.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/iqiyi ================================================ FILE: .github/workflows/stream_ixigua.yml ================================================ name: ixigua on: push: paths: - "extractors/ixigua/*.go" - ".github/workflows/stream_ixigua.yml" pull_request: paths: - "extractors/ixigua/*.go" - ".github/workflows/stream_ixigua.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/ixigua ================================================ FILE: .github/workflows/stream_kuaishou.yml ================================================ name: kuaishou on: push: paths: - "extractors/kuaishou/*.go" - ".github/workflows/stream_kuaishou.yml" pull_request: paths: - "extractors/kuaishou/*.go" - ".github/workflows/stream_kuaishou.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/kuaishou ================================================ FILE: .github/workflows/stream_mgtv.yml ================================================ name: mgtv on: push: paths: - "extractors/mgtv/*.go" - ".github/workflows/stream_mgtv.yml" pull_request: paths: - "extractors/mgtv/*.go" - ".github/workflows/stream_mgtv.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/mgtv ================================================ FILE: .github/workflows/stream_miaopai.yml ================================================ name: miaopai on: push: paths: - "extractors/miaopai/*.go" - ".github/workflows/stream_miaopai.yml" pull_request: paths: - "extractors/miaopai/*.go" - ".github/workflows/stream_miaopai.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/miaopai ================================================ FILE: .github/workflows/stream_netease.yml ================================================ name: netease on: push: paths: - "extractors/netease/*.go" - ".github/workflows/stream_netease.yml" pull_request: paths: - "extractors/netease/*.go" - ".github/workflows/stream_netease.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/netease ================================================ FILE: .github/workflows/stream_odysee.yml ================================================ name: odysee on: push: paths: - "extractors/odysee/*.go" - ".github/workflows/stream_odysee.yml" pull_request: paths: - "extractors/odysee/*.go" - ".github/workflows/stream_odysee.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/odysee ================================================ FILE: .github/workflows/stream_pinterest.yml ================================================ name: pinterest on: push: paths: - "extractors/pinterest/*.go" - ".github/workflows/stream_pinterest.yml" pull_request: paths: - "extractors/pinterest/*.go" - ".github/workflows/stream_pinterest.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pinterest ================================================ FILE: .github/workflows/stream_pixivision.yml ================================================ name: pixivision on: push: paths: - "extractors/pixivision/*.go" - ".github/workflows/stream_pixivision.yml" pull_request: paths: - "extractors/pixivision/*.go" - ".github/workflows/stream_pixivision.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pixivision ================================================ FILE: .github/workflows/stream_pornhub.yml ================================================ name: pornhub on: push: paths: - "extractors/pornhub/*.go" - ".github/workflows/stream_pornhub.yml" pull_request: paths: - "extractors/pornhub/*.go" - ".github/workflows/stream_pornhub.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pornhub ================================================ FILE: .github/workflows/stream_qq.yml ================================================ name: qq on: push: paths: - "extractors/qq/*.go" - ".github/workflows/stream_qq.yml" pull_request: paths: - "extractors/qq/*.go" - ".github/workflows/stream_qq.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/qq ================================================ FILE: .github/workflows/stream_reddit.yml ================================================ name: reddit on: push: paths: - "extractors/reddit/*.go" - ".github/workflows/stream_reddit.yml" pull_request: paths: - "extractors/reddit/*.go" - ".github/workflows/stream_reddit.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/reddit ================================================ FILE: .github/workflows/stream_rumble.yml ================================================ name: rumble on: push: paths: - "extractors/rumble/*.go" - ".github/workflows/stream_rumble.yml" pull_request: paths: - "extractors/rumble/*.go" - ".github/workflows/stream_rumble.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/rumble ================================================ FILE: .github/workflows/stream_streamtape.yml ================================================ name: streamtape on: push: paths: - "extractors/streamtape/*.go" - ".github/workflows/stream_streamtape.yml" pull_request: paths: - "extractors/streamtape/*.go" - ".github/workflows/stream_streamtape.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/streamtape ================================================ FILE: .github/workflows/stream_tangdou.yml ================================================ name: tangdou on: push: paths: - "extractors/tangdou/*.go" - ".github/workflows/stream_tangdou.yml" pull_request: paths: - "extractors/tangdou/*.go" - ".github/workflows/stream_tangdou.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tangdou ================================================ FILE: .github/workflows/stream_threads.yml ================================================ name: instagram on: push: paths: - "extractors/threads/*.go" - ".github/workflows/stream_threads.yml" pull_request: paths: - "extractors/threads/*.go" - ".github/workflows/stream_threads.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/threads ================================================ FILE: .github/workflows/stream_tiktok.yml ================================================ name: tiktok on: push: paths: - "extractors/tiktok/*.go" - ".github/workflows/stream_tiktok.yml" pull_request: paths: - "extractors/tiktok/*.go" - ".github/workflows/stream_tiktok.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tiktok ================================================ FILE: .github/workflows/stream_tumblr.yml ================================================ name: tumblr on: push: paths: - "extractors/tumblr/*.go" - ".github/workflows/stream_tumblr.yml" pull_request: paths: - "extractors/tumblr/*.go" - ".github/workflows/stream_tumblr.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/tumblr ================================================ FILE: .github/workflows/stream_twitter.yml ================================================ name: twitter on: push: paths: - "extractors/twitter/*.go" - ".github/workflows/stream_twitter.yml" pull_request: paths: - "extractors/twitter/*.go" - ".github/workflows/stream_twitter.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/twitter ================================================ FILE: .github/workflows/stream_udn.yml ================================================ name: udn on: push: paths: - "extractors/udn/*.go" - ".github/workflows/stream_udn.yml" pull_request: paths: - "extractors/udn/*.go" - ".github/workflows/stream_udn.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/udn ================================================ FILE: .github/workflows/stream_vimeo.yml ================================================ name: vimeo on: push: paths: - "extractors/vimeo/*.go" - ".github/workflows/stream_vimeo.yml" pull_request: paths: - "extractors/vimeo/*.go" - ".github/workflows/stream_vimeo.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/vimeo ================================================ FILE: .github/workflows/stream_vk.yml ================================================ name: vk on: push: paths: - "extractors/vk/*.go" - ".github/workflows/stream_vk.yml" pull_request: paths: - "extractors/vk/*.go" - ".github/workflows/stream_vk.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/vk ================================================ FILE: .github/workflows/stream_weibo.yml ================================================ name: weibo on: push: paths: - "extractors/weibo/*.go" - ".github/workflows/stream_weibo.yml" pull_request: paths: - "extractors/weibo/*.go" - ".github/workflows/stream_weibo.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/weibo ================================================ FILE: .github/workflows/stream_xiaohongshu.yml ================================================ name: xiaohongshu on: push: paths: - "extractors/xiaohongshu/*.go" - ".github/workflows/stream_xiaohongshu.yml" pull_request: paths: - "extractors/xiaohongshu/*.go" - ".github/workflows/stream_xiaohongshu.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xiaohongshu ================================================ FILE: .github/workflows/stream_ximalaya.yml ================================================ name: ximalaya on: push: paths: - "extractors/ximalaya/*.go" - ".github/workflows/stream_ximalaya.yml" pull_request: paths: - "extractors/ximalaya/*.go" - ".github/workflows/stream_ximalaya.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/ximalaya ================================================ FILE: .github/workflows/stream_xinpianchang.yml ================================================ name: xinpianchang on: push: paths: - "extractors/xinpianchang/*.go" - ".github/workflows/stream_xinpianchang.yml" pull_request: paths: - "extractors/xinpianchang/*.go" - ".github/workflows/stream_xinpianchang.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xinpianchang ================================================ FILE: .github/workflows/stream_xvideos.yml ================================================ name: xvideos on: push: paths: - "extractors/xvideos/*.go" - ".github/workflows/stream_xvideos.yml" pull_request: paths: - "extractors/xvideos/*.go" - ".github/workflows/stream_xvideos.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xvideos ================================================ FILE: .github/workflows/stream_yinyuetai.yml ================================================ name: yinyuetai on: push: paths: - "extractors/yinyuetai/*.go" - ".github/workflows/stream_yinyuetai.yml" pull_request: paths: - "extractors/yinyuetai/*.go" - ".github/workflows/stream_yinyuetai.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/yinyuetai ================================================ FILE: .github/workflows/stream_youku.yml ================================================ name: youku on: push: paths: - "extractors/youku/*.go" - ".github/workflows/stream_youku.yml" pull_request: paths: - "extractors/youku/*.go" - ".github/workflows/stream_youku.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/youku ================================================ FILE: .github/workflows/stream_youtube.yml ================================================ name: youtube on: push: paths: - "extractors/youtube/*.go" - ".github/workflows/stream_youtube.yml" pull_request: paths: - "extractors/youtube/*.go" - ".github/workflows/stream_youtube.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/youtube ================================================ FILE: .github/workflows/stream_zhihu.yml ================================================ name: zhihu on: push: paths: - "extractors/zhihu/*.go" - ".github/workflows/stream_zhihu.yml" pull_request: paths: - "extractors/zhihu/*.go" - ".github/workflows/stream_zhihu.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/zhihu ================================================ FILE: .github/workflows/stream_zingmp3.yml ================================================ name: zingmp3 on: push: paths: - "extractors/zingmp3/*.go" - ".github/workflows/stream_zingmp3.yml" pull_request: paths: - "extractors/zingmp3/*.go" - ".github/workflows/stream_zingmp3.yml" schedule: # run ci weekly - cron: "0 0 * * 0" jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: go: ["1.24"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/zingmp3 ================================================ FILE: .gitignore ================================================ # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out coverage.txt # Python *.pyc .vscode .idea dist/ *_token downloader/*.jpg # Ignore compiled binaries # - native annie # - gox builds annie_* # macOS .DS_Store *.mp4 *.mkv *.webm lux ================================================ FILE: .golangci.yml ================================================ run: concurrency: 2 timeout: 5m go: 1.24 linter-settings: goconst: min-len: 2 min-occurrences: 2 linters: enable: - bodyclose - errcheck - goconst - gofmt - goimports - gosimple - govet - ineffassign - misspell - nilerr - staticcheck - typecheck - unconvert - unparam - unused - whitespace issues: exclude-use-default: false exclude-rules: - path: _test.go linters: - errcheck ================================================ FILE: .goreleaser.yml ================================================ project_name: lux env: - GO111MODULE=on - CGO_ENABLED=0 before: hooks: - go mod download builds: - binary: lux ldflags: -s -w -X github.com/iawia002/lux/app.version={{ .RawVersion }} goos: - windows - darwin - linux - freebsd - openbsd - netbsd goarch: - "386" - amd64 - arm - arm64 ignore: - goos: freebsd goarch: arm goarm: 6 - goos: freebsd goarch: arm64 - goos: openbsd goarch: arm goarm: 6 archives: - name_template: >- {{ .ProjectName }}_ {{- .Version }}_ {{- title .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "386" }}i386 {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} format: tar.gz format_overrides: - goos: windows format: zip files: - none* wrap_in_directory: false ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Guide * [Style Guide](#style-guide) * [Build](#build) * [Features Requested](#features-requested) ## Style Guide ### Code format Lux uses [gofmt](https://golang.org/cmd/gofmt) to format the code, you must use [gofmt](https://golang.org/cmd/gofmt) to format your code before submitting. ### linter We recommend using [golint](https://github.com/golang/lint) or [gometalinter](https://github.com/alecthomas/gometalinter) to check your code format. ## Build Make sure that this folder is in `GOPATH`, then: ```bash $ go build ``` ## Features Requested There are several [features](https://github.com/iawia002/lux/issues?q=is%3Aissue+is%3Aopen+label%3Afeature-request) requested by the community. If you have any idea, feel free to fork the repo, follow the style guide above, push and merge it after passing the test. Besides, you are welcomed to propose new features through the issue. ================================================ FILE: Cask.toml ================================================ [package] name = "github.com/iawia002/lux" bin = "lux" authors = ["Xinzhao Xu "] keywords = ["go", "golang", "crawler", "scraper", "downloader", "youtube", "video", "download", "tumblr", "bilibili", "qq", "hacktoberfest", "youku", "iqiyi"] repository = "https://github.com/iawia002/lux" description = """ 👾 Fast and simple video download library and CLI tool written in Go """ [darwin] x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_x86_64.tar.gz" } aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_arm64.tar.gz" } [windows] x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Windows_x86_64.zip" } [linux] x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_x86_64.tar.gz" } aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_arm64.tar.gz" } ================================================ FILE: LICENSE ================================================ MIT License Copyright 2018-present, iawia002 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================

Lux

Let there be Lux!

Codecov GitHub Workflow Status Go Report Card GitHub release Homebrew
👾 Lux is a fast and simple video downloader built with Go. - [Installation](#installation) - [Prerequisites](#prerequisites) - [Install via `go install`](#install-via-go-install) - [Homebrew (macOS only)](#homebrew-macos-only) - [Arch Linux](#arch-linux) - [Void Linux](#void-linux) - [Scoop on Windows](#scoop-on-windows) - [Chocolatey on Windows](#chocolatey-on-windows) - [Cask on Windows/macOS/Linux](#cask-on-windowsmacoslinux) - [Getting Started](#getting-started) - [Download a video](#download-a-video) - [Download anything else](#download-anything-else) - [Download playlist](#download-playlist) - [Multiple inputs](#multiple-inputs) - [Resume a download](#resume-a-download) - [Auto retry](#auto-retry) - [Cookies](#cookies) - [Proxy](#proxy) - [Multi-Thread](#multi-thread) - [Short link](#short-link) - [bilibili](#bilibili) - [Use specified Referrer](#use-specified-referrer) - [Specify the output path and name](#specify-the-output-path-and-name) - [Debug Mode](#debug-mode) - [Reuse extracted data](#reuse-extracted-data) - [Options](#options) - [Download:](#download) - [Network:](#network) - [Playlist:](#playlist) - [Filesystem:](#filesystem) - [Subtitle:](#subtitle) - [Youku:](#youku) - [aria2:](#aria2) - [Supported Sites](#supported-sites) - [Known issues](#known-issues) - [优酷](#优酷) - [西瓜/头条视频](#西瓜头条视频) - [Contributing](#contributing) - [Authors](#authors) - [Similar projects](#similar-projects) - [License](#license) ## Installation ### Prerequisites The following dependencies are required and must be installed separately. - **[FFmpeg](https://www.ffmpeg.org)** > **Note**: FFmpeg does not affect the download, only affects the final file merge. ### Install via `go install` To install Lux, use `go install`, or download the binary file from [Releases](https://github.com/iawia002/lux/releases) page. ```bash $ go install github.com/iawia002/lux@latest ``` ### Homebrew (macOS only) For macOS users, you can install `lux` via: ```bash $ brew install lux ``` ### Arch Linux For Arch Users [AUR](https://aur.archlinux.org/packages/lux-dl/) package is available. ### Void Linux For Void linux users, you can install `lux` via: ``` $ xbps-install -S lux ``` ### [Scoop](https://scoop.sh/) on Windows ```sh $ scoop install lux ``` ### [Chocolatey](https://chocolatey.org/) on Windows ``` $ choco install lux ``` ### [Cask](https://github.com/axetroy/cask.rs) on Windows/macOS/Linux ```sh $ cask install github.com/iawia002/lux ``` ## Getting Started Usage: ``` lux [OPTIONS] URL [URL...] ``` ### Download a video ```console $ lux "https://www.youtube.com/watch?v=dQw4w9WgXcQ" Site: YouTube youtube.com Title: Rick Astley - Never Gonna Give You Up (Video) Type: video Stream: [248] ------------------- Quality: 1080p video/webm; codecs="vp9" Size: 63.93 MiB (67038963 Bytes) # download with: lux -f 248 ... 41.88 MiB / 63.93 MiB [=================>-------------] 65.51% 4.22 MiB/s 00m05s ``` The `-i` option displays all available quality of video without downloading. ```console $ lux -i "https://www.youtube.com/watch?v=dQw4w9WgXcQ" Site: YouTube youtube.com Title: Rick Astley - Never Gonna Give You Up (Video) Type: video Streams: # All available quality [248] ------------------- Quality: 1080p video/webm; codecs="vp9" Size: 49.29 MiB (51687554 Bytes) # download with: lux -f 248 ... [137] ------------------- Quality: 1080p video/mp4; codecs="avc1.640028" Size: 43.45 MiB (45564306 Bytes) # download with: lux -f 137 ... [398] ------------------- Quality: 720p video/mp4; codecs="av01.0.05M.08" Size: 37.12 MiB (38926432 Bytes) # download with: lux -f 398 ... [136] ------------------- Quality: 720p video/mp4; codecs="avc1.4d401f" Size: 31.34 MiB (32867324 Bytes) # download with: lux -f 136 ... [247] ------------------- Quality: 720p video/webm; codecs="vp9" Size: 31.03 MiB (32536181 Bytes) # download with: lux -f 247 ... ``` Use `lux -f stream "URL"` to download a specific stream listed in the output of `-i` option. ### Download anything else If Lux is provided the URL of a specific resource, then it will be downloaded directly: ```console $ lux "https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg" lux doesn't support this URL right now, but it will try to download it directly Site: Universal Title: 1f5a87801a0711e898b12b640777720f Type: image/jpeg Stream: [default] ------------------- Size: 1.00 MiB (1051042 Bytes) # download with: lux -f default "URL" 1.00 MiB / 1.00 MiB [===================================] 100.00% 1.21 MiB/s 0s ``` ### Download playlist The `-p` option downloads an entire playlist instead of a single video. ```console $ lux -i -p "https://www.bilibili.com/bangumi/play/ep198061" Site: 哔哩哔哩 bilibili.com Title: Doctor X 第四季：第一集 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 845.66 MiB (886738354 Bytes) # download with: lux -f default "URL" Site: 哔哩哔哩 bilibili.com Title: Doctor X 第四季：第二集 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 930.71 MiB (975919195 Bytes) # download with: lux -f default "URL" ...... ``` You can use the `-start`, `-end` or `-items` option to specify the download range of the list: ``` -start Playlist video to start at (default 1) -end Playlist video to end at -items Playlist video items to download. Separated by commas like: 1,5,6,8-10 ``` For bilibili playlists only: ``` -eto File name of each bilibili episode doesn't include the playlist title ``` ### Multiple inputs You can also download multiple URLs at once: ```console $ lux -i "https://www.bilibili.com/video/av21877586" "https://www.bilibili.com/video/av21990740" Site: 哔哩哔哩 bilibili.com Title: 【莓机会了】甜到虐哭的13集单集MAD「我现在什么都不想干,更不想看14集」 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 51.88 MiB (54403767 Bytes) # download with: lux -f default "URL" Site: 哔哩哔哩 bilibili.com Title: 【莓救了】甜到虐哭！！！国家队单集MAD-当熟悉的bgm响起，眼泪从脸颊滑下 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 77.63 MiB (81404093 Bytes) # download with: lux -f default "URL" ``` These URLs will be downloaded one by one. You can also use the `-F` option to read URLs from file: ```console $ lux -F ~/Desktop/u.txt Site: 微博 weibo.com Title: 在Google，我们设计什么？ via@阑夕 Type: video Stream: [default] ------------------- Size: 19.19 MiB (20118196 Bytes) # download with: lux -f default "URL" 19.19 MiB / 19.19 MiB [=================================] 100.00% 9.69 MiB/s 1s ...... ``` You can use the `-start`, `-end` or `-items` option to specify the download range of the list: ``` -start File line to start at (default 1) -end File line to end at -items File lines to download. Separated by commas like: 1,5,6,8-10 ``` ### Resume a download Ctrl+C interrupts a download. A temporary `.download` file is kept in the output directory. If `lux` is ran with the same arguments, then the download progress will resume from the last session. ### Auto retry lux will auto retry when the download failed, you can specify the retry times by `-retry` option (default is 100). ### Cookies Cookies can be provided to `lux` with the `-c` option if they are required for accessing the video. Cookies can be the following format or [Netscape Cookie](https://curl.haxx.se/rfc/cookie_spec.html) format: ```console name=value; name2=value2; ... ``` Cookies can be a string or a text file, supply cookies in one of the two following ways. As a string: ```console $ lux -c "name=value; name2=value2" "https://www.bilibili.com/video/av20203945" ``` As a text file: ```console $ lux -c cookies.txt "https://www.bilibili.com/video/av20203945" ``` ### Proxy You can set the HTTP/SOCKS5 proxy using environment variables: ```console $ HTTP_PROXY="http://127.0.0.1:1087/" lux -i "https://www.youtube.com/watch?v=Gnbch2osEeo" ``` ```console $ HTTP_PROXY="socks5://127.0.0.1:1080/" lux -i "https://www.youtube.com/watch?v=Gnbch2osEeo" ``` ### Multi-Thread Use `--multi-thread` or `-m` multiple threads to download single video. Use `--thread` or `-n` option to set the number of download threads(default is 10). > Note: If the video has multi fragment, the number of actual download threads will increase. > > For example: > * If `-n` is set to 10, and the video has 2 fragments, then 20 threads will actually be used. > * If the video has 20 fragments, only 10 fragments are downloaded in the same time, the actual threads count is 100. > **Special Tips:** Use too many threads in **mgtv** download will cause HTTP 403 error, we recommend setting the number of threads to **1**. ### Short link #### bilibili You can just use `av` or `ep` number to download bilibili's video: ```console $ lux -i ep198381 av21877586 Site: 哔哩哔哩 bilibili.com Title: 狐妖小红娘：第79话 南国公主的吃货本色 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 485.23 MiB (508798478 Bytes) # download with: lux -f default "URL" Site: 哔哩哔哩 bilibili.com Title: 【莓机会了】甜到虐哭的13集单集MAD「我现在什么都不想干,更不想看14集」 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 51.88 MiB (54403767 Bytes) # download with: lux -f default "URL" ``` ### Use specified Referrer A Referrer can be used for the request with the `-r` option: ```console $ lux -r "https://www.bilibili.com/video/av20383055/" "http://cn-scnc1-dx.acgvideo.com/" ``` ### Specify the output path and name The `-o` option sets the path, and `-O` option sets the name of the downloaded file: ```console $ lux -o ../ -O "hello" "https://example.com" ``` ### Debug Mode The `-d` option outputs network request messages: ```console $ lux -i -d "http://www.bilibili.com/video/av20088587" URL: http://www.bilibili.com/video/av20088587 Method: GET Headers: http.Header{ "Referer": {"http://www.bilibili.com/video/av20088587"}, "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, "Accept-Charset": {"UTF-8,*;q=0.5"}, "Accept-Encoding": {"gzip,deflate,sdch"}, "Accept-Language": {"en-US,en;q=0.8"}, "User-Agent": {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"}, } Status Code: 200 URL: https://interface.bilibili.com/v2/playurl?appkey=84956560bc028eb7&cid=32782944&otype=json&qn=116&quality=116&type=&sign=fb2e3f261fec398652f96d358517e535 Method: GET Headers: http.Header{ "Accept-Charset": {"UTF-8,*;q=0.5"}, "Accept-Encoding": {"gzip,deflate,sdch"}, "Accept-Language": {"en-US,en;q=0.8"}, "User-Agent": {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"}, "Referer": {"https://interface.bilibili.com/v2/playurl?appkey=84956560bc028eb7&cid=32782944&otype=json&qn=116&quality=116&type=&sign=fb2e3f261fec398652f96d358517e535"}, "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, } Status Code: 200 Site: 哔哩哔哩 bilibili.com Title: 燃油动力的遥控奥迪R8跑赛道 Type: video Streams: # All available quality [default] ------------------- Quality: 高清 1080P Size: 64.38 MiB (67504795 Bytes) # download with: lux -f default "URL" ``` ### Reuse extracted data The `-j` option will print the extracted data in JSON format. ```console $ lux -j "https://www.bilibili.com/video/av20203945" { "site": "哔哩哔哩 bilibili.com", "title": "【2018拜年祭单品】相遇day by day", "type": "video", "streams": { "15": { "urls": [ { "url": "...", "size": 18355205, "ext": "flv" } ], "quality": "流畅 360P", "size": 18355205 }, "32": { "urls": [ { "url": "...", "size": 40058632, "ext": "flv" } ], "quality": "清晰 480P", "size": 40058632 }, "64": { "urls": [ { "url": "...", "size": 82691087, "ext": "flv" } ], "quality": "高清 720P", "size": 82691087 }, "80": { "urls": [ { "url": "...", "size": 121735559, "ext": "flv" } ], "quality": "高清 1080P", "size": 121735559 } } } ``` ### Options ``` -i Information only -F string URLs file path -d Debug mode -j Print extracted data -s Minimum outputs -v Show version ``` #### Download: ``` -f string Select specific stream to download -p Download playlist -n int The number of download thread (only works for multiple-parts video) (default 10) -c string Cookie -r string Use specified Referrer -cs int HTTP chunk size for downloading (in MB) (default 1) ``` #### Network: ``` -retry int How many times to retry when the download failed (default 10) ``` #### Playlist: ``` -start int Playlist video to start at (default 1) -end int Playlist video to end at -items string Playlist video items to download. Separated by commas like: 1,5,6,8-10 ``` #### Filesystem: ``` -o string Specify the output path -O string Specify the output file name ``` #### Subtitle: ``` -C Download subtitles -C -items en,zh Download specific languages (YouTube only) -C -items en,zh -embed Embed subtitles into the video (YouTube only) ``` #### Youku: ``` -ccode string Youku ccode (default "0502") -ckey string Youku ckey (default "7B19C0AB12633B22E7FE81271162026020570708D6CC189E4924503C49D243A0DE6CD84A766832C2C99898FC5ED31F3709BB3CDD82C96492E721BDD381735026") -password string Youku password ``` #### aria2: > Note: If you use aria2 to download, you need to merge the multi-part videos yourself. ``` -aria2 Use Aria2 RPC to download -aria2addr string Aria2 Address (default "localhost:6800") -aria2method string Aria2 Method (default "http") -aria2token string Aria2 RPC Token ``` ## Supported Sites | Site | URL | 🎬 Videos | 🌁 Images | 🔊 Audio | 📚 Playlist | 🍪 VIP adaptation | Build Status | | ---------------- | ------------------------------------------------------------------------- | -------- | -------- | ------- | ---------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 抖音 | | ✓ | ✓ | | | | [![douyin](https://github.com/iawia002/lux/actions/workflows/stream_douyin.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_douyin.yml) | | 哔哩哔哩 | | ✓ | | | ✓ | ✓ | [![bilibili](https://github.com/iawia002/lux/actions/workflows/stream_bilibili.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bilibili.yml) | | 半次元 | | | ✓ | | | | [![bcy](https://github.com/iawia002/lux/actions/workflows/stream_bcy.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bcy.yml) | | pixivision | | | ✓ | | | | [![pixivision](https://github.com/iawia002/lux/actions/workflows/stream_pixivision.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pixivision.yml) | | 优酷 | | ✓ | | | | ✓ | [![youku](https://github.com/iawia002/lux/actions/workflows/stream_youku.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_youku.yml) | | YouTube | | ✓ | | | ✓ | | [![youtube](https://github.com/iawia002/lux/actions/workflows/stream_youtube.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_youtube.yml) | | 西瓜视频（头条） | , , | ✓ | | | | | [![ixigua](https://github.com/iawia002/lux/actions/workflows/stream_ixigua.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_ixigua.yml) | | 爱奇艺 | | ✓ | | | | | [![iqiyi](https://github.com/iawia002/lux/actions/workflows/stream_iqiyi.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_iqiyi.yml) | | 新片场 | | ✓ | | | | | [![xinpianchang](https://github.com/iawia002/lux/actions/workflows/stream_xinpianchang.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xinpianchang.yml) | | 芒果 TV | | ✓ | | | | | [![mgtv](https://github.com/iawia002/lux/actions/workflows/stream_mgtv.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_mgtv.yml) | | 糖豆广场舞 | | ✓ | | | | | [![tangdou](https://github.com/iawia002/lux/actions/workflows/stream_tangdou.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tangdou.yml) | | Tumblr | | ✓ | ✓ | | | | [![tumblr](https://github.com/iawia002/lux/actions/workflows/stream_tumblr.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tumblr.yml) | | Vimeo | | ✓ | | | | | [![vimeo](https://github.com/iawia002/lux/actions/workflows/stream_vimeo.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vimeo.yml) | | Facebook | | ✓ | | | | | [![facebook](https://github.com/iawia002/lux/actions/workflows/stream_facebook.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_facebook.yml) | | 斗鱼视频 | | ✓ | | | | | [![douyu](https://github.com/iawia002/lux/actions/workflows/stream_douyu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_douyu.yml) | | 秒拍 | | ✓ | | | | | [![miaopai](https://github.com/iawia002/lux/actions/workflows/stream_miaopai.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_miaopai.yml) | | 微博 | | ✓ | | | | | [![weibo](https://github.com/iawia002/lux/actions/workflows/stream_weibo.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_weibo.yml) | | Instagram | | ✓ | ✓ | | | | [![instagram](https://github.com/iawia002/lux/actions/workflows/stream_instagram.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_instagram.yml) | | Threads | | ✓ | ✓ | | | | [![threads](https://github.com/iawia002/lux/actions/workflows/stream_threads.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_threads.yml) | | Twitter | | ✓ | | | | | [![twitter](https://github.com/iawia002/lux/actions/workflows/stream_twitter.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_twitter.yml) | | 腾讯视频 | | ✓ | | | | | [![qq](https://github.com/iawia002/lux/actions/workflows/stream_qq.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_qq.yml) | | 网易云音乐 | | ✓ | | | | | [![netease](https://github.com/iawia002/lux/actions/workflows/stream_netease.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_netease.yml) | | 音悦台 | | ✓ | | | | | [![yinyuetai](https://github.com/iawia002/lux/actions/workflows/stream_yinyuetai.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_yinyuetai.yml) | | 极客时间 | | ✓ | | | | | [![geekbang](https://github.com/iawia002/lux/actions/workflows/stream_geekbang.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_geekbang.yml) | | Pornhub | | ✓ | | | | | [![pornhub](https://github.com/iawia002/lux/actions/workflows/stream_pornhub.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pornhub.yml) | | XVIDEOS | | ✓ | | | | | [![xvideos](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml) | | 聯合新聞網 | | ✓ | | | | | [![udn](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml) | | TikTok | | ✓ | | | | | [![tiktok](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml) | | Pinterest | | ✓ | | | | | [![pinterest](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml) | | 好看视频 | | ✓ | | | | | [![haokan](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml) | | AcFun | | ✓ | | | ✓ | | [![acfun](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml) | | Eporner | | ✓ | | | | | [![eporner](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml) | | StreamTape | | ✓ | | | | | [![streamtape](https://github.com/iawia002/lux/actions/workflows/stream_streamtape.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_streamtape.yml) | | 虎扑 | | ✓ | | | | | [![hupu](https://github.com/iawia002/lux/actions/workflows/stream_hupu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_hupu.yml) | | 虎牙视频 | | ✓ | | | | | [![huya](https://github.com/iawia002/lux/actions/workflows/stream_huya.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_huya.yml) | | 喜马拉雅 | | | | ✓ | | | [![ximalaya](https://github.com/iawia002/lux/actions/workflows/stream_ximalaya.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_ximalaya.yml) | | 快手 | | ✓ | | | | | [![kuaishou](https://github.com/iawia002/lux/actions/workflows/stream_kuaishou.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_kuaishou.yml) | | Reddit | | ✓ | ✓ | | | | [![reddit](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml) | | VKontakte | | ✓ | | | | | [![vk](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/) | | 知乎 | | ✓ | | | | | [![zhihu](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/) | | Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) | | 小红书 | | ✓ | | | | | [![xiaohongshu](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/) | | Zing MP3 | | ✓ | | ✓ | | | [![zingmp3](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/) | | Bitchute | | ✓ | | | | | [![bitchute](https://github.com/iawia002/lux/actions/workflows/stream_bitchute.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_bitchute.yml/) | | Odysee | | ✓ | | ✓ | | | [![odysee](https://github.com/iawia002/lux/actions/workflows/stream_odysee.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_odysee.yml/) | ## Known issues ### 优酷 优酷的 `ccode` 经常变化导致 lux 不可用，如果你知道有新的可用的 `ccode`，可以直接使用 `lux -ccode ...` 而不用等待 lux 更新（当然，也欢迎你给我们提一个 Pull request 来更新默认的 `ccode`） 最好是每次下载都附带登录过的 Cookie 以避免部分 `ccode` 的问题 ### 西瓜/头条视频 西瓜/头条视频必须带 Cookie 才能下载成功，西瓜和头条可共用西瓜视频的 Cookie，Cookie 的有效期可能较短，下载失败就更新 Cookie 尝试： ``` $ lux -c "msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C..." "https://m.toutiao.com/is/iYbTfJ79/" ``` ## Contributing Lux is an open source project and built on the top of open-source projects. Check out the [Contributing Guide](./CONTRIBUTING.md) to get started. ## Authors Code with ❤️ by [iawia002](https://github.com/iawia002) and lovely [contributors](https://github.com/iawia002/lux/graphs/contributors) ## Similar projects - [youtube](https://github.com/kkdai/youtube) - [youtube-dl](https://github.com/rg3/youtube-dl) - [you-get](https://github.com/soimort/you-get) - [ytdl](https://github.com/rylio/ytdl) ## License MIT Copyright (c) 2018-present, iawia002 ================================================ FILE: app/app.go ================================================ package app import ( "encoding/json" "errors" "fmt" "os" "sort" "strings" "github.com/fatih/color" "github.com/urfave/cli/v2" "github.com/iawia002/lux/downloader" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) // Name is the name of this app. const Name = "lux" // This value will be injected into the corresponding git tag value at build time using `-ldflags`. var version = "v0.0.0" func init() { cli.VersionPrinter = func(c *cli.Context) { blue := color.New(color.FgBlue) cyan := color.New(color.FgCyan) fmt.Fprintf( color.Output, "\n%s: version %s, A fast and simple video downloader.\n\n", cyan.Sprintf(Name), blue.Sprintf(c.App.Version), ) } } // New returns the App instance. func New() *cli.App { app := &cli.App{ Name: Name, Usage: "A fast and simple video downloader.", Version: version, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "debug", Aliases: []string{"d"}, Usage: "Debug mode", }, &cli.BoolFlag{ Name: "silent", Aliases: []string{"s"}, Usage: "Minimum outputs", }, &cli.BoolFlag{ Name: "info", Aliases: []string{"i"}, Usage: "Information only", }, &cli.BoolFlag{ Name: "json", Aliases: []string{"j"}, Usage: "Print extracted JSON data", }, &cli.StringFlag{ Name: "cookie", Aliases: []string{"c"}, Usage: "Cookie", }, &cli.BoolFlag{ Name: "playlist", Aliases: []string{"p"}, Usage: "Download playlist", }, &cli.StringFlag{ Name: "user-agent", Aliases: []string{"u"}, Usage: "Use specified User-Agent", }, &cli.StringFlag{ Name: "refer", Aliases: []string{"r"}, Usage: "Use specified Referrer", }, &cli.StringFlag{ Name: "stream-format", Aliases: []string{"f"}, Usage: "Select specific stream to download", }, &cli.BoolFlag{ Name: "audio-only", Aliases: []string{"ao"}, Usage: "Download audio only at best quality", }, &cli.StringFlag{ Name: "file", Aliases: []string{"F"}, Usage: "URLs file path", }, &cli.StringFlag{ Name: "output-path", Aliases: []string{"o"}, Usage: "Specify the output path", }, &cli.StringFlag{ Name: "output-name", Aliases: []string{"O"}, Usage: "Specify the output file name", }, &cli.UintFlag{ Name: "file-name-length", Value: 255, Usage: "The maximum length of a file name, 0 means unlimited", }, &cli.BoolFlag{ Name: "caption", Aliases: []string{"C"}, Usage: "Download captions", }, &cli.BoolFlag{ Name: "embed-subtitle", Aliases: []string{"embed"}, Usage: "Embed subtitles into the video (requires ffmpeg)", }, &cli.UintFlag{ Name: "start", Value: 1, Usage: "Define the starting item of a playlist or a file input", }, &cli.UintFlag{ Name: "end", Value: 0, Usage: "Define the ending item of a playlist or a file input", }, &cli.StringFlag{ Name: "items", Usage: "Define wanted items from a file or playlist. Separated by commas like: 1,5,6,8-10", }, &cli.BoolFlag{ Name: "multi-thread", Aliases: []string{"m"}, Usage: "Multiple threads to download single video", }, &cli.UintFlag{ Name: "retry", Value: 10, Usage: "How many times to retry when the download failed", }, &cli.UintFlag{ Name: "chunk-size", Aliases: []string{"cs"}, Value: 1, Usage: "HTTP chunk size for downloading (in MB)", }, &cli.UintFlag{ Name: "thread", Aliases: []string{"n"}, Value: 10, Usage: "The number of download thread (only works for multiple-parts video)", }, // Aria2 &cli.BoolFlag{ Name: "aria2", Usage: "Use Aria2 RPC to download", }, &cli.StringFlag{ Name: "aria2-token", Usage: "Aria2 RPC Token", }, &cli.StringFlag{ Name: "aria2-addr", Value: "localhost:6800", Usage: "Aria2 Address", }, &cli.StringFlag{ Name: "aria2-method", Value: "http", Usage: "Aria2 Method", }, // youku &cli.StringFlag{ Name: "youku-ccode", Aliases: []string{"ccode"}, Value: "0502", Usage: "Youku ccode", }, &cli.StringFlag{ Name: "youku-ckey", Aliases: []string{"ckey"}, Value: "7B19C0AB12633B22E7FE81271162026020570708D6CC189E4924503C49D243A0DE6CD84A766832C2C99898FC5ED31F3709BB3CDD82C96492E721BDD381735026", Usage: "Youku ckey", }, &cli.StringFlag{ Name: "youku-password", Aliases: []string{"password"}, Usage: "Youku password", }, &cli.BoolFlag{ Name: "episode-title-only", Aliases: []string{"eto"}, Usage: "File name of each bilibili episode doesn't include the playlist title", }, }, Action: func(c *cli.Context) error { args := c.Args().Slice() if c.Bool("debug") { cli.VersionPrinter(c) } if file := c.String("file"); file != "" { f, err := os.Open(file) if err != nil { return err } defer f.Close() // nolint fileItems := utils.ParseInputFile(f, c.String("items"), int(c.Uint("start")), int(c.Uint("end"))) args = append(args, fileItems...) } if len(args) < 1 { return errors.New("too few arguments") } cookie := c.String("cookie") if cookie != "" { // If cookie is a file path, convert it to a string to ensure cookie is always string if _, fileErr := os.Stat(cookie); fileErr == nil { // Cookie is a file data, err := os.ReadFile(cookie) if err != nil { return err } cookie = strings.TrimSpace(string(data)) } } request.SetOptions(request.Options{ RetryTimes: int(c.Uint("retry")), Cookie: cookie, UserAgent: c.String("user-agent"), Refer: c.String("refer"), Debug: c.Bool("debug"), Silent: c.Bool("silent"), }) var isErr bool for _, videoURL := range args { if err := download(c, videoURL); err != nil { fmt.Fprintf( color.Output, "Downloading %s error:\n", color.CyanString("%s", videoURL), ) fmt.Printf("%+v\n", err) isErr = true } } if isErr { return cli.Exit("", 1) } return nil }, EnableBashCompletion: true, } sort.Sort(cli.FlagsByName(app.Flags)) return app } func download(c *cli.Context, videoURL string) error { data, err := extractors.Extract(videoURL, extractors.Options{ Playlist: c.Bool("playlist"), Items: c.String("items"), ItemStart: int(c.Uint("start")), ItemEnd: int(c.Uint("end")), ThreadNumber: int(c.Uint("thread")), EpisodeTitleOnly: c.Bool("episode-title-only"), Cookie: c.String("cookie"), YoukuCcode: c.String("youku-ccode"), YoukuCkey: c.String("youku-ckey"), YoukuPassword: c.String("youku-password"), }) if err != nil { // if this error occurs, it means that an error occurred before actually starting to extract data // (there is an error in the preparation step), and the data list is empty. return err } if c.Bool("json") { e := json.NewEncoder(os.Stdout) e.SetIndent("", "\t") e.SetEscapeHTML(false) if err := e.Encode(data); err != nil { return err } return nil } defaultDownloader := downloader.New(downloader.Options{ Silent: c.Bool("silent"), InfoOnly: c.Bool("info"), Stream: c.String("stream-format"), AudioOnly: c.Bool("audio-only"), Refer: c.String("refer"), OutputPath: c.String("output-path"), OutputName: c.String("output-name"), FileNameLength: int(c.Uint("file-name-length")), Caption: c.Bool("caption"), EmbedSubtitle: c.Bool("embed-subtitle"), MultiThread: c.Bool("multi-thread"), ThreadNumber: int(c.Uint("thread")), RetryTimes: int(c.Uint("retry")), ChunkSizeMB: int(c.Uint("chunk-size")), UseAria2RPC: c.Bool("aria2"), Aria2Token: c.String("aria2-token"), Aria2Method: c.String("aria2-method"), Aria2Addr: c.String("aria2-addr"), }) errors := make([]error, 0) for _, item := range data { if item.Err != nil { // if this error occurs, the preparation step is normal, but the data extraction is wrong. // the data is an empty struct. errors = append(errors, item.Err) continue } if err = defaultDownloader.Download(item); err != nil { errors = append(errors, err) } } if len(errors) != 0 { return errors[0] } return nil } ================================================ FILE: app/register.go ================================================ package app import ( _ "github.com/iawia002/lux/extractors/acfun" _ "github.com/iawia002/lux/extractors/bcy" _ "github.com/iawia002/lux/extractors/bilibili" _ "github.com/iawia002/lux/extractors/bitchute" _ "github.com/iawia002/lux/extractors/douyin" _ "github.com/iawia002/lux/extractors/douyu" _ "github.com/iawia002/lux/extractors/eporner" _ "github.com/iawia002/lux/extractors/facebook" _ "github.com/iawia002/lux/extractors/geekbang" _ "github.com/iawia002/lux/extractors/haokan" _ "github.com/iawia002/lux/extractors/hupu" _ "github.com/iawia002/lux/extractors/huya" _ "github.com/iawia002/lux/extractors/instagram" _ "github.com/iawia002/lux/extractors/iqiyi" _ "github.com/iawia002/lux/extractors/ixigua" _ "github.com/iawia002/lux/extractors/kuaishou" _ "github.com/iawia002/lux/extractors/mgtv" _ "github.com/iawia002/lux/extractors/miaopai" _ "github.com/iawia002/lux/extractors/netease" _ "github.com/iawia002/lux/extractors/odysee" _ "github.com/iawia002/lux/extractors/pinterest" _ "github.com/iawia002/lux/extractors/pixivision" _ "github.com/iawia002/lux/extractors/pornhub" _ "github.com/iawia002/lux/extractors/qq" _ "github.com/iawia002/lux/extractors/reddit" _ "github.com/iawia002/lux/extractors/rumble" _ "github.com/iawia002/lux/extractors/streamtape" _ "github.com/iawia002/lux/extractors/tangdou" _ "github.com/iawia002/lux/extractors/threads" _ "github.com/iawia002/lux/extractors/tiktok" _ "github.com/iawia002/lux/extractors/tumblr" _ "github.com/iawia002/lux/extractors/twitter" _ "github.com/iawia002/lux/extractors/udn" _ "github.com/iawia002/lux/extractors/universal" _ "github.com/iawia002/lux/extractors/vimeo" _ "github.com/iawia002/lux/extractors/vk" _ "github.com/iawia002/lux/extractors/weibo" _ "github.com/iawia002/lux/extractors/xiaohongshu" _ "github.com/iawia002/lux/extractors/ximalaya" _ "github.com/iawia002/lux/extractors/xinpianchang" _ "github.com/iawia002/lux/extractors/xvideos" _ "github.com/iawia002/lux/extractors/yinyuetai" _ "github.com/iawia002/lux/extractors/youku" _ "github.com/iawia002/lux/extractors/youtube" _ "github.com/iawia002/lux/extractors/zhihu" _ "github.com/iawia002/lux/extractors/zingmp3" ) ================================================ FILE: codecov.yml ================================================ codecov: token: e0f2d44f-c6a7-469a-a688-37c72c0f18f9 ================================================ FILE: config/config.go ================================================ package config // FakeHeaders fake http headers var FakeHeaders = map[string]string{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Charset": "UTF-8,*;q=0.5", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "en-US,en;q=0.8", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", } ================================================ FILE: downloader/downloader.go ================================================ package downloader import ( "bytes" "encoding/binary" "encoding/json" "fmt" "io" "net/http" "os" "path" "path/filepath" "regexp" "sort" "strings" "sync" "time" "github.com/cheggaaa/pb/v3" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) // Options defines options used in downloading. type Options struct { InfoOnly bool Silent bool Stream string AudioOnly bool Refer string OutputPath string OutputName string FileNameLength int Caption bool EmbedSubtitle bool MultiThread bool ThreadNumber int RetryTimes int ChunkSizeMB int // Aria2 UseAria2RPC bool Aria2Token string Aria2Method string Aria2Addr string } // Downloader is the default downloader. type Downloader struct { Bar *pb.ProgressBar option Options } const ( DOWNLOAD_FILE_EXT = ".download" ) func progressBar(size int64) *pb.ProgressBar { tmpl := `{{counters .}} {{bar . "[" "=" ">" "-" "]"}} {{speed .}} {{percent . | green}} {{rtime .}}` return pb.New64(size). Set(pb.Bytes, true). SetMaxWidth(1000). SetTemplate(pb.ProgressBarTemplate(tmpl)) } // New returns a new Downloader implementation. func New(option Options) *Downloader { downloader := &Downloader{ option: option, } return downloader } // caption downloads danmaku, subtitles, etc func (downloader *Downloader) caption(url, fileName, ext string, transform func([]byte) ([]byte, error)) error { refer := downloader.option.Refer if refer == "" { refer = url } body, err := request.GetByte(url, refer, nil) if err != nil { return err } if transform != nil { body, err = transform(body) if err != nil { return err } } filePath, err := utils.FilePath(fileName, ext, downloader.option.FileNameLength, downloader.option.OutputPath, true) if err != nil { return err } file, fileError := os.Create(filePath) if fileError != nil { return fileError } defer file.Close() // nolint if _, err = file.Write(body); err != nil { return err } return nil } func (downloader *Downloader) writeFile(url string, file *os.File, headers map[string]string) (int64, error) { res, err := request.Request(http.MethodGet, url, nil, headers) if err != nil { return 0, err } defer res.Body.Close() // nolint barWriter := downloader.Bar.NewProxyWriter(file) // Note that io.Copy reads 32kb(maximum) from input and writes them to output, then repeats. // So don't worry about memory. written, copyErr := io.Copy(barWriter, res.Body) if copyErr != nil && copyErr != io.EOF { return written, errors.Errorf("file copy error: %s", copyErr) } return written, nil } func (downloader *Downloader) save(part *extractors.Part, refer, fileName string) error { filePath, err := utils.FilePath(fileName, part.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false) if err != nil { return err } fileSize, exists, err := utils.FileSize(filePath) if err != nil { return err } // Skip segment file // TODO: Live video URLs will not return the size if exists && fileSize == part.Size { downloader.Bar.Add64(fileSize) return nil } tempFilePath := filePath + DOWNLOAD_FILE_EXT tempFileSize, _, err := utils.FileSize(tempFilePath) if err != nil { return err } headers := map[string]string{ "Referer": refer, } var ( file *os.File fileError error ) if tempFileSize > 0 { // range start from 0, 0-1023 means the first 1024 bytes of the file headers["Range"] = fmt.Sprintf("bytes=%d-", tempFileSize) file, fileError = os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY, 0644) downloader.Bar.Add64(tempFileSize) } else { file, fileError = os.Create(tempFilePath) } if fileError != nil { return fileError } // close and rename temp file at the end of this function defer func() { // must close the file before rename or it will cause // `The process cannot access the file because it is being used by another process.` error. file.Close() // nolint if err == nil { os.Rename(tempFilePath, filePath) // nolint } }() if downloader.option.ChunkSizeMB > 0 { var start, end, chunkSize int64 chunkSize = int64(downloader.option.ChunkSizeMB) * 1024 * 1024 remainingSize := part.Size if tempFileSize > 0 { start = tempFileSize remainingSize -= tempFileSize } chunk := remainingSize / chunkSize if remainingSize%chunkSize != 0 { chunk++ } var i int64 = 1 for ; i <= chunk; i++ { end = start + chunkSize - 1 headers["Range"] = fmt.Sprintf("bytes=%d-%d", start, end) temp := start for i := 0; ; i++ { written, err := downloader.writeFile(part.URL, file, headers) if err == nil { break } else if i+1 >= downloader.option.RetryTimes { return err } temp += written headers["Range"] = fmt.Sprintf("bytes=%d-%d", temp, end) time.Sleep(1 * time.Second) } start = end + 1 } } else { temp := tempFileSize for i := 0; ; i++ { written, err := downloader.writeFile(part.URL, file, headers) if err == nil { break } else if i+1 >= downloader.option.RetryTimes { return err } temp += written headers["Range"] = fmt.Sprintf("bytes=%d-", temp) time.Sleep(1 * time.Second) } } return nil } func (downloader *Downloader) multiThreadSave(dataPart *extractors.Part, refer, fileName string) error { filePath, err := utils.FilePath(fileName, dataPart.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false) if err != nil { return err } fileSize, exists, err := utils.FileSize(filePath) if err != nil { return err } // Skip segment file // TODO: Live video URLs will not return the size if exists && fileSize == dataPart.Size { downloader.Bar.Add64(fileSize) return nil } tmpFilePath := filePath + DOWNLOAD_FILE_EXT tmpFileSize, tmpExists, err := utils.FileSize(tmpFilePath) if err != nil { return err } if tmpExists { if tmpFileSize == dataPart.Size { downloader.Bar.Add64(dataPart.Size) return os.Rename(tmpFilePath, filePath) } if err = os.Remove(tmpFilePath); err != nil { return err } } // Scan all parts parts, err := readDirAllFilePart(filePath, fileName, dataPart.Ext) if err != nil { return err } var unfinishedPart []*FilePartMeta savedSize := int64(0) if len(parts) > 0 { lastEnd := int64(-1) for i, part := range parts { // If some parts are lost, re-insert one part. if part.Start-lastEnd != 1 { newPart := &FilePartMeta{ Index: part.Index - 0.000001, Start: lastEnd + 1, End: part.Start - 1, Cur: lastEnd + 1, } tmp := append([]*FilePartMeta{}, parts[:i]...) tmp = append(tmp, newPart) parts = append(tmp, parts[i:]...) unfinishedPart = append(unfinishedPart, newPart) } // When the part has been downloaded in whole, part.Cur is equal to part.End + 1 if part.Cur <= part.End+1 { savedSize += part.Cur - part.Start if part.Cur < part.End+1 { unfinishedPart = append(unfinishedPart, part) } } else { // The size of this part has been saved greater than the part size, delete it transparently and re-download. err = os.Remove(filePartPath(filePath, part)) if err != nil { return err } part.Cur = part.Start unfinishedPart = append(unfinishedPart, part) } lastEnd = part.End } if lastEnd != dataPart.Size-1 { newPart := &FilePartMeta{ Index: parts[len(parts)-1].Index + 1, Start: lastEnd + 1, End: dataPart.Size - 1, Cur: lastEnd + 1, } parts = append(parts, newPart) unfinishedPart = append(unfinishedPart, newPart) } } else { var start, end, partSize int64 var i float32 partSize = dataPart.Size / int64(downloader.option.ThreadNumber) i = 0 for start < dataPart.Size { end = start + partSize - 1 if end > dataPart.Size { end = dataPart.Size - 1 } else if int(i+1) == downloader.option.ThreadNumber && end < dataPart.Size { end = dataPart.Size - 1 } part := &FilePartMeta{ Index: i, Start: start, End: end, Cur: start, } parts = append(parts, part) unfinishedPart = append(unfinishedPart, part) start = end + 1 i++ } } if savedSize > 0 { downloader.Bar.Add64(savedSize) if savedSize == dataPart.Size { return mergeMultiPart(filePath, parts) } } wgp := utils.NewWaitGroupPool(downloader.option.ThreadNumber) var errs []error var mu sync.Mutex for _, part := range unfinishedPart { wgp.Add() go func(part *FilePartMeta) { file, err := os.OpenFile(filePartPath(filePath, part), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) if err != nil { mu.Lock() errs = append(errs, err) mu.Unlock() return } defer func() { file.Close() // nolint wgp.Done() }() var end, chunkSize int64 headers := map[string]string{ "Referer": refer, } if downloader.option.ChunkSizeMB <= 0 { chunkSize = part.End - part.Start + 1 } else { chunkSize = int64(downloader.option.ChunkSizeMB) * 1024 * 1024 } remainingSize := part.End - part.Cur + 1 if part.Cur == part.Start { // Only write part to new file. err = writeFilePartMeta(file, part) if err != nil { mu.Lock() errs = append(errs, err) mu.Unlock() return } } for remainingSize > 0 { end = computeEnd(part.Cur, chunkSize, part.End) headers["Range"] = fmt.Sprintf("bytes=%d-%d", part.Cur, end) temp := part.Cur for i := 0; ; i++ { written, err := downloader.writeFile(dataPart.URL, file, headers) if err == nil { remainingSize -= chunkSize break } else if i+1 >= downloader.option.RetryTimes { mu.Lock() errs = append(errs, err) mu.Unlock() return } temp += written headers["Range"] = fmt.Sprintf("bytes=%d-%d", temp, end) } part.Cur = end + 1 } }(part) } wgp.Wait() if len(errs) > 0 { return errs[0] } return mergeMultiPart(filePath, parts) } func filePartPath(filepath string, part *FilePartMeta) string { return fmt.Sprintf("%s.part%f", filepath, part.Index) } func computeEnd(s, chunkSize, max int64) int64 { var end int64 end = s + chunkSize - 1 if end > max { end = max } return end } func readDirAllFilePart(filePath, filename, extname string) ([]*FilePartMeta, error) { dirPath := filepath.Dir(filePath) dir, err := os.Open(dirPath) if err != nil { return nil, errors.WithStack(err) } defer dir.Close() // nolint fns, err := dir.Readdir(0) if err != nil { return nil, errors.WithStack(err) } var metas []*FilePartMeta reg := regexp.MustCompile(fmt.Sprintf("%s.%s.part.+", regexp.QuoteMeta(filename), extname)) for _, fn := range fns { if reg.MatchString(fn.Name()) { meta, err := parseFilePartMeta(path.Join(dirPath, fn.Name()), fn.Size()) if err != nil { return nil, errors.WithStack(err) } metas = append(metas, meta) } } sort.SliceStable(metas, func(i, j int) bool { return metas[i].Index < metas[j].Index }) return metas, nil } func parseFilePartMeta(filepath string, fileSize int64) (*FilePartMeta, error) { meta := new(FilePartMeta) size := binary.Size(*meta) file, err := os.OpenFile(filepath, os.O_RDWR, 0666) if err != nil { return nil, errors.WithStack(err) } defer file.Close() // nolint var buf [512]byte readSize, err := file.ReadAt(buf[0:size], 0) if err != nil && err != io.EOF { return nil, errors.WithStack(err) } if readSize < size { return nil, errors.Errorf("the file has been broken, please delete all part files and re-download") } err = binary.Read(bytes.NewBuffer(buf[:size]), binary.LittleEndian, meta) if err != nil { return nil, errors.WithStack(err) } savedSize := fileSize - int64(binary.Size(meta)) meta.Cur = meta.Start + savedSize return meta, nil } func writeFilePartMeta(file *os.File, meta *FilePartMeta) error { return binary.Write(file, binary.LittleEndian, meta) } func mergeMultiPart(filepath string, parts []*FilePartMeta) error { tempFilePath := filepath + DOWNLOAD_FILE_EXT tempFile, err := os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err } var partFiles []*os.File defer func() { for _, f := range partFiles { f.Close() // nolint os.Remove(f.Name()) // nolint } }() for _, part := range parts { file, err := os.Open(filePartPath(filepath, part)) if err != nil { return err } partFiles = append(partFiles, file) _, err = file.Seek(int64(binary.Size(part)), 0) if err != nil { return err } _, err = io.Copy(tempFile, file) if err != nil { return err } } tempFile.Close() // nolint err = os.Rename(tempFilePath, filepath) return err } func (downloader *Downloader) aria2(title string, stream *extractors.Stream) error { rpcData := Aria2RPCData{ JSONRPC: "2.0", ID: "lux", // can be modified Method: "aria2.addUri", } rpcData.Params[0] = "token:" + downloader.option.Aria2Token var urls []string for _, p := range stream.Parts { urls = append(urls, p.URL) } var inputs Aria2Input inputs.Header = append(inputs.Header, "Referer: "+downloader.option.Refer) for i := range urls { rpcData.Params[1] = urls[i : i+1] inputs.Out = fmt.Sprintf("%s[%d].%s", title, i, stream.Parts[0].Ext) rpcData.Params[2] = &inputs jsonData, err := json.Marshal(rpcData) if err != nil { return err } reqURL := fmt.Sprintf("%s://%s/jsonrpc", downloader.option.Aria2Method, downloader.option.Aria2Addr) req, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewBuffer(jsonData)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") var client = http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) if err != nil { return err } // The http Client and Transport guarantee that Body is always // non-nil, even on responses without a body or responses with // a zero-length body. res.Body.Close() // nolint } return nil } // Download download urls func (downloader *Downloader) Download(data *extractors.Data) error { if len(data.Streams) == 0 { return errors.Errorf("no streams in title %s", data.Title) } sortedStreams := genSortedStreams(data.Streams) if downloader.option.InfoOnly { printInfo(data, sortedStreams) return nil } title := downloader.option.OutputName if title == "" { title = data.Title } title = utils.FileName(title, "", downloader.option.FileNameLength) streamName := downloader.option.Stream if streamName == "" { streamName = sortedStreams[0].ID } stream, ok := data.Streams[streamName] if !ok { return errors.Errorf("no stream named %s", streamName) } if downloader.option.AudioOnly { var isFound bool reg, err := regexp.Compile("audio+") if err != nil { return err } for _, s := range sortedStreams { // Looking for the best quality if reg.MatchString(s.Quality) { isFound = true stream = data.Streams[s.ID] break } for _, part := range s.Parts { if part.Ext == "m4a" { isFound = true stream = data.Streams[s.ID] break } } } if !isFound { return errors.Errorf("No audio stream found") } } if !downloader.option.Silent { printStreamInfo(data, stream) } // download caption var subtitlePaths []string var subtitleLangs []string var subtitleFilesToDelete []string if downloader.option.Caption && data.Captions != nil { fmt.Println("\nDownloading captions...") for k, v := range data.Captions { if v != nil { fmt.Printf("Downloading %s ...\n", k) if err := downloader.caption(v.URL, title, v.Ext, v.Transform); err != nil { // nolint } else if downloader.option.EmbedSubtitle { subtitlePath, _ := utils.FilePath(title, v.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, true) subtitleFilesToDelete = append(subtitleFilesToDelete, subtitlePath) if strings.HasSuffix(v.Ext, "xml") { if srtPath, err := utils.ConvertXMLFileToSRT(subtitlePath); err == nil { subtitlePath = srtPath subtitleFilesToDelete = append(subtitleFilesToDelete, srtPath) } } subtitlePaths = append(subtitlePaths, subtitlePath) subtitleLangs = append(subtitleLangs, k) } } } } // Use aria2 rpc to download if downloader.option.UseAria2RPC { return downloader.aria2(title, stream) } // Skip the complete file that has been merged mergedFilePath, err := utils.FilePath(title, stream.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false) if err != nil { return err } _, mergedFileExists, err := utils.FileSize(mergedFilePath) if err != nil { return err } // After the merge, the file size has changed, so we do not check whether the size matches if mergedFileExists { fmt.Printf("%s: file already exists, skipping\n", mergedFilePath) return nil } downloader.Bar = progressBar(stream.Size) if !downloader.option.Silent { downloader.Bar.Start() } if len(stream.Parts) == 1 { // only one fragment var err error if downloader.option.MultiThread { err = downloader.multiThreadSave(stream.Parts[0], data.URL, title) } else { err = downloader.save(stream.Parts[0], data.URL, title) } if err != nil { return err } downloader.Bar.Finish() if downloader.option.EmbedSubtitle && len(subtitlePaths) > 0 { if !downloader.option.Silent { fmt.Println("Embedding subtitles...") } if err := utils.EmbedSubtitles(mergedFilePath, subtitlePaths, subtitleLangs); err != nil { return err } for _, path := range subtitleFilesToDelete { os.Remove(path) } } return nil } wgp := utils.NewWaitGroupPool(downloader.option.ThreadNumber) // multiple fragments errs := make([]error, 0) lock := sync.Mutex{} parts := make([]string, len(stream.Parts)) for index, part := range stream.Parts { if len(errs) > 0 { break } if downloader.option.AudioOnly && (part.Ext != "m4a") { continue } partFileName := fmt.Sprintf("%s[%d]", title, index) partFilePath, err := utils.FilePath(partFileName, part.Ext, downloader.option.FileNameLength, downloader.option.OutputPath, false) if err != nil { return err } parts[index] = partFilePath wgp.Add() go func(part *extractors.Part, fileName string) { defer wgp.Done() var err error if downloader.option.MultiThread { err = downloader.multiThreadSave(part, data.URL, fileName) } else { err = downloader.save(part, data.URL, fileName) } if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() } }(part, partFileName) } wgp.Wait() if len(errs) > 0 { return errs[0] } downloader.Bar.Finish() if data.Type != extractors.DataTypeVideo || downloader.option.AudioOnly { return nil } if !downloader.option.Silent { fmt.Printf("Merging video parts into %s\n", mergedFilePath) } if stream.Ext != "mp4" || stream.NeedMux { if err := utils.MergeFilesWithSameExtension(parts, mergedFilePath); err != nil { return err } } else { if err := utils.MergeToMP4(parts, mergedFilePath, title); err != nil { return err } } if downloader.option.EmbedSubtitle && len(subtitlePaths) > 0 { if !downloader.option.Silent { fmt.Println("Embedding subtitles...") } if err := utils.EmbedSubtitles(mergedFilePath, subtitlePaths, subtitleLangs); err != nil { return err } for _, path := range subtitleFilesToDelete { os.Remove(path) } } return nil } ================================================ FILE: downloader/downloader_test.go ================================================ package downloader import ( "testing" "github.com/iawia002/lux/extractors" ) func TestDownload(t *testing.T) { testCases := []struct { name string data *extractors.Data }{ { name: "normal test", data: &extractors.Data{ Site: "douyin", Title: "test", Type: extractors.DataTypeVideo, URL: "https://www.douyin.com", Streams: map[string]*extractors.Stream{ "default": { ID: "default", Parts: []*extractors.Part{ { URL: "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f9a0000bc117isuatl67cees890&line=0", Size: 4927877, Ext: "mp4", }, }, }, }, }, }, { name: "multi-stream test", data: &extractors.Data{ Site: "douyin", Title: "test2", Type: extractors.DataTypeVideo, URL: "https://www.douyin.com", Streams: map[string]*extractors.Stream{ "miaopai": { ID: "miaopai", Parts: []*extractors.Part{ { URL: "https://txycdn.miaopai.com/stream/KwR26jUGh2ySnVjYbQiFmomNjP14LtMU3vi6sQ__.mp4?ssig=6594aa01a78e78f50c65c164d186ba9e&time_stamp=1537070910786", Size: 4011590, Ext: "mp4", }, }, Size: 4011590, }, "douyin": { ID: "douyin", Parts: []*extractors.Part{ { URL: "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f9a0000bc117isuatl67cees890&line=0", Size: 4927877, Ext: "mp4", }, }, Size: 4927877, }, }, }, }, { name: "image test", data: &extractors.Data{ Site: "bcy", Title: "bcy image test", Type: extractors.DataTypeImage, URL: "https://www.bcyimg.com", Streams: map[string]*extractors.Stream{ "default": { ID: "default", Parts: []*extractors.Part{ { URL: "http://img5.bcyimg.com/coser/143767/post/c0j7x/0d713eb41a614053ac6a3b146914f6bc.jpg/w650", Size: 56107, Ext: "jpg", }, { URL: "http://img9.bcyimg.com/coser/143767/post/c0j7x/d17e9b8587794d939a1363c5f715014b.jpg/w650", Size: 142100, Ext: "jpg", }, }, }, }, }, }, } for _, testCase := range testCases { err := New(Options{}).Download(testCase.data) if err != nil { t.Error(err) } } } ================================================ FILE: downloader/types.go ================================================ package downloader // Aria2RPCData defines the data structure of json RPC 2.0 info for Aria2 type Aria2RPCData struct { // More info about RPC interface please refer to // https://aria2.github.io/manual/en/html/aria2c.html#rpc-interface JSONRPC string `json:"jsonrpc"` ID string `json:"id"` // For a simple download, only implemented `addUri` Method string `json:"method"` // secret, uris, options Params [3]interface{} `json:"params"` } // Aria2Input is options for `aria2.addUri` // https://aria2.github.io/manual/en/html/aria2c.html#id3 type Aria2Input struct { // The file name of the downloaded file Out string `json:"out"` // For a simple download, only add headers Header []string `json:"header"` } // FilePartMeta defines the data structure of file meta info. type FilePartMeta struct { Index float32 Start int64 End int64 Cur int64 } ================================================ FILE: downloader/utils.go ================================================ package downloader import ( "fmt" "sort" "github.com/fatih/color" "github.com/iawia002/lux/extractors" ) var ( blue = color.New(color.FgBlue) cyan = color.New(color.FgCyan) ) func genSortedStreams(streams map[string]*extractors.Stream) []*extractors.Stream { sortedStreams := make([]*extractors.Stream, 0, len(streams)) for _, data := range streams { sortedStreams = append(sortedStreams, data) } if len(sortedStreams) > 1 { sort.SliceStable( sortedStreams, func(i, j int) bool { return sortedStreams[i].Size > sortedStreams[j].Size }, ) } return sortedStreams } func printHeader(data *extractors.Data) { fmt.Println() cyan.Printf(" Site: ") // nolint fmt.Println(data.Site) cyan.Printf(" Title: ") // nolint fmt.Println(data.Title) cyan.Printf(" Type: ") // nolint fmt.Println(data.Type) } func printStream(stream *extractors.Stream) { blue.Println(fmt.Sprintf(" [%s] -------------------", stream.ID)) // nolint if stream.Quality != "" { cyan.Printf(" Quality: ") // nolint fmt.Println(stream.Quality) } cyan.Printf(" Size: ") // nolint fmt.Printf("%.2f MiB (%d Bytes)\n", float64(stream.Size)/(1024*1024), stream.Size) cyan.Printf(" # download with: ") // nolint fmt.Printf("lux -f %s ...\n\n", stream.ID) } func printInfo(data *extractors.Data, sortedStreams []*extractors.Stream) { printHeader(data) if len(data.Captions) > 0 { cyan.Printf(" Captions: ") // nolint languages := make([]string, 0, len(data.Captions)) for lang := range data.Captions { languages = append(languages, lang) } sort.Strings(languages) captionList := "" for _, lang := range languages { caption := data.Captions[lang] if caption == nil { continue } captionList += fmt.Sprintf("%s ", lang) } fmt.Println(captionList) } cyan.Printf(" Streams: ") // nolint fmt.Println("# All available quality") for _, stream := range sortedStreams { printStream(stream) } } func printStreamInfo(data *extractors.Data, stream *extractors.Stream) { printHeader(data) cyan.Printf(" Stream: ") // nolint fmt.Println() printStream(stream) } ================================================ FILE: extractors/acfun/acfun.go ================================================ package acfun import ( "fmt" "net/url" "regexp" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("acfun", New()) } const ( bangumiDataPattern = "window.pageInfo = window.bangumiData = (.*);" bangumiListPattern = "window.bangumiList = (.*);" bangumiHTMLURL = "https://www.acfun.cn/bangumi/aa%d_36188_%d" referer = "https://www.acfun.cn" ) type extractor struct{} // New returns a new acfun bangumi extractor func New() extractors.Extractor { return &extractor{} } // Extract ... func (e *extractor) Extract(URL string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.GetByte(URL, referer, nil) if err != nil { return nil, errors.WithStack(err) } epDatas := make([]*episodeData, 0) if option.Playlist { list, err := resolvingEpisodes(html) if err != nil { return nil, errors.WithStack(err) } items := utils.NeedDownloadList(option.Items, option.ItemStart, option.ItemEnd, len(list.Episodes)) for _, item := range items { epDatas = append(epDatas, list.Episodes[item-1]) } } else { bgData, _, err := resolvingData(html) if err != nil { return nil, errors.WithStack(err) } epDatas = append(epDatas, &bgData.episodeData) } datas := make([]*extractors.Data, 0) wgp := utils.NewWaitGroupPool(option.ThreadNumber) for _, epData := range epDatas { t := epData wgp.Add() go func() { defer wgp.Done() datas = append(datas, extractBangumi(concatURL(t))) }() } wgp.Wait() return datas, nil } func concatURL(epData *episodeData) string { return fmt.Sprintf(bangumiHTMLURL, epData.BangumiID, epData.ItemID) } func extractBangumi(URL string) *extractors.Data { var err error html, err := request.GetByte(URL, referer, nil) if err != nil { return extractors.EmptyData(URL, err) } _, vInfo, err := resolvingData(html) if err != nil { return extractors.EmptyData(URL, err) } streams := make(map[string]*extractors.Stream) for _, stm := range vInfo.AdaptationSet[0].Streams { m3u8URL, err := url.Parse(stm.URL) if err != nil { return extractors.EmptyData(URL, err) } urls, err := utils.M3u8URLs(m3u8URL.String()) if err != nil { _, err = url.Parse(stm.URL) if err != nil { return extractors.EmptyData(URL, err) } urls, err = utils.M3u8URLs(stm.BackURL) if err != nil { return extractors.EmptyData(URL, err) } } // There is no size information in the m3u8 file and the calculation will take too much time, just ignore it. parts := make([]*extractors.Part, 0) for _, u := range urls { parts = append(parts, &extractors.Part{ URL: u, Ext: "ts", }) } streams[stm.QualityLabel] = &extractors.Stream{ ID: stm.QualityType, Parts: parts, Quality: stm.QualityType, NeedMux: false, } } doc, err := parser.GetDoc(string(html)) if err != nil { return extractors.EmptyData(URL, err) } data := &extractors.Data{ Site: "AcFun acfun.cn", Title: parser.Title(doc), Type: extractors.DataTypeVideo, Streams: streams, URL: URL, } return data } func resolvingData(html []byte) (*bangumiData, *videoInfo, error) { bgData := &bangumiData{} vInfo := &videoInfo{} pattern, _ := regexp.Compile(bangumiDataPattern) groups := pattern.FindSubmatch(html) err := jsoniter.Unmarshal(groups[1], bgData) if err != nil { return nil, nil, errors.WithStack(err) } err = jsoniter.UnmarshalFromString(bgData.CurrentVideoInfo.KsPlayJSON, vInfo) if err != nil { return nil, nil, errors.WithStack(err) } return bgData, vInfo, nil } func resolvingEpisodes(html []byte) (*episodeList, error) { list := &episodeList{} pattern, _ := regexp.Compile(bangumiListPattern) groups := pattern.FindSubmatch(html) err := jsoniter.Unmarshal(groups[1], list) if err != nil { return nil, errors.WithStack(err) } return list, nil } ================================================ FILE: extractors/acfun/acfun_test.go ================================================ package acfun import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.acfun.cn/bangumi/aa6000686_36188_1704167", Title: "瑞克和莫蒂 第四季 ：第2话 注释版", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/acfun/types.go ================================================ package acfun type episodeData struct { ItemID int64 `json:"itemId"` EpisodeName string `json:"episodeName"` BangumiID int64 `json:"bangumiId"` VideoID int64 `json:"videoId"` } type bangumiData struct { episodeData BangumiTitle string `json:"bangumiTitle"` CurrentVideoInfo struct { KsPlayJSON string `json:"ksPlayJson"` } `json:"currentVideoInfo"` } type videoInfo struct { AdaptationSet []struct { Streams streams `json:"representation"` } `json:"adaptationSet"` } type streams []stream type episodeList struct { Episodes []*episodeData `json:"items"` } type stream struct { ID int64 `json:"id"` BackURL string `json:"backUrl"` Codecs string `json:"codecs"` URL string `json:"url"` BitRate int64 `json:"avgBitrate"` QualityType string `json:"qualityType"` QualityLabel string `json:"qualityLabel"` } ================================================ FILE: extractors/bcy/bcy.go ================================================ package bcy import ( "encoding/json" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("bcy", New()) } type bcyData struct { Detail struct { PostData struct { Multi []struct { OriginalPath string `json:"original_path"` } `json:"multi"` } `json:"post_data"` } `json:"detail"` } type extractor struct{} // New returns a bcy extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } // parse json data rep := strings.NewReplacer(`\"`, `"`, `\\`, `\`) realURLs := utils.MatchOneOf(html, `JSON.parse\("(.+?)"\);`) if realURLs == nil || len(realURLs) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } jsonString := rep.Replace(realURLs[1]) var data bcyData if err = json.Unmarshal([]byte(jsonString), &data); err != nil { return nil, errors.WithStack(err) } doc, err := parser.GetDoc(html) if err != nil { return nil, errors.WithStack(err) } title := strings.Replace(parser.Title(doc), " - 半次元 banciyuan - ACG爱好者社区", "", -1) parts := make([]*extractors.Part, 0, len(data.Detail.PostData.Multi)) var totalSize int64 for _, img := range data.Detail.PostData.Multi { size, err := request.Size(img.OriginalPath, url) if err != nil { return nil, errors.WithStack(err) } totalSize += size _, ext, err := utils.GetNameAndExt(img.OriginalPath) if err != nil { return nil, errors.WithStack(err) } parts = append(parts, &extractors.Part{ URL: img.OriginalPath, Size: size, Ext: ext, }) } streams := map[string]*extractors.Stream{ "default": { Parts: parts, Size: totalSize, }, } return []*extractors.Data{ { Site: "半次元 bcy.net", Title: title, Type: extractors.DataTypeImage, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/bcy/bcy_test.go ================================================ package bcy import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://bcy.net/item/detail/6558738153367142664", Title: "cos正片 命运石之门 牧濑红莉栖 克里斯蒂娜… - 半次元 - ACG爱好者社区", Size: 13035763, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/bilibili/bilibili.go ================================================ package bilibili import ( "encoding/json" "fmt" "slices" "sort" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { bilibiliExtractor := New() extractors.Register("bilibili", bilibiliExtractor) extractors.Register("b23", bilibiliExtractor) } const ( bilibiliAPI = "https://api.bilibili.com/x/player/playurl?" bilibiliBangumiAPI = "https://api.bilibili.com/pgc/player/web/playurl?" bilibiliTokenAPI = "https://api.bilibili.com/x/player/playurl/token?" ) const referer = "https://www.bilibili.com" var utoken string func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (string, error) { var ( err error baseAPIURL string params string ) if cookie != "" && utoken == "" { utoken, err = request.Get( fmt.Sprintf("%said=%d&cid=%d", bilibiliTokenAPI, aid, cid), referer, nil, ) if err != nil { return "", err } var t token err = json.Unmarshal([]byte(utoken), &t) if err != nil { return "", err } if t.Code != 0 { return "", errors.Errorf("cookie error: %s", t.Message) } utoken = t.Data.Token } var api string if bangumi { // The parameters need to be sorted by name // qn=0 flag makes the CDN address different every time // quality=120(4k) is the highest quality so far params = fmt.Sprintf( "cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16", cid, bvid, quality, ) baseAPIURL = bilibiliBangumiAPI } else { params = fmt.Sprintf( "avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=2000", aid, cid, bvid, quality, ) baseAPIURL = bilibiliAPI } api = baseAPIURL + params // bangumi utoken also need to put in params to sign, but the ordinary video doesn't need if !bangumi && utoken != "" { api = fmt.Sprintf("%s&utoken=%s", api, utoken) } return api, nil } type bilibiliOptions struct { url string html string bangumi bool aid int cid int bvid string page int subtitle string } func extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { dataString := utils.MatchOneOf(html, `const playurlSSRData = ({[\s\S]+})`)[1] epArrayString := utils.MatchOneOf(dataString, `"episode_info"\s*:\s*(.+?)\s*,\s*"season_info"`)[1] fullVideoIdString := utils.MatchOneOf(dataString, `"videoId"\s*:\s*"(ep|ss)(\d+)"`) epSsString := fullVideoIdString[1] // "ep" or "ss" videoIdString := fullVideoIdString[2] var epArray EpVideoInfo err := json.Unmarshal([]byte(epArrayString), &epArray) if err != nil { return nil, errors.WithStack(err) } var data bangumiData videoId, err := strconv.ParseInt(videoIdString, 10, 0) if err != nil { return nil, errors.WithStack(err) } if epArray.EpID == int(videoId) || (epSsString == "ss" && epArray.Title == "第1话") { data.EpInfo = epArray } data.EpList = append(data.EpList, epArray) sort.Slice(data.EpList, func(i, j int) bool { return data.EpList[i].EpID < data.EpList[j].EpID }) if !extractOption.Playlist { aid := data.EpInfo.Aid cid := data.EpInfo.Cid bvid := data.EpInfo.Bvid titleFormat := data.EpInfo.Title longTitle := data.EpInfo.LongTitle if aid <= 0 || cid <= 0 || bvid == "" { aid = data.EpList[0].Aid cid = data.EpList[0].Cid bvid = data.EpList[0].Bvid titleFormat = data.EpList[0].Title longTitle = data.EpList[0].LongTitle } options := bilibiliOptions{ url: url, html: html, bangumi: true, aid: aid, cid: cid, bvid: bvid, subtitle: fmt.Sprintf("%s %s", titleFormat, longTitle), } return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil } // handle bangumi playlist needDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(data.EpList)) extractedData := make([]*extractors.Data, len(needDownloadItems)) wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range data.EpList { if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() id := u.EpID if id == 0 { id = u.EpID } // html content can't be reused here options := bilibiliOptions{ url: fmt.Sprintf("https://www.bilibili.com/bangumi/play/ep%d", id), bangumi: true, aid: u.Aid, cid: u.Cid, bvid: u.Bvid, subtitle: fmt.Sprintf("%s %s", u.Title, u.LongTitle), } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() extractedData[index] = bilibiliDownload(options, extractOption) }(dataIndex, options, extractedData) dataIndex++ } wgp.Wait() return extractedData, nil } func getMultiPageData(html string) (*multiPage, error) { var data multiPage multiPageDataString := utils.MatchOneOf( html, `window.__INITIAL_STATE__=(.+?);\(function`, ) if multiPageDataString == nil { return &data, errors.New("this page has no playlist") } err := json.Unmarshal([]byte(multiPageDataString[1]), &data) if err != nil { return nil, errors.WithStack(err) } return &data, nil } func extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { matches := utils.MatchAll(html, "<\\s*script[^>]*>\\s*window\\.__INITIAL_STATE__=([\\s\\S]*?);\\s?\\(function[\\s\\S]*?<\\/\\s*script\\s*>") if len(matches) < 1 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } if len(matches[0]) < 2 { return nil, errors.New("could not find video in page") } var festivalData festival err := json.Unmarshal([]byte(matches[0][1]), &festivalData) if err != nil { return nil, errors.WithStack(err) } options := bilibiliOptions{ url: url, html: html, aid: festivalData.VideoInfo.Aid, bvid: festivalData.VideoInfo.BVid, cid: festivalData.VideoInfo.Cid, page: 0, } return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil } func extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { pageData, err := getMultiPageData(html) if err != nil { return nil, errors.WithStack(err) } if !extractOption.Playlist { // handle URL that has a playlist, mainly for unified titles //

tag does not include subtitles // bangumi doesn't need this pageString := utils.MatchOneOf(url, `\?p=(\d+)`) var p int if pageString == nil { // https://www.bilibili.com/video/av20827366/ p = 1 } else { // https://www.bilibili.com/video/av20827366/?p=2 p, _ = strconv.Atoi(pageString[1]) } if len(pageData.VideoData.Pages) < p || p < 1 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } page := pageData.VideoData.Pages[p-1] options := bilibiliOptions{ url: url, html: html, aid: pageData.Aid, bvid: pageData.BVid, cid: page.Cid, page: p, } // "part":"" or "part":"Untitled" if page.Part == "Untitled" || len(pageData.VideoData.Pages) == 1 { options.subtitle = "" } else { options.subtitle = page.Part } return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil } // handle normal video playlist if len(pageData.Sections) == 0 { // https://www.bilibili.com/video/av20827366/?p=* each video in playlist has different p=? return multiPageDownload(url, html, extractOption, pageData) } // handle another kind of playlist // https://www.bilibili.com/video/av*** each video in playlist has different av/bv id return multiEpisodeDownload(url, html, extractOption, pageData) } // handle multi episode download func multiEpisodeDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) { needDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.Sections[0].Episodes)) extractedData := make([]*extractors.Data, len(needDownloadItems)) wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range pageData.Sections[0].Episodes { if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() options := bilibiliOptions{ url: url, html: html, aid: u.Aid, bvid: u.BVid, cid: u.Cid, subtitle: fmt.Sprintf("%s P%d", u.Title, index+1), } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() extractedData[index] = bilibiliDownload(options, extractOption) }(dataIndex, options, extractedData) dataIndex++ } wgp.Wait() return extractedData, nil } // handle multi page download func multiPageDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) { needDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.VideoData.Pages)) extractedData := make([]*extractors.Data, len(needDownloadItems)) wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range pageData.VideoData.Pages { if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() options := bilibiliOptions{ url: url, html: html, aid: pageData.Aid, bvid: pageData.BVid, cid: u.Cid, subtitle: u.Part, page: u.Page, } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() extractedData[index] = bilibiliDownload(options, extractOption) }(dataIndex, options, extractedData) dataIndex++ } wgp.Wait() return extractedData, nil } type extractor struct{} // New returns a bilibili extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { var err error html, err := request.Get(url, referer, nil) if err != nil { return nil, errors.WithStack(err) } // set thread number to 1 manually to avoid http 412 error option.ThreadNumber = 1 if strings.Contains(url, "bangumi") { // handle bangumi return extractBangumi(url, html, option) } else if strings.Contains(url, "festival") { return extractFestival(url, html, option) } else { // handle normal video return extractNormalVideo(url, html, option) } } // bilibiliDownload is the download function for a single URL func bilibiliDownload(options bilibiliOptions, extractOption extractors.Options) *extractors.Data { var ( err error html string ) if options.html != "" { // reuse html string, but this can't be reused in case of playlist html = options.html } else { html, err = request.Get(options.url, referer, nil) if err != nil { return extractors.EmptyData(options.url, err) } } // Get "accept_quality" and "accept_description" // "accept_description":["超高清 8K","超清 4K","高清 1080P+","高清 1080P","高清 720P","清晰 480P","流畅 360P"], // "accept_quality":[127，120,112,80,48,32,16], api, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie) if err != nil { return extractors.EmptyData(options.url, err) } jsonString, err := request.Get(api, referer, nil) if err != nil { return extractors.EmptyData(options.url, err) } var data dash err = json.Unmarshal([]byte(jsonString), &data) if err != nil { return extractors.EmptyData(options.url, err) } var dashData dashInfo if data.Data.Description == nil { dashData = data.Result } else { dashData = data.Data } var audioPart *extractors.Part if dashData.Streams.Audio != nil { // Get audio part var audioID int audios := map[int]string{} bandwidth := 0 for _, stream := range dashData.Streams.Audio { if stream.Bandwidth > bandwidth { audioID = stream.ID bandwidth = stream.Bandwidth } audios[stream.ID] = stream.BaseURL } s, err := request.Size(audios[audioID], referer) if err != nil { return extractors.EmptyData(options.url, err) } audioPart = &extractors.Part{ URL: audios[audioID], Size: s, Ext: "m4a", } } streams := make(map[string]*extractors.Stream, len(dashData.Quality)) for _, stream := range dashData.Streams.Video { s, err := request.Size(stream.BaseURL, referer) if err != nil { return extractors.EmptyData(options.url, err) } parts := make([]*extractors.Part, 0, 2) parts = append(parts, &extractors.Part{ URL: stream.BaseURL, Size: s, Ext: getExtFromMimeType(stream.MimeType), }) if audioPart != nil { parts = append(parts, audioPart) } var size int64 for _, part := range parts { size += part.Size } id := fmt.Sprintf("%d-%d", stream.ID, stream.Codecid) streams[id] = &extractors.Stream{ Parts: parts, Size: size, Quality: fmt.Sprintf("%s %s", qualityString[stream.ID], stream.Codecs), } if audioPart != nil { streams[id].NeedMux = true } } for _, durl := range dashData.DURLs { var ext string switch dashData.DURLFormat { case "flv", "flv480": ext = "flv" case "mp4", "hdmp4": // nolint ext = "mp4" } parts := make([]*extractors.Part, 0, 1) parts = append(parts, &extractors.Part{ URL: durl.URL, Size: durl.Size, Ext: ext, }) streams[strconv.Itoa(dashData.CurQuality)] = &extractors.Stream{ Parts: parts, Size: durl.Size, Quality: qualityString[dashData.CurQuality], } } // get the title doc, err := parser.GetDoc(html) if err != nil { return extractors.EmptyData(options.url, err) } title := parser.Title(doc) if options.subtitle != "" { pageString := "" if options.page > 0 { pageString = fmt.Sprintf("P%d ", options.page) } if extractOption.EpisodeTitleOnly { title = fmt.Sprintf("%s%s", pageString, options.subtitle) } else { title = fmt.Sprintf("%s %s%s", title, pageString, options.subtitle) } } return &extractors.Data{ Site: "哔哩哔哩 bilibili.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, Captions: map[string]*extractors.CaptionPart{ "danmaku": { Part: extractors.Part{ URL: fmt.Sprintf("https://comment.bilibili.com/%d.xml", options.cid), Ext: "xml", }, }, "subtitle": getSubTitleCaptionPart(options.aid, options.cid), }, URL: options.url, } } func getExtFromMimeType(mimeType string) string { exts := strings.Split(mimeType, "/") if len(exts) == 2 { return exts[1] } return "mp4" } func getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart { jsonString, err := request.Get( fmt.Sprintf("http://api.bilibili.com/x/player/wbi/v2?aid=%d&cid=%d", aid, cid), referer, nil, ) if err != nil { return nil } stu := bilibiliWebInterface{} err = json.Unmarshal([]byte(jsonString), &stu) if err != nil || len(stu.Data.SubtitleInfo.SubtitleList) == 0 { return nil } return &extractors.CaptionPart{ Part: extractors.Part{ URL: fmt.Sprintf("https:%s", stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl), Ext: "srt", }, Transform: subtitleTransform, } } func subtitleTransform(body []byte) ([]byte, error) { bytes := "" captionData := bilibiliSubtitleFormat{} err := json.Unmarshal(body, &captionData) if err != nil { return nil, errors.WithStack(err) } for i := 0; i < len(captionData.Body); i++ { bytes += fmt.Sprintf("%d\n%s --> %s\n%s\n\n", i, time.Unix(0, int64(captionData.Body[i].From*1000)*int64(time.Millisecond)).UTC().Format("15:04:05.000"), time.Unix(0, int64(captionData.Body[i].To*1000)*int64(time.Millisecond)).UTC().Format("15:04:05.000"), captionData.Body[i].Content, ) } return []byte(bytes), nil } ================================================ FILE: extractors/bilibili/bilibili_test.go ================================================ package bilibili import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestBilibili(t *testing.T) { tests := []struct { name string args test.Args playlist bool }{ { name: "normal test 1", args: test.Args{ URL: "https://www.bilibili.com/video/av20203945/", Title: "【2018拜年祭单品】相遇day by day", }, playlist: false, }, { name: "normal test 2", args: test.Args{ URL: "https://www.bilibili.com/video/av41301960", Title: "【英雄联盟】2019赛季CG 《觉醒》", }, playlist: false, }, { name: "bangumi test", args: test.Args{ URL: "https://www.bilibili.com/bangumi/play/ep167000", Title: "狐妖小红娘 第70话 苏苏智商上线", }, }, { name: "bangumi playlist test", args: test.Args{ URL: "https://www.bilibili.com/bangumi/play/ss5050", Title: "一人之下：第1话 异人刀兵起，道炁携阴阳", }, playlist: true, }, { name: "playlist test", args: test.Args{ URL: "https://www.bilibili.com/video/av16907446/", Title: "\"不要相信歌词，他们为了押韵什么都干得出来\"", }, playlist: true, }, { name: "8k test", args: test.Args{ URL: "https://www.bilibili.com/video/BV1qM4y1w716", Title: "【8K演示片】B站首发！你的设备还顶得住吗？", }, }, { name: "b23 test", args: test.Args{ URL: "https://b23.tv/Fc9i7QF", Title: "【十年榜】2000-2009年最强华语金曲TOP100 P1 100爱转角-罗志祥", }, }, { name: "festival test", args: test.Args{ URL: "https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt", Title: "洛天依十周年官方演唱会", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var ( data []*extractors.Data err error ) if tt.playlist { // for playlist, we don't check the data _, err = New().Extract(tt.args.URL, extractors.Options{ Playlist: true, ThreadNumber: 9, }) test.CheckError(t, err) } else { data, err = New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) } }) } } ================================================ FILE: extractors/bilibili/types.go ================================================ package bilibili // {"code":0,"message":"0","ttl":1,"data":{"token":"aaa"}} // {"code":-101,"message":"账号未登录","ttl":1} type tokenData struct { Token string `json:"token"` } type token struct { Code int `json:"code"` Message string `json:"message"` Data tokenData `json:"data"` } type Interaction struct { Interaction bool `json:"interaction"` } type EpVideoInfo struct { Aid int `json:"aid"` Bvid string `json:"bvid"` Cid int `json:"cid"` DeliveryBusinessFragmentVideo bool `json:"delivery_business_fragment_video"` DeliveryFragmentVideo bool `json:"delivery_fragment_video"` EpID int `json:"ep_id"` EpStatus int `json:"ep_status"` Interaction Interaction `json:"interaction"` LongTitle string `json:"long_title"` Title string `json:"title"` } type bangumiData struct { EpInfo EpVideoInfo `json:"epInfo"` EpList []EpVideoInfo `json:"epList"` } type videoPagesData struct { Cid int `json:"cid"` Part string `json:"part"` Page int `json:"page"` } type multiPageVideoData struct { Title string `json:"title"` Pages []videoPagesData `json:"pages"` } type episode struct { Aid int `json:"aid"` Cid int `json:"cid"` Title string `json:"title"` BVid string `json:"bvid"` } type multiEpisodeData struct { Seasionid int `json:"season_id"` Episodes []episode `json:"episodes"` } type multiPage struct { Aid int `json:"aid"` BVid string `json:"bvid"` Sections []multiEpisodeData `json:"sections"` VideoData multiPageVideoData `json:"videoData"` } type dashStream struct { ID int `json:"id"` BaseURL string `json:"baseUrl"` Bandwidth int `json:"bandwidth"` MimeType string `json:"mimeType"` Codecid int `json:"codecid"` Codecs string `json:"codecs"` } type dashStreams struct { Video []dashStream `json:"video"` Audio []dashStream `json:"audio"` } type dashInfo struct { CurQuality int `json:"quality"` Description []string `json:"accept_description"` Quality []int `json:"accept_quality"` Streams dashStreams `json:"dash"` DURLFormat string `json:"format"` DURLs []dURL `json:"durl"` } type dURL struct { URL string `json:"url"` Size int64 `json:"size"` } type dash struct { Code int `json:"code"` Message string `json:"message"` Data dashInfo `json:"data"` Result dashInfo `json:"result"` } var qualityString = map[int]string{ 127: "超高清 8K", 120: "超清 4K", 116: "高清 1080P60", 74: "高清 720P60", 112: "高清 1080P+", 80: "高清 1080P", 64: "高清 720P", 48: "高清 720P", 32: "清晰 480P", 16: "流畅 360P", 15: "流畅 360P", } type subtitleData struct { From float32 `json:"from"` To float32 `json:"to"` Location int `json:"location"` Content string `json:"content"` } type bilibiliSubtitleFormat struct { FontSize float32 `json:"font_size"` FontColor string `json:"font_color"` BackgroundAlpha float32 `json:"background_alpha"` BackgroundColor string `json:"background_color"` Stroke string `json:"Stroke"` Body []subtitleData `json:"body"` } type subtitleProperty struct { ID int64 `json:"id"` Lan string `json:"lan"` LanDoc string `json:"lan_doc"` SubtitleUrl string `json:"subtitle_url"` } type subtitleInfo struct { AllowSubmit bool `json:"allow_submit"` SubtitleList []subtitleProperty `json:"subtitles"` } type bilibiliWebInterfaceData struct { Bvid string `json:"bvid"` SubtitleInfo subtitleInfo `json:"subtitle"` } type bilibiliWebInterface struct { Code int `json:"code"` Data bilibiliWebInterfaceData `json:"data"` } type festival struct { VideoSections []struct { Id int64 `json:"id"` Title string `json:"title"` Type int `json:"type"` } `json:"videoSections"` Episodes []episode `json:"episodes"` VideoInfo struct { Aid int `json:"aid"` BVid string `json:"bvid"` Cid int `json:"cid"` Title string `json:"title"` Desc string `json:"desc"` Pages []struct { Cid int `json:"cid"` Duration int `json:"duration"` Page int `json:"page"` Part string `json:"part"` Dimension struct { Width int `json:"width"` Height int `json:"height"` Rotate int `json:"rotate"` } `json:"dimension"` } `json:"pages"` } `json:"videoInfo"` } ================================================ FILE: extractors/bitchute/bitchute.go ================================================ package bitchute import ( "compress/flate" "compress/gzip" "fmt" "io" "net/http" "regexp" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" ) func init() { extractors.Register("bitchute", New()) } type extractor struct{} // New returns a bitchute extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) { regVideoID := regexp.MustCompile(`/video/([^?/]+)`) matchVideoID := regVideoID.FindStringSubmatch(u) if len(matchVideoID) < 2 { return nil, errors.New("Invalid video URL: Missing video ID parameter") } embedURL := fmt.Sprintf("https://www.bitchute.com/api/beta9/embed/?videoID=%s", matchVideoID[1]) res, err := request.Request(http.MethodGet, embedURL, nil, nil) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var reader io.ReadCloser switch res.Header.Get("Content-Encoding") { case "gzip": reader, _ = gzip.NewReader(res.Body) case "deflate": reader = flate.NewReader(res.Body) default: reader = res.Body } defer reader.Close() // nolint b, err := io.ReadAll(reader) if err != nil { return nil, errors.WithStack(err) } // There is also an API that provides meta data // POST https://api.bitchute.com/api/beta9/video {"video_id": } html := string(b) regMediaURL := regexp.MustCompile(`media_url\s*=\s*['|"](https:\/\/[^.]+\.bitchute\.com\/.*\.mp4)`) matchMediaURL := regMediaURL.FindStringSubmatch(html) if len(matchMediaURL) < 2 { return nil, errors.New("Could not extract media URL from page") } mediaURL := matchMediaURL[1] regVideoName := regexp.MustCompile(`(?m)video_name\s*=\s*["|']\\?"?(.*)["|'];?$`) matchVideoName := regVideoName.FindStringSubmatch(html) if len(matchVideoName) < 2 { return nil, errors.New("Could not extract media name from page") } videoName := strings.ReplaceAll(matchVideoName[1], `\"`, "") streams := make(map[string]*extractors.Stream, 1) size, err := request.Size(mediaURL, u) if err != nil { return nil, errors.WithStack(err) } streams["Default"] = &extractors.Stream{ Parts: []*extractors.Part{ { URL: mediaURL, Size: size, Ext: "mp4", }, }, Size: size, Quality: "Default", } return []*extractors.Data{ { Site: "Bitchute bitchute.com", Title: videoName, Type: extractors.DataTypeVideo, Streams: streams, URL: u, }, }, nil } ================================================ FILE: extractors/bitchute/bitchute_test.go ================================================ package bitchute import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "video test 1", args: test.Args{ URL: "https://www.bitchute.com/video/C17naZ6WlWPo", Title: "Everybody Dance Now", Size: 1794720, }, }, { name: "video test 2", args: test.Args{ URL: "https://www.bitchute.com/video/HFgoUz6HrvQd", Title: "Bear Level 1", Size: 971698, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/douyin/douyin.go ================================================ package douyin import ( "crypto/rand" _ "embed" "encoding/json" "fmt" "net/http" netURL "net/url" "regexp" "strings" "github.com/dop251/goja" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { e := New() extractors.Register("douyin", e) extractors.Register("iesdouyin", e) } //go:embed sign.js var script string type extractor struct{} // New returns a douyin extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { if strings.Contains(url, "v.douyin.com") { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, errors.WithStack(err) } c := http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } resp, err := c.Do(req) if err != nil { return nil, errors.WithStack(err) } defer resp.Body.Close() // nolint url = resp.Header.Get("location") } itemIds := utils.MatchOneOf(url, `/video/(\d+)`) if len(itemIds) == 0 { return nil, errors.New("unable to get video ID") } if itemIds == nil || len(itemIds) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } itemId := itemIds[len(itemIds)-1] // dynamic generate cookie cookie, err := createCookie() if err != nil { return nil, errors.WithStack(err) } api := "https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=" + itemId // parse api query params string query, err := netURL.Parse(api) if err != nil { return nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed) } // define request headers and sign agent headers := map[string]string{} headers["Cookie"] = cookie headers["Referer"] = "https://www.douyin.com/" headers["User-Agent"] = "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66" // init JavaScripts runtime vm := goja.New() // load sign scripts _, _ = vm.RunString(script) // sign sign, err := vm.RunString(fmt.Sprintf("sign('%s', '%s')", query.RawQuery, headers["User-Agent"])) if err != nil { return nil, errors.WithStack(err) } api = fmt.Sprintf("%s&X-Bogus=%s", api, sign) jsonData, err := request.Get(api, url, headers) if err != nil { return nil, errors.WithStack(err) } var douyin douyinData if err = json.Unmarshal([]byte(jsonData), &douyin); err != nil { return nil, errors.WithStack(err) } urlData := make([]*extractors.Part, 0) var douyinType extractors.DataType var totalSize int64 // AwemeType: 0:video 68:image if douyin.AwemeDetail.AwemeType == 68 { douyinType = extractors.DataTypeImage for _, img := range douyin.AwemeDetail.Images { realURL := img.URLList[len(img.URLList)-1] size, err := request.Size(realURL, url) if err != nil { return nil, errors.WithStack(err) } totalSize += size _, ext, err := utils.GetNameAndExt(realURL) if err != nil { return nil, errors.WithStack(err) } urlData = append(urlData, &extractors.Part{ URL: realURL, Size: size, Ext: ext, }) } } else { douyinType = extractors.DataTypeVideo realURL := douyin.AwemeDetail.Video.PlayAddr.URLList[0] totalSize, err = request.Size(realURL, url) if err != nil { return nil, errors.WithStack(err) } urlData = append(urlData, &extractors.Part{ URL: realURL, Size: totalSize, Ext: "mp4", }) } streams := map[string]*extractors.Stream{ "default": { Parts: urlData, Size: totalSize, }, } return []*extractors.Data{ { Site: "抖音 douyin.com", Title: douyin.AwemeDetail.Desc, Type: douyinType, Streams: streams, URL: url, }, }, nil } func createCookie() (string, error) { v1, err := msToken(107) if err != nil { return "", err } v2, err := ttwid() if err != nil { return "", err } v3 := "324fb4ea4a89c0c05827e18a1ed9cf9bf8a17f7705fcc793fec935b637867e2a5a9b8168c885554d029919117a18ba69" v4 := "eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWNsaWVudC1jc3IiOiItLS0tLUJFR0lOIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLVxyXG5NSUlCRFRDQnRRSUJBREFuTVFzd0NRWURWUVFHRXdKRFRqRVlNQllHQTFVRUF3d1BZbVJmZEdsamEyVjBYMmQxXHJcbllYSmtNRmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVKUDZzbjNLRlFBNUROSEcyK2F4bXAwNG5cclxud1hBSTZDU1IyZW1sVUE5QTZ4aGQzbVlPUlI4NVRLZ2tXd1FJSmp3Nyszdnc0Z2NNRG5iOTRoS3MvSjFJc3FBc1xyXG5NQ29HQ1NxR1NJYjNEUUVKRGpFZE1Cc3dHUVlEVlIwUkJCSXdFSUlPZDNkM0xtUnZkWGxwYmk1amIyMHdDZ1lJXHJcbktvWkl6ajBFQXdJRFJ3QXdSQUlnVmJkWTI0c0RYS0c0S2h3WlBmOHpxVDRBU0ROamNUb2FFRi9MQnd2QS8xSUNcclxuSURiVmZCUk1PQVB5cWJkcytld1QwSDZqdDg1czZZTVNVZEo5Z2dmOWlmeTBcclxuLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tXHJcbiJ9" cookie := fmt.Sprintf("msToken=%s;ttwid=%s;odin_tt=%s;bd_ticket_guard_client_data=%s;", v1, v2, v3, v4) return cookie, nil } func msToken(length int) (string, error) { const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" randomBytes := make([]byte, length) if _, err := rand.Read(randomBytes); err != nil { return "", err } token := make([]byte, length) for i, b := range randomBytes { token[i] = characters[int(b)%len(characters)] } return string(token), nil } func ttwid() (string, error) { body := map[string]interface{}{ "aid": 1768, "union": true, "needFid": false, "region": "cn", "cbUrlProtocol": "https", "service": "www.ixigua.com", "migrate_info": map[string]string{"ticket": "", "source": "node"}, } bytes, err := json.Marshal(body) if err != nil { return "", err } payload := strings.NewReader(string(bytes)) resp, err := request.Request(http.MethodPost, "https://ttwid.bytedance.com/ttwid/union/register/", payload, nil) if err != nil { return "", err } defer resp.Body.Close() // nolint cookie := resp.Header.Get("Set-Cookie") re := regexp.MustCompile(`ttwid=([^;]+)`) if match := re.FindStringSubmatch(cookie); match != nil { return match[1], nil } return "", errors.New("douyin ttwid request failed") } ================================================ FILE: extractors/douyin/douyin_test.go ================================================ package douyin import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home", Title: "是爱情，让父子相认#陈翔六点半 #关于爱情", }, }, { name: "image test", args: test.Args{ URL: "https://v.douyin.com/LvCYKvV", Title: "黑发限定#开春必备", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/douyin/sign.js ================================================ var window = null; function _0x5cd844(e) { var b = { exports: {} }; return e(b, b.exports), b.exports } jsvmp = function(e, b, a) { function f(e, b, a) { return (f = function() { if ("undefined" == typeof Reflect || !Reflect.construct || Reflect.construct.sham) return !1; if ("function" == typeof Proxy) return !0; try { return Date.prototype.toString.call(Reflect.construct(Date, [], function() {})), !0 } catch (e) { return !1 } }() ? Reflect.construct : function(e, b, a) { var f = [null]; f.push.apply(f, b); var c = new(Function.bind.apply(e, f)); return a && function(e, b) { (Object.setPrototypeOf || function(e, b) { return e.__proto__ = b, e })(e, b) }(c, a.prototype), c }).apply(null, arguments) } function c(e) { return function(e) { if (Array.isArray(e)) { for (var b = 0, a = new Array(e.length); b < e.length; b++) a[b] = e[b]; return a } }(e) || function(e) { if (Symbol.iterator in Object(e) || "[object Arguments]" === Object.prototype.toString.call(e)) return Array.from(e) }(e) || function() { throw new TypeError("Invalid attempt to spread non-iterable instance") }() } for (var r = [], t = 0, d = [], i = 0, n = function(e, b) { var a = e[b++], f = e[b], c = parseInt("" + a + f, 16); if (c >> 7 == 0) return [1, c]; if (c >> 6 == 2) { var r = parseInt("" + e[++b] + e[++b], 16); return c &= 63, [2, r = (c <<= 8) + r] } if (c >> 6 == 3) { var t = parseInt("" + e[++b] + e[++b], 16), d = parseInt("" + e[++b] + e[++b], 16); return c &= 63, [3, d = (c <<= 16) + (t <<= 8) + d] } }, s = function(e, b) { var a = parseInt("" + e[b] + e[b + 1], 16); return a > 127 ? -256 + a : a }, o = function(e, b) { var a = parseInt("" + e[b] + e[b + 1] + e[b + 2] + e[b + 3], 16); return a > 32767 ? -65536 + a : a }, l = function(e, b) { var a = parseInt("" + e[b] + e[b + 1] + e[b + 2] + e[b + 3] + e[b + 4] + e[b + 5] + e[b + 6] + e[b + 7], 16); return a > 2147483647 ? 0 + a : a }, _ = function(e, b) { return parseInt("" + e[b] + e[b + 1], 16) }, x = function(e, b) { return parseInt("" + e[b] + e[b + 1] + e[b + 2] + e[b + 3], 16) }, u = u || this || window, h = (e.length, 0), p = "", y = h; y < h + 16; y++) { var v = "" + e[y++] + e[y]; v = parseInt(v, 16), p += String.fromCharCode(v) } if ("HNOJ@?RC" != p) throw new Error("error magic number " + p); parseInt("" + e[h += 16] + e[h + 1], 16), h += 8, t = 0; for (var g = 0; g < 4; g++) { var w = h + 2 * g, A = parseInt("" + e[w++] + e[w], 16); t += (3 & A) << 2 * g } h += 16; var C = parseInt("" + e[h += 8] + e[h + 1] + e[h + 2] + e[h + 3] + e[h + 4] + e[h + 5] + e[h + 6] + e[h + 7], 16), m = C, S = h += 8, z = x(e, h += C); z[1], h += 4, r = { p: [], q: [] }; for (var B = 0; B < z; B++) { for (var R = n(e, h), q = h += 2 * R[0], I = r.p.length, k = 0; k < R[1]; k++) { var j = n(e, q); r.p.push(j[1]), q += 2 * j[0] } h = q, r.q.push([I, r.p.length]) } var O = { 5: 1, 6: 1, 70: 1, 22: 1, 23: 1, 37: 1, 73: 1 }, U = { 72: 1 }, D = { 74: 1 }, N = { 11: 1, 12: 1, 24: 1, 26: 1, 27: 1, 31: 1 }, J = { 10: 1 }, L = { 2: 1, 29: 1, 30: 1, 20: 1 }, T = [], E = []; function M(e, b, a) { for (var f = b; f < b + a;) { var c = _(e, f); T[f] = c, f += 2, U[c] ? (E[f] = s(e, f), f += 2) : O[c] ? (E[f] = o(e, f), f += 4) : D[c] ? (E[f] = l(e, f), f += 8) : N[c] ? (E[f] = _(e, f), f += 2) : J[c] ? (E[f] = x(e, f), f += 4) : L[c] && (E[f] = x(e, f), f += 4) } } return F(e, S, m / 2, [], b, a); function P(e, b, a, n, h, p, y, v) { null == p && (p = this); var g, w, A, C, m = [], S = 0; y && (w = y); var z, B, R = b, q = R + 2 * a; if (!v) for (; R < q;) { var I = parseInt("" + e[R] + e[R + 1], 16); R += 2; var j = 3 & (z = 13 * I % 241); if (z >>= 2, j < 1) if (j = 3 & z, z >>= 2, j < 1) { if ((j = z) < 1) return [1, m[S--]]; j < 5 ? (w = m[S--], m[S] = m[S] * w) : j < 7 ? (w = m[S--], m[S] = m[S] != w) : j < 14 ? (A = m[S--], C = m[S--], (j = m[S--]).x === P ? j.y >= 1 ? m[++S] = F(e, j.c, j.l, A, j.z, C, null, 1) : (m[++S] = F(e, j.c, j.l, A, j.z, C, null, 0), j.y++) : m[++S] = j.apply(C, A)) : j < 16 && (B = o(e, R), (g = function b() { var a = arguments; return b.y > 0 || b.y++, F(e, b.c, b.l, a, b.z, this, null, 0) }).c = R + 4, g.l = B - 2, g.x = P, g.y = 0, g.z = h, m[S] = g, R += 2 * B - 2) } else if (j < 2)(j = z) > 8 ? (w = m[S--], m[S] = typeof w) : j > 4 ? m[S -= 1] = m[S][m[S + 1]] : j > 2 && (A = m[S--], (j = m[S]).x === P ? j.y >= 1 ? m[S] = F(e, j.c, j.l, [A], j.z, C, null, 1) : (m[S] = F(e, j.c, j.l, [A], j.z, C, null, 0), j.y++) : m[S] = j(A)); else if (j < 3) { if ((j = z) < 9) { for (w = m[S--], B = x(e, R), j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); R += 4, m[S--][j] = w } else if (j < 13) throw m[S--] } else(j = z) < 1 ? m[++S] = null : j < 3 ? (w = m[S--], m[S] = m[S] >= w) : j < 12 && (m[++S] = void 0); else if (j < 2) if (j = 3 & z, z >>= 2, j < 1) if ((j = z) < 5) { B = o(e, R); try { if (d[i][2] = 1, 1 == (w = P(e, R + 4, B - 3, [], h, p, null, 0))[0]) return w } catch (b) { if (d[i] && d[i][1] && 1 == (w = P(e, d[i][1][0], d[i][1][1], [], h, p, b, 0))[0]) return w } finally { if (d[i] && d[i][0] && 1 == (w = P(e, d[i][0][0], d[i][0][1], [], h, p, null, 0))[0]) return w; d[i] = 0, i-- } R += 2 * B - 2 } else j < 7 ? (B = _(e, R), R += 2, m[S -= B] = 0 === B ? new m[S] : f(m[S], c(m.slice(S + 1, S + B + 1)))) : j < 9 && (w = m[S--], m[S] = m[S] & w); else if (j < 2) if ((j = z) > 12) m[++S] = s(e, R), R += 2; else if (j > 10) w = m[S--], m[S] = m[S] << w; else if (j > 8) { for (B = x(e, R), j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); R += 4, m[S] = m[S][j] } else j > 6 && (A = m[S--], w = delete m[S--][A]); else if (j < 3)(j = z) < 2 ? m[++S] = w : j < 11 ? (w = m[S -= 2][m[S + 1]] = m[S + 2], S--) : j < 13 && (w = m[S], m[++S] = w); else if ((j = z) > 12) m[++S] = p; else if (j > 5) w = m[S--], m[S] = m[S] !== w; else if (j > 3) w = m[S--], m[S] = m[S] / w; else if (j > 1) { if ((B = o(e, R)) < 0) { v = 1, M(e, b, 2 * a), R += 2 * B - 2; break } R += 2 * B - 2 } else j > -1 && (m[S] = !m[S]); else if (j < 3) if (j = 3 & z, z >>= 2, j < 1)(j = z) > 13 ? (m[++S] = o(e, R), R += 4) : j > 11 ? (w = m[S--], m[S] = m[S] >> w) : j > 9 ? (B = _(e, R), R += 2, w = m[S--], h[B] = w) : j > 7 ? (B = x(e, R), R += 4, A = S + 1, m[S -= B - 1] = B ? m.slice(S, A) : []) : j > 0 && (w = m[S--], m[S] = m[S] > w); else if (j < 2)(j = z) > 12 ? (w = m[S - 1], A = m[S], m[++S] = w, m[++S] = A) : j > 3 ? (w = m[S--], m[S] = m[S] == w) : j > 1 ? (w = m[S--], m[S] = m[S] + w) : j > -1 && (m[++S] = u); else if (j < 3) { if ((j = z) > 13) m[++S] = !1; else if (j > 6) w = m[S--], m[S] = m[S] instanceof w; else if (j > 4) w = m[S--], m[S] = m[S] % w; else if (j > 2) if (m[S--]) R += 4; else { if ((B = o(e, R)) < 0) { v = 1, M(e, b, 2 * a), R += 2 * B - 2; break } R += 2 * B - 2 } else if (j > 0) { for (B = x(e, R), w = "", k = r.q[B][0]; k < r.q[B][1]; k++) w += String.fromCharCode(t ^ r.p[k]); m[++S] = w, R += 4 } } else(j = z) > 7 ? (w = m[S--], m[S] = m[S] | w) : j > 5 ? (B = _(e, R), R += 2, m[++S] = h["$" + B]) : j > 3 && (B = o(e, R), d[i][0] && !d[i][2] ? d[i][1] = [R + 4, B - 3] : d[i++] = [0, [R + 4, B - 3], 0], R += 2 * B - 2); else if (j = 3 & z, z >>= 2, j > 2)(j = z) > 13 ? (m[++S] = l(e, R), R += 8) : j > 11 ? (w = m[S--], m[S] = m[S] >>> w) : j > 9 ? m[++S] = !0 : j > 7 ? (B = _(e, R), R += 2, m[S] = m[S][B]) : j > 0 && (w = m[S--], m[S] = m[S] < w); else if (j > 1)(j = z) > 10 ? (B = o(e, R), d[++i] = [ [R + 4, B - 3], 0, 0 ], R += 2 * B - 2) : j > 8 ? (w = m[S--], m[S] = m[S] ^ w) : j > 6 && (w = m[S--]); else if (j > 0) { if ((j = z) > 7) w = m[S--], m[S] = m[S] in w; else if (j > 5) m[S] = ++m[S]; else if (j > 3) B = _(e, R), R += 2, w = h[B], m[++S] = w; else if (j > 1) { var O = 0, U = m[S].length, D = m[S]; m[++S] = function() { var e = O < U; if (e) { var b = D[O++]; m[++S] = b } m[++S] = e } } } else if ((j = z) > 13) w = m[S], m[S] = m[S - 1], m[S - 1] = w; else if (j > 4) w = m[S--], m[S] = m[S] === w; else if (j > 2) w = m[S--], m[S] = m[S] - w; else if (j > 0) { for (B = x(e, R), j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); j = +j, R += 4, m[++S] = j } } if (v) for (; R < q;) if (I = T[R], R += 2, j = 3 & (z = 13 * I % 241), z >>= 2, j > 2) if (j = 3 & z, z >>= 2, j > 2)(j = z) < 2 ? (w = m[S--], m[S] = m[S] < w) : j < 9 ? (B = E[R], R += 2, m[S] = m[S][B]) : j < 11 ? m[++S] = !0 : j < 13 ? (w = m[S--], m[S] = m[S] >>> w) : j < 15 && (m[++S] = E[R], R += 8); else if (j > 1)(j = z) < 6 || (j < 8 ? w = m[S--] : j < 10 ? (w = m[S--], m[S] = m[S] ^ w) : j < 12 && (B = E[R], d[++i] = [ [R + 4, B - 3], 0, 0 ], R += 2 * B - 2)); else if (j > 0)(j = z) > 7 ? (w = m[S--], m[S] = m[S] in w) : j > 5 ? m[S] = ++m[S] : j > 3 ? (B = E[R], R += 2, w = h[B], m[++S] = w) : j > 1 && (O = 0, U = m[S].length, D = m[S], m[++S] = function() { var e = O < U; if (e) { var b = D[O++]; m[++S] = b } m[++S] = e }); else if ((j = z) < 2) { for (B = E[R], j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); j = +j, R += 4, m[++S] = j } else j < 4 ? (w = m[S--], m[S] = m[S] - w) : j < 6 ? (w = m[S--], m[S] = m[S] === w) : j < 15 && (w = m[S], m[S] = m[S - 1], m[S - 1] = w); else if (j > 1) if (j = 3 & z, z >>= 2, j < 1)(j = z) > 13 ? (m[++S] = E[R], R += 4) : j > 11 ? (w = m[S--], m[S] = m[S] >> w) : j > 9 ? (B = E[R], R += 2, w = m[S--], h[B] = w) : j > 7 ? (B = E[R], R += 4, A = S + 1, m[S -= B - 1] = B ? m.slice(S, A) : []) : j > 0 && (w = m[S--], m[S] = m[S] > w); else if (j < 2)(j = z) < 1 ? m[++S] = u : j < 3 ? (w = m[S--], m[S] = m[S] + w) : j < 5 ? (w = m[S--], m[S] = m[S] == w) : j < 14 && (w = m[S - 1], A = m[S], m[++S] = w, m[++S] = A); else if (j < 3) { if ((j = z) > 13) m[++S] = !1; else if (j > 6) w = m[S--], m[S] = m[S] instanceof w; else if (j > 4) w = m[S--], m[S] = m[S] % w; else if (j > 2) m[S--] ? R += 4 : R += 2 * (B = E[R]) - 2; else if (j > 0) { for (B = E[R], w = "", k = r.q[B][0]; k < r.q[B][1]; k++) w += String.fromCharCode(t ^ r.p[k]); m[++S] = w, R += 4 } } else(j = z) > 7 ? (w = m[S--], m[S] = m[S] | w) : j > 5 ? (B = E[R], R += 2, m[++S] = h["$" + B]) : j > 3 && (B = E[R], d[i][0] && !d[i][2] ? d[i][1] = [R + 4, B - 3] : d[i++] = [0, [R + 4, B - 3], 0], R += 2 * B - 2); else if (j > 0) if (j = 3 & z, z >>= 2, j < 1) { if ((j = z) > 9); else if (j > 7) w = m[S--], m[S] = m[S] & w; else if (j > 5) B = E[R], R += 2, m[S -= B] = 0 === B ? new m[S] : f(m[S], c(m.slice(S + 1, S + B + 1))); else if (j > 3) { B = E[R]; try { if (d[i][2] = 1, 1 == (w = P(e, R + 4, B - 3, [], h, p, null, 0))[0]) return w } catch (b) { if (d[i] && d[i][1] && 1 == (w = P(e, d[i][1][0], d[i][1][1], [], h, p, b, 0))[0]) return w } finally { if (d[i] && d[i][0] && 1 == (w = P(e, d[i][0][0], d[i][0][1], [], h, p, null, 0))[0]) return w; d[i] = 0, i-- } R += 2 * B - 2 } } else if (j < 2) if ((j = z) < 8) A = m[S--], w = delete m[S--][A]; else if (j < 10) { for (B = E[R], j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); R += 4, m[S] = m[S][j] } else j < 12 ? (w = m[S--], m[S] = m[S] << w) : j < 14 && (m[++S] = E[R], R += 2); else j < 3 ? (j = z) < 2 ? m[++S] = w : j < 11 ? (w = m[S -= 2][m[S + 1]] = m[S + 2], S--) : j < 13 && (w = m[S], m[++S] = w) : (j = z) > 12 ? m[++S] = p : j > 5 ? (w = m[S--], m[S] = m[S] !== w) : j > 3 ? (w = m[S--], m[S] = m[S] / w) : j > 1 ? R += 2 * (B = E[R]) - 2 : j > -1 && (m[S] = !m[S]); else if (j = 3 & z, z >>= 2, j < 1) { if ((j = z) < 1) return [1, m[S--]]; j < 5 ? (w = m[S--], m[S] = m[S] * w) : j < 7 ? (w = m[S--], m[S] = m[S] != w) : j < 14 ? (A = m[S--], C = m[S--], (j = m[S--]).x === P ? j.y >= 1 ? m[++S] = F(e, j.c, j.l, A, j.z, C, null, 1) : (m[++S] = F(e, j.c, j.l, A, j.z, C, null, 0), j.y++) : m[++S] = j.apply(C, A)) : j < 16 && (B = E[R], (g = function b() { var a = arguments; return b.y > 0 || b.y++, F(e, b.c, b.l, a, b.z, this, null, 0) }).c = R + 4, g.l = B - 2, g.x = P, g.y = 0, g.z = h, m[S] = g, R += 2 * B - 2) } else if (j < 2)(j = z) > 8 ? (w = m[S--], m[S] = typeof w) : j > 4 ? m[S -= 1] = m[S][m[S + 1]] : j > 2 && (A = m[S--], (j = m[S]).x === P ? j.y >= 1 ? m[S] = F(e, j.c, j.l, [A], j.z, C, null, 1) : (m[S] = F(e, j.c, j.l, [A], j.z, C, null, 0), j.y++) : m[S] = j(A)); else if (j < 3) { if ((j = z) < 9) { for (w = m[S--], B = E[R], j = "", k = r.q[B][0]; k < r.q[B][1]; k++) j += String.fromCharCode(t ^ r.p[k]); R += 4, m[S--][j] = w } else if (j < 13) throw m[S--] } else(j = z) < 1 ? m[++S] = null : j < 3 ? (w = m[S--], m[S] = m[S] >= w) : j < 12 && (m[++S] = void 0); return [0, null] } function F(e, b, a, f, c, r, t, d) { null == r && (r = this), c && !c.d && (c.d = 0, c.$0 = c, c[1] = {}); var i, n, s = {}, o = s.d = c ? c.d + 1 : 0; for (s["$" + o] = s, n = 0; n < o; n++) s[i = "$" + n] = c[i]; for (n = 0, o = s.length = f.length; n < o; n++) s[n] = f[n]; return d && !T[b] && M(e, b, 2 * a), T[b] ? P(e, b, a, 0, s, r, null, 1)[1] : P(e, b, a, 0, s, r, null, 0)[1] } }; var _0x397dc7 = "undefined" != typeof globalThis ? globalThis : void 0 !== window ? window : "undefined" != typeof global ? global : "undefined" != typeof self ? self : {}, _0x124d1a = _0x5cd844(function(_0x770f81) { ! function() { var _0x250d36 = "input is invalid type", _0x4cfaee = !1, _0x1702f9 = {}, _0x5ccbb3 = !_0x4cfaee && "object" == typeof self, _0x54d876 = !_0x1702f9.JS_MD5_NO_NODE_JS && "object" == typeof process && process.versions && process.versions.node, _0x185caf; _0x54d876 ? _0x1702f9 = _0x397dc7 : _0x5ccbb3 && (_0x1702f9 = self); var _0x17dcbf = !_0x1702f9.JS_MD5_NO_COMMON_JS && _0x770f81.exports, _0x554fed = !1, _0x2de28f = !_0x1702f9.JS_MD5_NO_ARRAY_BUFFER && "undefined" != typeof ArrayBuffer, _0x3a9a1b = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], _0x465562 = [128, 32768, 8388608, -2147483648], _0x20b37e = [0, 8, 16, 24], _0x323604 = ["hex", "array", "digest", "buffer", "arrayBuffer", "base64"], _0x2c185e = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "/"], _0x4b59e0 = []; if (_0x2de28f) { var _0x395837 = new ArrayBuffer(68); _0x185caf = new Uint8Array(_0x395837), _0x4b59e0 = new Uint32Array(_0x395837) }!_0x1702f9.JS_MD5_NO_NODE_JS && Array.isArray || (Array.isArray = function(e) { return "[object Array]" === Object.prototype.toString.call(e) }), _0x2de28f && (_0x1702f9.JS_MD5_NO_ARRAY_BUFFER_IS_VIEW || !ArrayBuffer.isView) && (ArrayBuffer.isView = function(e) { return "object" == typeof e && e.buffer && e.buffer.constructor === ArrayBuffer }); var _0x4e9930 = function(e) { return function(b) { return new _0x5887c8(!0).update(b)[e]() } }, _0x38ba77 = function() { var e = _0x4e9930("hex"); _0x54d876 && (e = _0x474989(e)), e.create = function() { return new _0x5887c8 }, e.update = function(b) { return e.create().update(b) }; for (var b = 0; b < _0x323604.length; ++b) { var a = _0x323604[b]; e[a] = _0x4e9930(a) } return e }, _0x474989 = function(_0x57eeaa) { var _0x114910, _0x226465 = eval("require('crypto');"), _0x1f6ae0 = eval("require('buffer')['Buffer'];"); return function(e) { if ("string" == typeof e) return _0x226465.createHash("md5").update(e, "utf8").digest("hex"); if (null == e) throw _0x250d36; return e.constructor === ArrayBuffer && (e = new Uint8Array(e)), Array.isArray(e) || ArrayBuffer.isView(e) || e.constructor === _0x1f6ae0 ? _0x226465.createHash("md5").update(new _0x1f6ae0.from(e)).digest("hex") : _0x57eeaa(e) } }; function _0x5887c8(e) { if (e) _0x4b59e0[0] = _0x4b59e0[16] = _0x4b59e0[1] = _0x4b59e0[2] = _0x4b59e0[3] = _0x4b59e0[4] = _0x4b59e0[5] = _0x4b59e0[6] = _0x4b59e0[7] = _0x4b59e0[8] = _0x4b59e0[9] = _0x4b59e0[10] = _0x4b59e0[11] = _0x4b59e0[12] = _0x4b59e0[13] = _0x4b59e0[14] = _0x4b59e0[15] = 0, this.blocks = _0x4b59e0, this.buffer8 = _0x185caf; else if (_0x2de28f) { var b = new ArrayBuffer(68); this.buffer8 = new Uint8Array(b), this.blocks = new Uint32Array(b) } else this.blocks = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; this.h0 = this.h1 = this.h2 = this.h3 = this.start = this.bytes = this.hBytes = 0, this.finalized = this.hashed = !1, this.first = !0 } _0x5887c8.prototype.update = function(e) { if (!this.finalized) { var b, a = typeof e; if ("string" !== a) { if ("object" !== a || null === e) throw _0x250d36; if (_0x2de28f && e.constructor === ArrayBuffer) e = new Uint8Array(e); else if (!(Array.isArray(e) || _0x2de28f && ArrayBuffer.isView(e))) throw _0x250d36; b = !0 } for (var f, c, r = 0, t = e.length, d = this.blocks, i = this.buffer8; r < t;) { if (this.hashed && (this.hashed = !1, d[0] = d[16], d[16] = d[1] = d[2] = d[3] = d[4] = d[5] = d[6] = d[7] = d[8] = d[9] = d[10] = d[11] = d[12] = d[13] = d[14] = d[15] = 0), b) if (_0x2de28f) for (c = this.start; r < t && c < 64; ++r) i[c++] = e[r]; else for (c = this.start; r < t && c < 64; ++r) d[c >> 2] |= e[r] << _0x20b37e[3 & c++]; else if (_0x2de28f) for (c = this.start; r < t && c < 64; ++r)(f = e.charCodeAt(r)) < 128 ? i[c++] = f : f < 2048 ? (i[c++] = 192 | f >> 6, i[c++] = 128 | 63 & f) : f < 55296 || f >= 57344 ? (i[c++] = 224 | f >> 12, i[c++] = 128 | f >> 6 & 63, i[c++] = 128 | 63 & f) : (f = 65536 + ((1023 & f) << 10 | 1023 & e.charCodeAt(++r)), i[c++] = 240 | f >> 18, i[c++] = 128 | f >> 12 & 63, i[c++] = 128 | f >> 6 & 63, i[c++] = 128 | 63 & f); else for (c = this.start; r < t && c < 64; ++r)(f = e.charCodeAt(r)) < 128 ? d[c >> 2] |= f << _0x20b37e[3 & c++] : f < 2048 ? (d[c >> 2] |= (192 | f >> 6) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]) : f < 55296 || f >= 57344 ? (d[c >> 2] |= (224 | f >> 12) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 6 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]) : (f = 65536 + ((1023 & f) << 10 | 1023 & e.charCodeAt(++r)), d[c >> 2] |= (240 | f >> 18) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 12 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | f >> 6 & 63) << _0x20b37e[3 & c++], d[c >> 2] |= (128 | 63 & f) << _0x20b37e[3 & c++]); this.lastByteIndex = c, this.bytes += c - this.start, c >= 64 ? (this.start = c - 64, this.hash(), this.hashed = !0) : this.start = c } return this.bytes > 4294967295 && (this.hBytes += this.bytes / 4294967296 << 0, this.bytes = this.bytes % 4294967296), this } }, _0x5887c8.prototype.finalize = function() { if (!this.finalized) { this.finalized = !0; var e = this.blocks, b = this.lastByteIndex; e[b >> 2] |= _0x465562[3 & b], b >= 56 && (this.hashed || this.hash(), e[0] = e[16], e[16] = e[1] = e[2] = e[3] = e[4] = e[5] = e[6] = e[7] = e[8] = e[9] = e[10] = e[11] = e[12] = e[13] = e[14] = e[15] = 0), e[14] = this.bytes << 3, e[15] = this.hBytes << 3 | this.bytes >>> 29, this.hash() } }, _0x5887c8.prototype.hash = function() { var e, b, a, f, c, r, t = this.blocks; this.first ? b = ((b = ((e = ((e = t[0] - 680876937) << 7 | e >>> 25) - 271733879 << 0) ^ (a = ((a = (-271733879 ^ (f = ((f = (-1732584194 ^ 2004318071 & e) + t[1] - 117830708) << 12 | f >>> 20) + e << 0) & (-271733879 ^ e)) + t[2] - 1126478375) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[3] - 1316259209) << 22 | b >>> 10) + a << 0 : (e = this.h0, b = this.h1, a = this.h2, b = ((b += ((e = ((e += ((f = this.h3) ^ b & (a ^ f)) + t[0] - 680876936) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[1] - 389564586) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[2] + 606105819) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[3] - 1044525330) << 22 | b >>> 10) + a << 0), b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[4] - 176418897) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[5] + 1200080426) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[6] - 1473231341) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[7] - 45705983) << 22 | b >>> 10) + a << 0, b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[8] + 1770035416) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[9] - 1958414417) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[10] - 42063) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[11] - 1990404162) << 22 | b >>> 10) + a << 0, b = ((b += ((e = ((e += (f ^ b & (a ^ f)) + t[12] + 1804603682) << 7 | e >>> 25) + b << 0) ^ (a = ((a += (b ^ (f = ((f += (a ^ e & (b ^ a)) + t[13] - 40341101) << 12 | f >>> 20) + e << 0) & (e ^ b)) + t[14] - 1502002290) << 17 | a >>> 15) + f << 0) & (f ^ e)) + t[15] + 1236535329) << 22 | b >>> 10) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[1] - 165796510) << 5 | e >>> 27) + b << 0) ^ b)) + t[6] - 1069501632) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[11] + 643717713) << 14 | a >>> 18) + f << 0) ^ f)) + t[0] - 373897302) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[5] - 701558691) << 5 | e >>> 27) + b << 0) ^ b)) + t[10] + 38016083) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[15] - 660478335) << 14 | a >>> 18) + f << 0) ^ f)) + t[4] - 405537848) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[9] + 568446438) << 5 | e >>> 27) + b << 0) ^ b)) + t[14] - 1019803690) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[3] - 187363961) << 14 | a >>> 18) + f << 0) ^ f)) + t[8] + 1163531501) << 20 | b >>> 12) + a << 0, b = ((b += ((f = ((f += (b ^ a & ((e = ((e += (a ^ f & (b ^ a)) + t[13] - 1444681467) << 5 | e >>> 27) + b << 0) ^ b)) + t[2] - 51403784) << 9 | f >>> 23) + e << 0) ^ e & ((a = ((a += (e ^ b & (f ^ e)) + t[7] + 1735328473) << 14 | a >>> 18) + f << 0) ^ f)) + t[12] - 1926607734) << 20 | b >>> 12) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[5] - 378558) << 4 | e >>> 28) + b << 0)) + t[8] - 2022574463) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[11] + 1839030562) << 16 | a >>> 16) + f << 0)) + t[14] - 35309556) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[1] - 1530992060) << 4 | e >>> 28) + b << 0)) + t[4] + 1272893353) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[7] - 155497632) << 16 | a >>> 16) + f << 0)) + t[10] - 1094730640) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[13] + 681279174) << 4 | e >>> 28) + b << 0)) + t[0] - 358537222) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[3] - 722521979) << 16 | a >>> 16) + f << 0)) + t[6] + 76029189) << 23 | b >>> 9) + a << 0, b = ((b += ((r = (f = ((f += ((c = b ^ a) ^ (e = ((e += (c ^ f) + t[9] - 640364487) << 4 | e >>> 28) + b << 0)) + t[12] - 421815835) << 11 | f >>> 21) + e << 0) ^ e) ^ (a = ((a += (r ^ b) + t[15] + 530742520) << 16 | a >>> 16) + f << 0)) + t[2] - 995338651) << 23 | b >>> 9) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[0] - 198630844) << 6 | e >>> 26) + b << 0) | ~a)) + t[7] + 1126891415) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[14] - 1416354905) << 15 | a >>> 17) + f << 0) | ~e)) + t[5] - 57434055) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[12] + 1700485571) << 6 | e >>> 26) + b << 0) | ~a)) + t[3] - 1894986606) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[10] - 1051523) << 15 | a >>> 17) + f << 0) | ~e)) + t[1] - 2054922799) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[8] + 1873313359) << 6 | e >>> 26) + b << 0) | ~a)) + t[15] - 30611744) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[6] - 1560198380) << 15 | a >>> 17) + f << 0) | ~e)) + t[13] + 1309151649) << 21 | b >>> 11) + a << 0, b = ((b += ((f = ((f += (b ^ ((e = ((e += (a ^ (b | ~f)) + t[4] - 145523070) << 6 | e >>> 26) + b << 0) | ~a)) + t[11] - 1120210379) << 10 | f >>> 22) + e << 0) ^ ((a = ((a += (e ^ (f | ~b)) + t[2] + 718787259) << 15 | a >>> 17) + f << 0) | ~e)) + t[9] - 343485551) << 21 | b >>> 11) + a << 0, this.first ? (this.h0 = e + 1732584193 << 0, this.h1 = b - 271733879 << 0, this.h2 = a - 1732584194 << 0, this.h3 = f + 271733878 << 0, this.first = !1) : (this.h0 = this.h0 + e << 0, this.h1 = this.h1 + b << 0, this.h2 = this.h2 + a << 0, this.h3 = this.h3 + f << 0) }, _0x5887c8.prototype.hex = function() { this.finalize(); var e = this.h0, b = this.h1, a = this.h2, f = this.h3; return _0x3a9a1b[e >> 4 & 15] + _0x3a9a1b[15 & e] + _0x3a9a1b[e >> 12 & 15] + _0x3a9a1b[e >> 8 & 15] + _0x3a9a1b[e >> 20 & 15] + _0x3a9a1b[e >> 16 & 15] + _0x3a9a1b[e >> 28 & 15] + _0x3a9a1b[e >> 24 & 15] + _0x3a9a1b[b >> 4 & 15] + _0x3a9a1b[15 & b] + _0x3a9a1b[b >> 12 & 15] + _0x3a9a1b[b >> 8 & 15] + _0x3a9a1b[b >> 20 & 15] + _0x3a9a1b[b >> 16 & 15] + _0x3a9a1b[b >> 28 & 15] + _0x3a9a1b[b >> 24 & 15] + _0x3a9a1b[a >> 4 & 15] + _0x3a9a1b[15 & a] + _0x3a9a1b[a >> 12 & 15] + _0x3a9a1b[a >> 8 & 15] + _0x3a9a1b[a >> 20 & 15] + _0x3a9a1b[a >> 16 & 15] + _0x3a9a1b[a >> 28 & 15] + _0x3a9a1b[a >> 24 & 15] + _0x3a9a1b[f >> 4 & 15] + _0x3a9a1b[15 & f] + _0x3a9a1b[f >> 12 & 15] + _0x3a9a1b[f >> 8 & 15] + _0x3a9a1b[f >> 20 & 15] + _0x3a9a1b[f >> 16 & 15] + _0x3a9a1b[f >> 28 & 15] + _0x3a9a1b[f >> 24 & 15] }, _0x5887c8.prototype.toString = _0x5887c8.prototype.hex, _0x5887c8.prototype.digest = function() { this.finalize(); var e = this.h0, b = this.h1, a = this.h2, f = this.h3; return [255 & e, e >> 8 & 255, e >> 16 & 255, e >> 24 & 255, 255 & b, b >> 8 & 255, b >> 16 & 255, b >> 24 & 255, 255 & a, a >> 8 & 255, a >> 16 & 255, a >> 24 & 255, 255 & f, f >> 8 & 255, f >> 16 & 255, f >> 24 & 255] }, _0x5887c8.prototype.array = _0x5887c8.prototype.digest, _0x5887c8.prototype.arrayBuffer = function() { this.finalize(); var e = new ArrayBuffer(16), b = new Uint32Array(e); return b[0] = this.h0, b[1] = this.h1, b[2] = this.h2, b[3] = this.h3, e }, _0x5887c8.prototype.buffer = _0x5887c8.prototype.arrayBuffer, _0x5887c8.prototype.base64 = function() { for (var e, b, a, f = "", c = this.array(), r = 0; r < 15;) e = c[r++], b = c[r++], a = c[r++], f += _0x2c185e[e >>> 2] + _0x2c185e[63 & (e << 4 | b >>> 4)] + _0x2c185e[63 & (b << 2 | a >>> 6)] + _0x2c185e[63 & a]; return f + (_0x2c185e[(e = c[r]) >>> 2] + _0x2c185e[e << 4 & 63] + "==") }; var _0x4dd781 = _0x38ba77(); _0x17dcbf ? _0x770f81.exports = _0x4dd781 : (_0x1702f9.md5 = _0x4dd781, _0x554fed && (void 0)(function() { return _0x4dd781 })) }() }); function _0x178cef(e) { return jsvmp("484e4f4a403f52430038001eab0015840e8ee21a00000000000000621b000200001d000146000306000e271f001b000200021d00010500121b001b000b021b000b04041d0001071b000b0500000003000126207575757575757575757575757575757575757575757575757575757575757575", [, , void 0 !== _0x124d1a ? _0x124d1a : void 0, _0x178cef, e]) } for (var _0xb55f3e = { boe: !1, aid: 0, dfp: !1, sdi: !1, enablePathList: [], _enablePathListRegex: [], urlRewriteRules: [], _urlRewriteRules: [], initialized: !1, enableTrack: !1, track: { unitTime: 0, unitAmount: 0, fre: 0 }, triggerUnload: !1, region: "", regionConf: {}, umode: 0, v: !1, perf: !1, xxbg: !0 }, _0x3eaf64 = { debug: function(e, b) { let a = !1; a = !1 } }, _0x233455 = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"], _0x2e9f6d = [], _0x511f86 = [], _0x3d35de = 0; _0x3d35de < 256; _0x3d35de++) _0x2e9f6d[_0x3d35de] = _0x233455[_0x3d35de >> 4 & 15] + _0x233455[15 & _0x3d35de], _0x3d35de < 16 && (_0x3d35de < 10 ? _0x511f86[48 + _0x3d35de] = _0x3d35de : _0x511f86[87 + _0x3d35de] = _0x3d35de); var _0x2ce54d = function(e) { for (var b = e.length, a = "", f = 0; f < b;) a += _0x2e9f6d[e[f++]]; return a }, _0x5960a2 = function(e) { for (var b = e.length >> 1, a = b << 1, f = new Uint8Array(b), c = 0, r = 0; r < a;) f[c++] = _0x511f86[e.charCodeAt(r++)] << 4 | _0x511f86[e.charCodeAt(r++)]; return f }, _0x4e46b6 = { encode: _0x2ce54d, decode: _0x5960a2 }; function sign(e, b) { return jsvmp("484e4f4a403f5243001f240fbf2031ccf317480300000000000007181b0002012e1d00921b000b171b000b02402217000a1c1b000b1726402217000c1c1b000b170200004017002646000306000e271f001b000200021d00920500121b001b000b031b000b17041d0092071b000b041e012f17000d1b000b05260a0000101c1b000b06260a0000101c1b001b000b071e01301d00931b001b000b081e00081d00941b0048021d00951b001b000b1b1d00961b0048401d009e1b001b000b031b000b16041d009f1b001b000b09221e0131241b000b031b000b09221e0131241b000b1e0a000110040a0001101d00d51b001b000b09221e0131241b000b031b000b09221e0131241b000b180a000110040a0001101d00d71b001b000b0a1e00101d00d91b001b000b0b261b000b1a1b000b190a0002101d00db1b001b000b0c261b000b221b000b210a0002101d00dc1b001b000b0d261b000b230200200a0002101d00dd1b001b000b09221e0131241b000b031b000b24040a0001101d00df1b001b000b0e1a00221e00de240a0000104903e82b1d00e31b001b000b0f260a0000101d00e41b001b000b1d1d00e71b001b000b1a4901002b1d00e81b001b000b1a4901002c1d00ea1b001b000b191d00f21b001b000b1f480e191d00f81b001b000b1f480f191d00f91b001b000b20480e191d00fb1b001b000b20480f191d00fe1b001b000b25480e191d01001b001b000b25480f191d01011b001b000b264818344900ff2f1d01031b001b000b264810344900ff2f1d01321b001b000b264808344900ff2f1d01331b001b000b264800344900ff2f1d01341b001b000b274818344900ff2f1d01351b001b000b274810344900ff2f1d01361b001b000b274808344900ff2f1d01371b001b000b274800344900ff2f1d01381b001b000b281b000b29311b000b2a311b000b2b311b000b2c311b000b2d311b000b2e311b000b2f311b000b30311b000b31311b000b32311b000b33311b000b34311b000b35311b000b36311b000b37311b000b38311b000b39311d01391b004900ff1d013a1b001b000b10261b000b281b000b2a1b000b2c1b000b2e1b000b301b000b321b000b341b000b361b000b381b000b3a1b000b291b000b2b1b000b2d1b000b2f1b000b311b000b331b000b351b000b371b000b390a0013101d013b1b001b000b0c261b000b111b000b3b041b000b3c0a0002101d013c1b001b000b12261b000b1c1b000b3b1b000b3d0a0003101d013d1b001b000b13261b000b3e0200240a0002101d013e1b000b3f0000013f000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b726152670f487c717976706733447a7d777c644e08577c70667e767d6712487c7179767067335d72657a7472677c614e057960777c7e10487c7179767067335b7a60677c616a4e07637f66747a7d60084c637b727d677c7e0b70727f7f437b727d677c7e0b4c4c7d7a747b677e726176055266777a7c1850727d65726041767d7776617a7d74507c7d67766b6721570964767177617a657661137476675c647d43617c637661676a5d727e7660097f727d74667274766006707b617c7e760761667d677a7e7607707c7d7d767067144c4c64767177617a6576614c7665727f66726776134c4c60767f767d7a667e4c7665727f667267761b4c4c64767177617a6576614c6070617a63674c75667d70677a7c7d174c4c64767177617a6576614c6070617a63674c75667d70154c4c64767177617a6576614c6070617a63674c757d134c4c756b77617a6576614c7665727f66726776124c4c77617a6576614c667d64617263637677154c4c64767177617a6576614c667d64617263637677114c4c77617a6576614c7665727f66726776144c4c60767f767d7a667e4c667d64617263637677144c4c756b77617a6576614c667d64617263637677094c60767f767d7a667e0c70727f7f40767f767d7a667e164c40767f767d7a667e4c5a57564c4176707c6177766108777c70667e767d670478766a60057e7267707b06417674566b630a4f3748723e694e77704c067072707b764c04607c7e7608707675407b72616308507675407b72616305767c72637a16767c44767151617c64607661577a60637267707b76610f717a7d775c717976706752606a7d700e7a60565c44767151617c646076610120047c63767d0467766067097a7d707c747d7a677c077c7d7661617c6104707c77761242465c47524c564b5056565756574c5641410e607660607a7c7d40677c61727476076076675a67767e10607c7e7658766a5b766176516a6776770a61767e7c65765a67767e097a7d77766b767757510c437c7a7d6776615665767d670e5e40437c7a7d6776615665767d670d706176726776567f767e767d670670727d65726009677c5772677246415f076176637f727076034f603901740a7d72677a6576707c777614487c717976706733437f66747a7d526161726a4e4a4d7b676763602c294f3c4f3c3b48233e2a4e68223f206e3b4f3d48233e2a4e68223f206e3a68206e6f48723e75233e2a4e68223f276e3b2948723e75233e2a4e68223f276e3a68246e3a0127087f7c7072677a7c7d047b61767504757a7f76107b676763293c3c7f7c70727f7b7c606708637f7267757c617e02222102222007647a7d777c646002222703647a7d02222607727d77617c7a77022225057f7a7d666b022224067a637b7c7d7602222b047a63727702222a047a637c77022123037e7270022122097e72707a7d677c607b0c7e72704c637c64766163703a0470617c60036b22220570617a7c6005756b7a7c6004637a787602212102212002212702212602212502212402212b08757a6176757c6b3c067c637661723c05337c63613c05337c63673c07707b617c7e763c0867617a77767d673c047e607a7602212a0220230665767d777c6106547c7c747f760e4c637261727e40647a67707b5c7d0a777a61767067407a747d0a707c7d607a6067767d670660647a67707b03777c7e07637b727d677c7e047b7c7c7840525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e3d03727a77017d01750161096067726167477a7e7601670972717a7f7a677a76600a677a7e766067727e6322137b72617764726176507c7d70666161767d706a0c7776657a70765e767e7c616a087f727d74667274760a6176607c7f66677a7c7d0f7265727a7f4176607c7f66677a7c7d0960706176767d477c630a60706176767d5f767567107776657a7076437a6b767f4172677a7c0a63617c77667067406671077172676776616a016309677c66707b5a7d757c08677a7e76697c7d760a677a7e766067727e6321077463665a7d757c0b7960557c7d67605f7a60670b637f66747a7d605f7a60670a677a7e766067727e63200a76657661507c7c787a760767674c60707a77017e0b606a7d67726b5661617c610c7d72677a65765f767d74677b056167705a43097563457661607a7c7d0b4c4c657661607a7c7d4c4c08707f7a767d675a770a677a7e766067727e63270b766b67767d77557a767f77046366607b03727f7f04677b767d097172607625274c707b0c75617c7e507b7261507c7776067125274c2023022022087172607625274c23022021087172607625274c22022020087172607625274c2102202702202602202507747667477a7e760220240b777c7e5d7c6745727f7a77096066716067617a7d740863617c677c707c7f02202b02202a01230e222323232323232322222323232302272302272207757c616176727f02272104717c776a096067617a7d747a756a02686e0b717c776a45727f216067610a717c776a4c7b72607b2e01350366617f02272005626676616a0a72607c7f774c607a747d096372677b7d727e762e0967674c6476717a772e063566667a772e0227270227260e4c716a6776774c6076704c777a770227250a27212a272a2524212a25097576457661607a7c7d0227240e4c232151274925647c232323232202272b02272a05607f7a7076022623074056505a5d555c037d7c6409677a7e766067727e6305757f7c7c610661727d777c7e0f7476674747447671507c7c787a7660056767647a770867674c6476717a770767674476715a770b67674c6476717a774c65210967674476717a7745210761667d7d7a7d7405757f66607b087e7c65765f7a60670660637f7a70760671765e7c657609707f7a70785f7a6067077176507f7a70780c78766a717c7261775f7a60670a717658766a717c7261770b7270677a657640677267760b647a7d777c6440677267760360477e05676172707808667d7a67477a7e76037270700a667d7a67527e7c667d670871767b72657a7c61077e6074476a637603645a5707727a775f7a60670b63617a6572706a5e7c777606706660677c7e067260607a747d0f4456514c5756455a50564c5a5d555c0479607c7d0a6176747a7c7d507c7d75096176637c616746617f04766b7a67094b3e5e403e404746510c4b3e5e403e43524a5f5c525720232323232323232323232323232323232323232323232323232323232323232320772722772b70772a2b75232371212327762a2b23232a2a2b7670752b272124760165066671707c7776067776707c777602262202262102262002262702262602262502262402262b02262a022523022522022521022520", [, , void 0, void 0 !== _0x178cef ? _0x178cef : void 0, { boe: !1, aid: 0, dfp: !1, sdi: !1, enablePathList: [], _enablePathListRegex: [/\/web\/report/], urlRewriteRules: [], _urlRewriteRules: [], initialized: !1, enableTrack: !1, track: { unitTime: 0, unitAmount: 0, fre: 0 }, triggerUnload: !1, region: "", regionConf: {}, umode: 0, v: !1, perf: !1, xxbg: !0 }, () => 0, () => "03v", { ubcode: 0 }, { bogusIndex: 0, msNewTokenList: [], moveList: [], clickList: [], keyboardList: [], activeState: [], aidList: [], envcode: 0, msToken: "", msStatus: 0, __ac_testid: "", ttwid: "", tt_webid: "", tt_webid_v2: "" }, void 0 !== _0x4e46b6 ? _0x4e46b6 : void 0, { userAgent: b }, (e, b) => { let a = new Uint8Array(3); return a[0] = e / 256, a[1] = e % 256, a[2] = b % 256, String.fromCharCode.apply(null, a) }, (e, b) => { let a, f = [], c = 0, r = ""; for (let e = 0; e < 256; e++) f[e] = e; for (let b = 0; b < 256; b++) c = (c + f[b] + e.charCodeAt(b % e.length)) % 256, a = f[b], f[b] = f[c], f[c] = a; let t = 0; c = 0; for (let e = 0; e < b.length; e++) c = (c + f[t = (t + 1) % 256]) % 256, a = f[t], f[t] = f[c], f[c] = a, r += String.fromCharCode(b.charCodeAt(e) ^ f[(f[t] + f[c]) % 256]); return r }, (e, b) => jsvmp("484e4f4a403f524300281018f7b851f02d296e5b00000000000004a21b0002001d1d001e1b00131e00061a001d001f1b000b070200200200210d1b000b070200220200230d1b000b070200240200250d1b000b070200260200270d1b001b000b071b000b05191d00031b000200001d00281b0048001d00291b000b041e002a1b000b0b4803283b1700f11b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f480833301b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b08221e002d241b000b0a490fc02f4806340a000110281d00281b00220b091b000b08221e002d241b000b0a483f2f0a000110281d002816ff031b000b041e002a1b000b0b294800391700e01b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b041e002a1b000b0b3917001e1b000b04221e002b241b000b0b0a0001104900ff2f4808331600054800301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b041e002a1b000b0b3917001e1b000b08221e002d241b000b0a490fc02f4806340a0001101600071b000b06281d00281b00220b091b000b06281d00281b000b090000002e000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b72615267", [, , , , e, b]), "undefined" != typeof Date ? Date : void 0, () => 0, (e, b, a, f, c, r, t, d, i, n, s, o, l, _, x, u, h, p, y) => { let v = new Uint8Array(19); return v[0] = e, v[1] = s, v[2] = b, v[3] = o, v[4] = a, v[5] = l, v[6] = f, v[7] = _, v[8] = c, v[9] = x, v[10] = r, v[11] = u, v[12] = t, v[13] = h, v[14] = d, v[15] = p, v[16] = i, v[17] = y, v[18] = n, String.fromCharCode.apply(null, v) }, e => String.fromCharCode(e), (e, b, a) => String.fromCharCode(e) + String.fromCharCode(b) + a, (e, b) => jsvmp("484e4f4a403f524300281018f7b851f02d296e5b00000000000004a21b0002001d1d001e1b00131e00061a001d001f1b000b070200200200210d1b000b070200220200230d1b000b070200240200250d1b000b070200260200270d1b001b000b071b000b05191d00031b000200001d00281b0048001d00291b000b041e002a1b000b0b4803283b1700f11b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f480833301b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b08221e002d241b000b0a490fc02f4806340a000110281d00281b00220b091b000b08221e002d241b000b0a483f2f0a000110281d002816ff031b000b041e002a1b000b0b294800391700e01b001b000b04221e002b241b001e0029222d1b00241d00290a0001104900ff2f4810331b000b041e002a1b000b0b3917001e1b000b04221e002b241b000b0b0a0001104900ff2f4808331600054800301d002c1b00220b091b000b08221e002d241b000b0a4a00fc00002f4812340a000110281d00281b00220b091b000b08221e002d241b000b0a4a0003f0002f480c340a000110281d00281b00220b091b000b041e002a1b000b0b3917001e1b000b08221e002d241b000b0a490fc02f4806340a0001101600071b000b06281d00281b00220b091b000b06281d00281b000b090000002e000126207575757575757575757575757575757575757575757575757575757575757575012b0e7776757a7d7643617c637661676a027a77065c717976706708777671667474766107767d65707c77760374766707707c7d607c7f7607757a61767166740a7c66677661447a77677b0a7a7d7d7661447a77677b0b7c666776615b767a747b670b7a7d7d76615b767a747b6709666076615274767d670b677c5f7c64766150726076077a7d77766b5c7508767f767067617c7d09667d7776757a7d76770963617c677c676a637608677c4067617a7d740470727f7f0763617c7076606010487c71797670673363617c707660604e067c717976706705677a677f76047d7c7776012e0125012402602341525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a383c2e0260224157787763747b2749586042512b233c5e75656420254b5a22412126384446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e0260214157787763747b2749586042512b233c5e75656420254b5a224121263e4446527f567a245d5f717c624a475c4366697e5579597d616a6b2a5b45547072406750762e02602041525150575655545b5a59585f5e5d5c43424140474645444b4a49727170777675747b7a79787f7e7d7c63626160676665646b6a6923222120272625242b2a3e4c2e012a022222067f767d74677b0a707b7261507c7776526702222306707b72615267", [, , , , e, b]), , sign, e, void 0]) } module.exports = { sign }; ================================================ FILE: extractors/douyin/types.go ================================================ package douyin type douyinData struct { StatusCode int `json:"status_code"` AwemeDetail struct { AdmireAuth struct { AdmireButton int `json:"admire_button"` IsAdmire int `json:"is_admire"` IsShowAdmireButton int `json:"is_show_admire_button"` IsShowAdmireTab int `json:"is_show_admire_tab"` } `json:"admire_auth"` Anchors interface{} `json:"anchors"` Author struct { AvatarThumb struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"avatar_thumb"` CfList interface{} `json:"cf_list"` CloseFriendType int `json:"close_friend_type"` ContactsStatus int `json:"contacts_status"` ContrailList interface{} `json:"contrail_list"` CoverURL []struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_url"` CreateTime int `json:"create_time"` CustomVerify string `json:"custom_verify"` DataLabelList interface{} `json:"data_label_list"` EndorsementInfoList interface{} `json:"endorsement_info_list"` EnterpriseVerifyReason string `json:"enterprise_verify_reason"` FavoritingCount int `json:"favoriting_count"` FollowStatus int `json:"follow_status"` FollowerCount int `json:"follower_count"` FollowerListSecondaryInformationStruct interface{} `json:"follower_list_secondary_information_struct"` FollowerStatus int `json:"follower_status"` FollowingCount int `json:"following_count"` ImRoleIds interface{} `json:"im_role_ids"` IsAdFake bool `json:"is_ad_fake"` IsBlockedV2 bool `json:"is_blocked_v2"` IsBlockingV2 bool `json:"is_blocking_v2"` IsCf int `json:"is_cf"` MaxFollowerCount int `json:"max_follower_count"` Nickname string `json:"nickname"` NotSeenItemIDList interface{} `json:"not_seen_item_id_list"` NotSeenItemIDListV2 interface{} `json:"not_seen_item_id_list_v2"` OfflineInfoList interface{} `json:"offline_info_list"` PersonalTagList interface{} `json:"personal_tag_list"` PreventDownload bool `json:"prevent_download"` RiskNoticeText string `json:"risk_notice_text"` SecUID string `json:"sec_uid"` Secret int `json:"secret"` ShareInfo struct { ShareDesc string `json:"share_desc"` ShareDescInfo string `json:"share_desc_info"` ShareQrcodeURL struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"share_qrcode_url"` ShareTitle string `json:"share_title"` ShareTitleMyself string `json:"share_title_myself"` ShareTitleOther string `json:"share_title_other"` ShareURL string `json:"share_url"` ShareWeiboDesc string `json:"share_weibo_desc"` } `json:"share_info"` ShortID string `json:"short_id"` Signature string `json:"signature"` SignatureExtra interface{} `json:"signature_extra"` SpecialPeopleLabels interface{} `json:"special_people_labels"` Status int `json:"status"` TextExtra interface{} `json:"text_extra"` TotalFavorited int `json:"total_favorited"` UID string `json:"uid"` UniqueID string `json:"unique_id"` UserAge int `json:"user_age"` UserCanceled bool `json:"user_canceled"` UserPermissions interface{} `json:"user_permissions"` VerificationType int `json:"verification_type"` } `json:"author"` AuthorMaskTag int `json:"author_mask_tag"` AuthorUserID int64 `json:"author_user_id"` AwemeACL struct { DownloadMaskPanel struct { Code int `json:"code"` ShowType int `json:"show_type"` } `json:"download_mask_panel"` } `json:"aweme_acl"` AwemeControl struct { CanComment bool `json:"can_comment"` CanForward bool `json:"can_forward"` CanShare bool `json:"can_share"` CanShowComment bool `json:"can_show_comment"` } `json:"aweme_control"` AwemeID string `json:"aweme_id"` AwemeType int `json:"aweme_type"` ChallengePosition interface{} `json:"challenge_position"` ChapterList interface{} `json:"chapter_list"` CollectStat int `json:"collect_stat"` CommentGid int64 `json:"comment_gid"` CommentList interface{} `json:"comment_list"` CommentPermissionInfo struct { CanComment bool `json:"can_comment"` CommentPermissionStatus int `json:"comment_permission_status"` ItemDetailEntry bool `json:"item_detail_entry"` PressEntry bool `json:"press_entry"` ToastGuide bool `json:"toast_guide"` } `json:"comment_permission_info"` CommerceConfigData interface{} `json:"commerce_config_data"` CommonBarInfo string `json:"common_bar_info"` ComponentInfoV2 string `json:"component_info_v2"` CoverLabels interface{} `json:"cover_labels"` CreateTime int `json:"create_time"` Desc string `json:"desc"` DiggLottie struct { CanBomb int `json:"can_bomb"` LottieID string `json:"lottie_id"` } `json:"digg_lottie"` DisableRelationBar int `json:"disable_relation_bar"` DislikeDimensionList interface{} `json:"dislike_dimension_list"` DuetAggregateInMusicTab bool `json:"duet_aggregate_in_music_tab"` Duration int `json:"duration"` FeedCommentConfig struct { AuthorAuditStatus int `json:"author_audit_status"` InputConfigText string `json:"input_config_text"` } `json:"feed_comment_config"` Geofencing []interface{} `json:"geofencing"` GeofencingRegions interface{} `json:"geofencing_regions"` GroupID string `json:"group_id"` HybridLabel interface{} `json:"hybrid_label"` ImageAlbumMusicInfo struct { BeginTime int `json:"begin_time"` EndTime int `json:"end_time"` Volume int `json:"volume"` } `json:"image_album_music_info"` ImageInfos interface{} `json:"image_infos"` ImageList interface{} `json:"image_list"` Images []struct { DownloadURLList []string `json:"download_url_list"` Height int `json:"height"` MaskURLList interface{} `json:"mask_url_list"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"images"` ImgBitrate []struct { Images []struct { DownloadURLList []string `json:"download_url_list"` Height int `json:"height"` MaskURLList interface{} `json:"mask_url_list"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"images"` Name string `json:"name"` } `json:"img_bitrate"` ImpressionData struct { GroupIDListA []int64 `json:"group_id_list_a"` GroupIDListB []int64 `json:"group_id_list_b"` SimilarIDListA interface{} `json:"similar_id_list_a"` SimilarIDListB interface{} `json:"similar_id_list_b"` } `json:"impression_data"` InteractionStickers interface{} `json:"interaction_stickers"` IsAds bool `json:"is_ads"` IsCollectsSelected int `json:"is_collects_selected"` IsDuetSing bool `json:"is_duet_sing"` IsImageBeat bool `json:"is_image_beat"` IsLifeItem bool `json:"is_life_item"` IsMultiContent int `json:"is_multi_content"` IsStory int `json:"is_story"` IsTop int `json:"is_top"` ItemWarnNotification struct { Content string `json:"content"` Show bool `json:"show"` Type int `json:"type"` } `json:"item_warn_notification"` LabelTopText interface{} `json:"label_top_text"` LongVideo interface{} `json:"long_video"` Music struct { Album string `json:"album"` ArtistUserInfos interface{} `json:"artist_user_infos"` Artists []interface{} `json:"artists"` AuditionDuration int `json:"audition_duration"` Author string `json:"author"` AuthorDeleted bool `json:"author_deleted"` AuthorPosition interface{} `json:"author_position"` AuthorStatus int `json:"author_status"` AvatarLarge struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"avatar_large"` AvatarMedium struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"avatar_medium"` AvatarThumb struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"avatar_thumb"` BindedChallengeID int `json:"binded_challenge_id"` CanBackgroundPlay bool `json:"can_background_play"` CollectStat int `json:"collect_stat"` CoverColorHsv struct { H int `json:"h"` S int `json:"s"` V int `json:"v"` } `json:"cover_color_hsv"` CoverHd struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_hd"` CoverLarge struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_large"` CoverMedium struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_medium"` CoverThumb struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_thumb"` DmvAutoShow bool `json:"dmv_auto_show"` DspStatus int `json:"dsp_status"` Duration int `json:"duration"` EndTime int `json:"end_time"` ExternalSongInfo []interface{} `json:"external_song_info"` Extra string `json:"extra"` ID int64 `json:"id"` IDStr string `json:"id_str"` IsAudioURLWithCookie bool `json:"is_audio_url_with_cookie"` IsCommerceMusic bool `json:"is_commerce_music"` IsDelVideo bool `json:"is_del_video"` IsMatchedMetadata bool `json:"is_matched_metadata"` IsOriginal bool `json:"is_original"` IsOriginalSound bool `json:"is_original_sound"` IsPgc bool `json:"is_pgc"` IsRestricted bool `json:"is_restricted"` IsVideoSelfSee bool `json:"is_video_self_see"` LunaInfo struct { HasCopyright bool `json:"has_copyright"` IsLunaUser bool `json:"is_luna_user"` } `json:"luna_info"` LyricShortPosition interface{} `json:"lyric_short_position"` MatchedPgcSound struct { Author string `json:"author"` CoverMedium struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_medium"` MixedAuthor string `json:"mixed_author"` MixedTitle string `json:"mixed_title"` Title string `json:"title"` } `json:"matched_pgc_sound"` Mid string `json:"mid"` MusicChartRanks interface{} `json:"music_chart_ranks"` MusicStatus int `json:"music_status"` MusicianUserInfos interface{} `json:"musician_user_infos"` MuteShare bool `json:"mute_share"` OfflineDesc string `json:"offline_desc"` OwnerHandle string `json:"owner_handle"` OwnerID string `json:"owner_id"` OwnerNickname string `json:"owner_nickname"` PgcMusicType int `json:"pgc_music_type"` PlayURL struct { Height int `json:"height"` URI string `json:"uri"` URLKey string `json:"url_key"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"play_url"` Position interface{} `json:"position"` PreventDownload bool `json:"prevent_download"` PreventItemDownloadStatus int `json:"prevent_item_download_status"` PreviewEndTime int `json:"preview_end_time"` PreviewStartTime float64 `json:"preview_start_time"` ReasonType int `json:"reason_type"` Redirect bool `json:"redirect"` SchemaURL string `json:"schema_url"` SearchImpr struct { EntityID string `json:"entity_id"` } `json:"search_impr"` SecUID string `json:"sec_uid"` ShootDuration int `json:"shoot_duration"` Song struct { Artists interface{} `json:"artists"` ID int64 `json:"id"` IDStr string `json:"id_str"` } `json:"song"` SourcePlatform int `json:"source_platform"` StartTime int `json:"start_time"` Status int `json:"status"` TagList interface{} `json:"tag_list"` Title string `json:"title"` UnshelveCountries interface{} `json:"unshelve_countries"` UserCount int `json:"user_count"` VideoDuration int `json:"video_duration"` } `json:"music"` NicknamePosition interface{} `json:"nickname_position"` OriginCommentIds interface{} `json:"origin_comment_ids"` OriginTextExtra []interface{} `json:"origin_text_extra"` OriginalImages interface{} `json:"original_images"` PackedClips interface{} `json:"packed_clips"` PhotoSearchEntrance struct { EcomType int `json:"ecom_type"` } `json:"photo_search_entrance"` Position interface{} `json:"position"` PressPanelInfo string `json:"press_panel_info"` PreviewTitle string `json:"preview_title"` PreviewVideoStatus int `json:"preview_video_status"` Promotions []interface{} `json:"promotions"` Rate int `json:"rate"` Region string `json:"region"` RelationLabels interface{} `json:"relation_labels"` SearchImpr struct { EntityID string `json:"entity_id"` EntityType string `json:"entity_type"` } `json:"search_impr"` SeriesPaidInfo struct { ItemPrice int `json:"item_price"` SeriesPaidStatus int `json:"series_paid_status"` } `json:"series_paid_info"` ShareInfo struct { ShareDesc string `json:"share_desc"` ShareDescInfo string `json:"share_desc_info"` ShareLinkDesc string `json:"share_link_desc"` ShareURL string `json:"share_url"` } `json:"share_info"` ShareURL string `json:"share_url"` ShouldOpenAdReport bool `json:"should_open_ad_report"` ShowFollowButton struct { } `json:"show_follow_button"` SocialTagList interface{} `json:"social_tag_list"` StandardBarInfoList interface{} `json:"standard_bar_info_list"` Statistics struct { AdmireCount int `json:"admire_count"` AwemeID string `json:"aweme_id"` CollectCount int `json:"collect_count"` CommentCount int `json:"comment_count"` DiggCount int `json:"digg_count"` PlayCount int `json:"play_count"` ShareCount int `json:"share_count"` } `json:"statistics"` Status struct { AllowShare bool `json:"allow_share"` AwemeID string `json:"aweme_id"` InReviewing bool `json:"in_reviewing"` IsDelete bool `json:"is_delete"` IsProhibited bool `json:"is_prohibited"` ListenVideoStatus int `json:"listen_video_status"` PartSee int `json:"part_see"` PrivateStatus int `json:"private_status"` ReviewResult struct { ReviewStatus int `json:"review_status"` } `json:"review_result"` } `json:"status"` TextExtra []struct { End int `json:"end"` HashtagID string `json:"hashtag_id"` HashtagName string `json:"hashtag_name"` IsCommerce bool `json:"is_commerce"` Start int `json:"start"` Type int `json:"type"` } `json:"text_extra"` UniqidPosition interface{} `json:"uniqid_position"` UserDigged int `json:"user_digged"` Video struct { BigThumbs []struct { Duration float64 `json:"duration"` Fext string `json:"fext"` ImgNum int `json:"img_num"` ImgURL string `json:"img_url"` ImgXLen int `json:"img_x_len"` ImgXSize int `json:"img_x_size"` ImgYLen int `json:"img_y_len"` ImgYSize int `json:"img_y_size"` Interval float64 `json:"interval"` URI string `json:"uri"` } `json:"big_thumbs"` BitRate []struct { FPS int `json:"FPS"` HDRBit string `json:"HDR_bit"` HDRType string `json:"HDR_type"` BitRate int `json:"bit_rate"` GearName string `json:"gear_name"` IsBytevc1 int `json:"is_bytevc1"` IsH265 int `json:"is_h265"` PlayAddr struct { DataSize int `json:"data_size"` FileCs string `json:"file_cs"` FileHash string `json:"file_hash"` Height int `json:"height"` URI string `json:"uri"` URLKey string `json:"url_key"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"play_addr"` QualityType int `json:"quality_type"` } `json:"bit_rate"` Cover struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover"` CoverOriginalScale struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"cover_original_scale"` Duration int `json:"duration"` DynamicCover struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"dynamic_cover"` Height int `json:"height"` IsH265 int `json:"is_h265"` IsLongVideo int `json:"is_long_video"` IsSourceHDR int `json:"is_source_HDR"` Meta string `json:"meta"` OriginCover struct { Height int `json:"height"` URI string `json:"uri"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"origin_cover"` PlayAddr struct { DataSize int `json:"data_size"` FileCs string `json:"file_cs"` FileHash string `json:"file_hash"` Height int `json:"height"` URI string `json:"uri"` URLKey string `json:"url_key"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"play_addr"` PlayAddr265 struct { DataSize int `json:"data_size"` FileCs string `json:"file_cs"` FileHash string `json:"file_hash"` Height int `json:"height"` URI string `json:"uri"` URLKey string `json:"url_key"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"play_addr_265"` PlayAddrH264 struct { DataSize int `json:"data_size"` FileCs string `json:"file_cs"` FileHash string `json:"file_hash"` Height int `json:"height"` URI string `json:"uri"` URLKey string `json:"url_key"` URLList []string `json:"url_list"` Width int `json:"width"` } `json:"play_addr_h264"` Ratio string `json:"ratio"` VideoModel string `json:"video_model"` Width int `json:"width"` } `json:"video"` VideoLabels interface{} `json:"video_labels"` VideoTag []struct { Level int `json:"level"` TagID int `json:"tag_id"` TagName string `json:"tag_name"` } `json:"video_tag"` VideoText []interface{} `json:"video_text"` WannaTag struct { } `json:"wanna_tag"` } `json:"aweme_detail"` Extra struct { Now int64 `json:"now"` Logid string `json:"logid"` } `json:"extra"` } ================================================ FILE: extractors/douyu/douyu.go ================================================ package douyu import ( "encoding/json" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("douyu", New()) } type douyuData struct { Error int `json:"error"` Data struct { VideoURL string `json:"video_url"` } `json:"data"` } type douyuURLInfo struct { URL string Size int64 } func douyuM3u8(url string) ([]douyuURLInfo, int64, error) { var ( data []douyuURLInfo temp douyuURLInfo size, totalSize int64 err error ) urls, err := utils.M3u8URLs(url) if err != nil { return nil, 0, err } for _, u := range urls { size, err = request.Size(u, url) if err != nil { return nil, 0, err } totalSize += size temp = douyuURLInfo{ URL: u, Size: size, } data = append(data, temp) } return data, totalSize, nil } type extractor struct{} // New returns a douyu extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { var err error liveVid := utils.MatchOneOf(url, `https?://www.douyu.com/(\S+)`) if liveVid != nil { return nil, errors.New("暂不支持斗鱼直播") } html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } titles := utils.MatchOneOf(html, `(.*?)`) if titles == nil || len(titles) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } title := titles[1] vids := utils.MatchOneOf(url, `https?://v.douyu.com/show/(\S+)`) if vids == nil || len(vids) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } vid := vids[1] dataString, err := request.Get("http://vmobile.douyu.com/video/getInfo?vid="+vid, url, nil) if err != nil { return nil, errors.WithStack(err) } dataDict := new(douyuData) if err := json.Unmarshal([]byte(dataString), dataDict); err != nil { return nil, errors.WithStack(err) } m3u8URLs, totalSize, err := douyuM3u8(dataDict.Data.VideoURL) if err != nil { return nil, errors.WithStack(err) } urls := make([]*extractors.Part, len(m3u8URLs)) for index, u := range m3u8URLs { urls[index] = &extractors.Part{ URL: u.URL, Size: u.Size, Ext: "ts", } } streams := map[string]*extractors.Stream{ "default": { Parts: urls, Size: totalSize, }, } return []*extractors.Data{ { Site: "斗鱼 douyu.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/douyu/douyu_test.go ================================================ package douyu import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://v.douyu.com/show/l0Q8mMY3wZqv49Ad", Title: "每日撸报_每日撸报：有些人死了其实它还可以把你带走_斗鱼视频 - 最6的弹幕视频网站", Size: 10558080, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { New().Extract(tt.args.URL, extractors.Options{}) }) } } ================================================ FILE: extractors/eporner/eporner.go ================================================ package eporner import ( "net/url" "strconv" "strings" "github.com/PuerkitoBio/goquery" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("eporner", New()) } const ( downloadclass = ".dloaddivcol" ) type src struct { url string quality string sizestr string size int64 } func getSrcMeta(text string) *src { sti := strings.Index(text, "(") ste := strings.Index(text, ")") itext := text[sti+1 : ste] strs := strings.Split(itext, ",") s := &src{} if len(strs) == 2 { s.quality = strings.Trim(strs[0], " ") s.sizestr = strings.Trim(strs[1], " ") } if s.sizestr == "" { s.size = 0 return s } valunit := strings.Split(s.sizestr, " ") val, err := strconv.ParseFloat(valunit[0], 64) if err != nil { s.size = 0 return s } unit := valunit[1] switch unit { case "KB": s.size = int64(val * 1024) case "MB": s.size = int64(val * 1024 * 1024) case "GB": s.size = int64(val * 1024 * 1024 * 1024) default: s.size = int64(val) } return s } func getSrc(html string) []*src { srcs := []*src{} d, err := parser.GetDoc(html) if err != nil { return nil } d.Find(downloadclass).Each(func(i int, s *goquery.Selection) { s.Contents().Each(func(i int, s *goquery.Selection) { for ns := range s.Nodes { n := s.Get(ns) if n.Data == "a" { var sr *src if n.FirstChild != nil { sr = getSrcMeta(n.FirstChild.Data) } for _, a := range n.Attr { if a.Key == "href" { sr.url = a.Val } } srcs = append(srcs, sr) } } }) }) return srcs } type extractor struct{} // New returns a eporner extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(u, u, nil) if err != nil { return nil, errors.WithStack(err) } var title string desc := utils.MatchOneOf(html, `(.+?)`) if len(desc) > 1 { title = desc[1] } else { title = "eporner" } uu, err := url.Parse(u) if err != nil { return nil, errors.WithStack(err) } srcs := getSrc(html) streams := make(map[string]*extractors.Stream, len(srcs)) for _, src := range srcs { srcurl := uu.Scheme + "://" + uu.Host + src.url // skipping an extra HEAD request to the URL. // size, err := request.Size(srcurl, u) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: srcurl, Size: src.size, Ext: "mp4", } streams[src.quality] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: src.size, Quality: src.quality, } } return []*extractors.Data{ { Site: "EPORNER eporner.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: u, }, }, nil } ================================================ FILE: extractors/eporner/eporner_test.go ================================================ package eporner import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.eporner.com/video-mbubfvXYFip/dirtywivesclub-becky-bandini/", Quality: "1080p", Size: 1525510307, Title: "DirtyWivesClub - Becky Bandini - EPORNER", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/errors.go ================================================ package extractors import ( "errors" ) var ( // ErrURLParseFailed defines url parse failed error. ErrURLParseFailed = errors.New("url parse failed") ErrInvalidRegularExpression = errors.New("invalid regular expression") ErrURLQueryParamsParseFailed = errors.New("url query params parse failed") ErrBodyParseFailed = errors.New("body parse failed") ) ================================================ FILE: extractors/extractors.go ================================================ package extractors import ( "net/url" "strings" "sync" "github.com/pkg/errors" "github.com/iawia002/lux/utils" ) var lock sync.RWMutex var extractorMap = make(map[string]Extractor) // Register registers an Extractor. func Register(domain string, e Extractor) { lock.Lock() extractorMap[domain] = e lock.Unlock() } // Extract is the main function to extract the data. func Extract(u string, option Options) ([]*Data, error) { u = strings.TrimSpace(u) var domain string bilibiliShortLink := utils.MatchOneOf(u, `^(av|BV|ep)\w+`) if len(bilibiliShortLink) > 1 { bilibiliURL := map[string]string{ "av": "https://www.bilibili.com/video/", "BV": "https://www.bilibili.com/video/", "ep": "https://www.bilibili.com/bangumi/play/", } domain = "bilibili" u = bilibiliURL[bilibiliShortLink[1]] + u } else { u, err := url.ParseRequestURI(u) if err != nil { return nil, errors.WithStack(err) } if u.Host == "haokan.baidu.com" { domain = "haokan" } else if u.Host == "xhslink.com" { domain = "xiaohongshu" } else { domain = utils.Domain(u.Host) } } extractor := extractorMap[domain] if extractor == nil { extractor = extractorMap[""] } videos, err := extractor.Extract(u, option) if err != nil { return nil, errors.WithStack(err) } for _, v := range videos { v.FillUpStreamsData() } return videos, nil } ================================================ FILE: extractors/facebook/facebook.go ================================================ package facebook import ( "regexp" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("facebook", New()) } type extractor struct{} // New returns a facebook extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { var err error html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } titles := utils.MatchOneOf(html, `([^<]+)`) if titles == nil || len(titles) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } title := strings.TrimSpace(titles[1]) title = regexp.MustCompile(`\n+`).ReplaceAllString(title, " ") qualityRegMap := map[string]*regexp.Regexp{ "sd": regexp.MustCompile(`"playable_url":\s*"([^"]+)"`), // "hd": regexp.MustCompile(`"playable_url_quality_hd":\s*"([^"]+)"`), } streams := make(map[string]*extractors.Stream, 2) for quality, qualityReg := range qualityRegMap { matcher := qualityReg.FindStringSubmatch(html) if len(matcher) == 0 { continue } u := strings.ReplaceAll(matcher[1], "\\", "") size, err := request.Size(u, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: u, Size: size, Ext: "mp4", } streams[quality] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, Quality: quality, } } return []*extractors.Data{ { Site: "Facebook facebook.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/facebook/facebook_test.go ================================================ package facebook import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.facebook.com/100058251872436/videos/424557726111987", Title: "Роман Грищук - Підтримка з Японії 🇯🇵 Гурт Yokohama Sisters 👏", Size: 1441128, Quality: "sd", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/geekbang/geekbang.go ================================================ package geekbang import ( "encoding/json" "fmt" "net/http" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("geekbang", New()) } type geekData struct { Code int `json:"code"` Error json.RawMessage `json:"error"` Data struct { VideoID string `json:"video_id"` Title string `json:"article_sharetitle"` ColumnHadSub bool `json:"column_had_sub"` } `json:"data"` } type videoPlayAuth struct { Code int `json:"code"` Error json.RawMessage `json:"error"` Data struct { PlayAuth string `json:"play_auth"` } `json:"data"` } type playInfo struct { VideoBase struct { VideoID string `json:"VideoId"` Title string `json:"Title"` CoverURL string `josn:"CoverURL"` } `json:"VideoBase"` PlayInfoList struct { PlayInfo []struct { URL string `json:"PlayURL"` Size int64 `json:"Size"` Definition string `json:"Definition"` } `json:"PlayInfo"` } `json:"PlayInfoList"` } type geekURLInfo struct { URL string Size int64 } func geekM3u8(url string) ([]geekURLInfo, error) { var ( data []geekURLInfo temp geekURLInfo size int64 err error ) urls, err := utils.M3u8URLs(url) if err != nil { return nil, errors.WithStack(err) } for _, u := range urls { temp = geekURLInfo{ URL: u, Size: size, } data = append(data, temp) } return data, nil } type extractor struct{} // New returns a geekbang extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) { var err error matches := utils.MatchOneOf(url, `https?://time.geekbang.org/course/detail/(\d+)-(\d+)`) if matches == nil || len(matches) < 3 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } // Get video information heanders := map[string]string{"Origin": "https://time.geekbang.org", "Content-Type": "application/json", "Referer": url} params := strings.NewReader(fmt.Sprintf(`{"id": %q}`, matches[2])) res, err := request.Request(http.MethodPost, "https://time.geekbang.org/serv/v1/article", params, heanders) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var data geekData if err = json.NewDecoder(res.Body).Decode(&data); err != nil { return nil, errors.WithStack(err) } if data.Code < 0 { return nil, errors.New(string(data.Error)) } if data.Data.VideoID == "" && !data.Data.ColumnHadSub { return nil, errors.New("请先购买课程，或使用Cookie登录。") } // Get video license token information params = strings.NewReader("{\"source_type\":1,\"aid\":" + matches[2] + ",\"video_id\":\"" + data.Data.VideoID + "\"}") res, err = request.Request(http.MethodPost, "https://time.geekbang.org/serv/v3/source_auth/video_play_auth", params, heanders) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var playAuth videoPlayAuth if err = json.NewDecoder(res.Body).Decode(&playAuth); err != nil { return nil, errors.WithStack(err) } if playAuth.Code < 0 { return nil, errors.New(string(playAuth.Error)) } // Get video playback information heanders = map[string]string{"Accept-Encoding": ""} res, err = request.Request(http.MethodGet, "http://ali.mantv.top/play/info?playAuth="+playAuth.Data.PlayAuth, nil, heanders) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var playInfo playInfo if err = json.NewDecoder(res.Body).Decode(&playInfo); err != nil { return nil, errors.WithStack(err) } title := data.Data.Title streams := make(map[string]*extractors.Stream, len(playInfo.PlayInfoList.PlayInfo)) for _, media := range playInfo.PlayInfoList.PlayInfo { m3u8URLs, err := geekM3u8(media.URL) if err != nil { return nil, errors.WithStack(err) } urls := make([]*extractors.Part, len(m3u8URLs)) for index, u := range m3u8URLs { urls[index] = &extractors.Part{ URL: u.URL, Size: u.Size, Ext: "ts", } } streams[media.Definition] = &extractors.Stream{ Parts: urls, Size: media.Size, } } return []*extractors.Data{ { Site: "极客时间 geekbang.org", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/geekbang/geekbang_test.go ================================================ package geekbang import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://time.geekbang.org/course/detail/190-97203", Title: "02 | 内容综述 - 玩转webpack", Size: 10752472, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/haokan/haokan.go ================================================ package haokan import ( "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("haokan", New()) } type extractor struct{} // New returns a haokan extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } titles := utils.MatchOneOf(html, `property="og:title"\s+content="(.+?)"`) if titles == nil || len(titles) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } title := titles[1] // 之前的好看网页中，视频地址是放在 video 标签下 urls := utils.MatchOneOf(html, ``) if urls == nil || len(urls) < 2 { // fallbak: 新的好看网页中，视频地址在 json 数据里 urls = utils.MatchOneOf(html, `"playurl":"(http.+?)"`) } if urls == nil || len(urls) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } playurl := strings.Replace(urls[1], `\/`, `/`, -1) size, err := request.Size(playurl, url) if err != nil { return nil, errors.WithStack(err) } _, ext, err := utils.GetNameAndExt(playurl) if err != nil { return nil, errors.WithStack(err) } streams := map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{ { URL: playurl, Size: size, Ext: ext, }, }, Size: size, }, } return []*extractors.Data{ { Site: "好看视频 haokan.baidu.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/haokan/haokan_test.go ================================================ package haokan import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://haokan.baidu.com/v?vid=10057409468467026969", Title: "听歌学英语小学篇（6）：my new pen pal", Size: 2027354, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/hupu/hupu.go ================================================ package hupu import ( "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("hupu", New()) } type extractor struct{} // New returns a hupu extractor. func New() extractors.Extractor { return &extractor{} } func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } var title string titleDesc := utils.MatchOneOf(html, `(.+?)`) if len(titleDesc) > 1 { title = titleDesc[1] } else { title = "hupu video" } var videoUrl string urlDesc := utils.MatchOneOf(html, ``) if len(urlDesc) > 1 { videoUrl = urlDesc[1] } else { return nil, errors.WithStack(extractors.ErrURLParseFailed) } size, err := request.Size(videoUrl, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: videoUrl, Size: size, Ext: "mp4", } quality := "normal" streams := map[string]*extractors.Stream{ quality: { Parts: []*extractors.Part{urlData}, Size: size, Quality: quality, }, } return []*extractors.Data{ { Site: "虎扑 hupu.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/hupu/hupu_test.go ================================================ package hupu import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestHupu(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://bbs.hupu.com/47401018.html?is_reflow=1&cid=84752419&bddid=56KXU5QUJH4VGM26SFPTYTKNI5CFNJMX736TIZ52DXLGUAAMBJVA01&puid=16522089&client=8577E496-4D9B-4E5C-A9DB-A8EF5C1956D2", Title: "结局引起舒适", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { New().Extract(tt.args.URL, extractors.Options{}) }) } } ================================================ FILE: extractors/huya/huya.go ================================================ package huya import ( "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("huya", New()) } type extractor struct{} const huyaVideoHost = "https://videotx-platform.cdn.huya.com/" // New returns a huya extractor. func New() extractors.Extractor { return &extractor{} } func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } var title string titleDesc := utils.MatchOneOf(html, `

(.+?)

`) if len(titleDesc) > 1 { title = titleDesc[1] } else { title = "huya video" } var videoUrl string videoDesc := utils.MatchOneOf(html, `//videotx-platform.cdn.huya.com/(.*)" poster=(.+?)`) if len(videoDesc) > 1 { videoUrl = huyaVideoHost + videoDesc[1] } else { return nil, errors.WithStack(extractors.ErrURLParseFailed) } size, err := request.Size(videoUrl, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: videoUrl, Size: size, Ext: "mp4", } quality := "normal" streams := map[string]*extractors.Stream{ quality: { Parts: []*extractors.Part{urlData}, Size: size, Quality: quality, }, } return []*extractors.Data{ { Site: "虎牙 huya.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/huya/huya_test.go ================================================ package huya import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestHuya(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2", Title: "12.28 集梦薛小谦【封号斗罗】直播名场面", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { New().Extract(tt.args.URL, extractors.Options{}) }) } } ================================================ FILE: extractors/instagram/instagram.go ================================================ package instagram import ( "encoding/json" "fmt" "net" "net/http" netURL "net/url" "regexp" "strings" "time" browser "github.com/EDDYCJY/fake-useragent" "github.com/gocolly/colly/v2" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) var client *http.Client func init() { extractors.Register("instagram", New()) client = &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{ Dial: (&net.Dialer{ Timeout: 5 * time.Second, }).Dial, TLSHandshakeTimeout: 5 * time.Second, }, } } // sliderItemNode contains information about the Instagram post type sliderItemNode struct { DisplayURL string `json:"display_url"` // URL of the Media (resolution is dynamic) IsVideo bool `json:"is_video"` // Is type of the Media equals to video VideoURL string `json:"video_url"` // Direct URL to the Video } func (s sliderItemNode) extractMediaURL() string { if s.IsVideo { return s.VideoURL } return s.DisplayURL } type instagramPayload struct { Media struct { ID string `json:"id"` // Unique ID of the Media SliderItems struct { Edges []struct { Node sliderItemNode `json:"node"` } `json:"edges"` } `json:"edge_sidecar_to_children"` // Children of the Media } `json:"shortcode_media"` // Media } func (s instagramPayload) isEmpty() bool { return s.Media.ID == "" } func getPostWithCode(code string) ([]string, error) { URL := fmt.Sprintf("https://www.instagram.com/p/%v/embed/captioned/", code) var embeddedMediaImage string var embedResponse = instagramPayload{} collector := colly.NewCollector() collector.SetClient(client) var collectorErr error collector.OnHTML("img.EmbeddedMediaImage", func(e *colly.HTMLElement) { embeddedMediaImage = e.Attr("src") }) collector.OnHTML("script", func(e *colly.HTMLElement) { r := regexp.MustCompile(`\\\"gql_data\\\":([\s\S]*)\}\"\}\]\]\,\[\"NavigationMetrics`) match := r.FindStringSubmatch(e.Text) if len(match) < 2 { return } s := strings.ReplaceAll(match[1], `\"`, `"`) s = strings.ReplaceAll(s, `\\/`, `/`) s = strings.ReplaceAll(s, `\\`, `\`) err := json.Unmarshal([]byte(s), &embedResponse) if err != nil { collectorErr = err } }) collector.OnRequest(func(r *colly.Request) { r.Headers.Set("User-Agent", browser.Chrome()) }) if err := collector.Visit(URL); err != nil { return nil, fmt.Errorf("failed to send HTTP request to the Instagram: %v", err) } if collectorErr != nil { return nil, fmt.Errorf("failed to parse the Instagram response: %v", collectorErr) } // If the method one which is JSON parsing didn't fail if !embedResponse.isEmpty() { result := make([]string, 0, len(embedResponse.Media.SliderItems.Edges)) for _, item := range embedResponse.Media.SliderItems.Edges { result = append(result, item.Node.extractMediaURL()) } return result, nil } if embeddedMediaImage != "" { return []string{embeddedMediaImage}, nil } // If every two methods have failed, then return an error return nil, errors.New("failed to fetch the post, the page might be \"private\", or the link is completely wrong") } func extractShortCodeFromLink(link string) (string, error) { values := regexp.MustCompile(`(p|tv|reel|reels\/videos)\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link) if len(values) != 3 { return "", errors.New("couldn't extract the media short code from the link") } return values[2], nil } type extractor struct{} // New returns a instagram extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { u, err := netURL.Parse(url) if err != nil { return nil, errors.WithStack(err) } shortCode, err := extractShortCodeFromLink(u.String()) if err != nil { return nil, errors.WithStack(err) } urls, err := getPostWithCode(shortCode) if err != nil { return nil, errors.WithStack(err) } var totalSize int64 var parts []*extractors.Part for _, u := range urls { _, ext, err := utils.GetNameAndExt(u) if err != nil { return nil, errors.WithStack(err) } fileSize, err := request.Size(u, url) if err != nil { return nil, errors.WithStack(err) } part := &extractors.Part{ URL: u, Size: fileSize, Ext: ext, } parts = append(parts, part) } for _, part := range parts { totalSize += part.Size } streams := map[string]*extractors.Stream{ "default": { Parts: parts, Size: totalSize, }, } return []*extractors.Data{ { Site: "Instagram instagram.com", Title: "Instagram " + shortCode, Type: extractors.DataTypeImage, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/instagram/instagram_test.go ================================================ package instagram import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "video test", args: test.Args{ URL: "https://www.instagram.com/p/BlIka1ZFCNr", Title: "Instagram BlIka1ZFCNr", Size: 992330, }, }, { name: "image test", args: test.Args{ URL: "https://www.instagram.com/p/Bl5oVUyl9Yx", Title: "Instagram Bl5oVUyl9Yx", Size: 250596, }, }, { name: "image album test", args: test.Args{ URL: "https://www.instagram.com/p/Bjyr-gxF4Rb", Title: "Instagram Bjyr-gxF4Rb", Size: 656476, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/iqiyi/iqiyi.go ================================================ package iqiyi import ( "encoding/json" "fmt" "math/rand" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("iqiyi", New(SiteTypeIqiyi)) extractors.Register("iq", New(SiteTypeIQ)) } type iqiyi struct { Code string `json:"code"` Data struct { VP struct { Du string `json:"du"` Tkl []struct { Vs []struct { Bid int `json:"bid"` Scrsz string `json:"scrsz"` Vsize int64 `json:"vsize"` Fs []struct { L string `json:"l"` B int64 `json:"b"` } `json:"fs"` } `json:"vs"` } `json:"tkl"` } `json:"vp"` } `json:"data"` Msg string `json:"msg"` } type iqiyiURL struct { L string `json:"l"` } // SiteType indicates the site type of iqiyi type SiteType int const ( // SiteTypeIQ indicates the site is iq.com SiteTypeIQ SiteType = iota // SiteTypeIqiyi indicates the site is iqiyi.com SiteTypeIqiyi iqReferer = "https://www.iq.com" iqiyiReferer = "https://www.iqiyi.com" ) func getMacID() string { var macID string chars := []string{ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "n", "m", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", } size := len(chars) for i := 0; i < 32; i++ { macID += chars[rand.Intn(size)] } return macID } func getVF(params string) string { var suffix string for j := 0; j < 8; j++ { for k := 0; k < 4; k++ { var v8 int v4 := 13 * (66*k + 27*j) % 35 if v4 >= 10 { v8 = v4 + 88 } else { v8 = v4 + 49 } suffix += string(rune(v8)) // string(97) -> "a" } } params += suffix return utils.Md5(params) } func getVPS(tvid, vid, refer string) (*iqiyi, error) { t := time.Now().Unix() * 1000 host := "http://cache.video.qiyi.com" params := fmt.Sprintf( "/vps?tvid=%s&vid=%s&v=0&qypid=%s_12&src=01012001010000000000&t=%d&k_tag=1&k_uid=%s&rs=1", tvid, vid, tvid, t, getMacID(), ) vf := getVF(params) apiURL := fmt.Sprintf("%s%s&vf=%s", host, params, vf) info, err := request.Get(apiURL, refer, nil) if err != nil { return nil, errors.WithStack(err) } data := new(iqiyi) if err := json.Unmarshal([]byte(info), data); err != nil { return nil, errors.WithStack(err) } return data, nil } type extractor struct { siteType SiteType } // New returns a iqiyi extractor. func New(siteType SiteType) extractors.Extractor { return &extractor{ siteType: siteType, } } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) { refer := iqiyiReferer headers := make(map[string]string) if e.siteType == SiteTypeIQ { headers = map[string]string{ "Accept-Language": "zh-TW", } refer = iqReferer } html, err := request.Get(url, refer, headers) if err != nil { return nil, errors.WithStack(err) } tvid := utils.MatchOneOf( url, `#curid=(.+)_`, `tvid=([^&]+)`, ) if tvid == nil { tvid = utils.MatchOneOf( html, `data-player-tvid="([^"]+)"`, `param\['tvid'\]\s*=\s*"(.+?)"`, `"tvid":"(\d+)"`, `"tvId":(\d+)`, ) } if tvid == nil || len(tvid) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } vid := utils.MatchOneOf( url, `#curid=.+_(.*)$`, `vid=([^&]+)`, ) if vid == nil { vid = utils.MatchOneOf( html, `data-player-videoid="([^"]+)"`, `param\['vid'\]\s*=\s*"(.+?)"`, `"vid":"(\w+)"`, ) } if vid == nil || len(vid) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } doc, err := parser.GetDoc(html) if err != nil { return nil, errors.WithStack(err) } var title string if e.siteType == SiteTypeIqiyi { title = strings.TrimSpace(doc.Find("h1>a").First().Text()) var sub string for _, k := range []string{"span", "em"} { if sub != "" { break } sub = strings.TrimSpace(doc.Find("h1>" + k).First().Text()) } title += sub } else { title = strings.TrimSpace(doc.Find("span#pageMetaTitle").First().Text()) sub := utils.MatchOneOf(html, `"subTitle":"([^"]+)","isoDuration":`) if len(sub) > 1 { title += fmt.Sprintf(" %s", sub[1]) } } if title == "" { title = doc.Find("title").Text() } videoDatas, err := getVPS(tvid[1], vid[1], refer) if err != nil { return nil, errors.WithStack(err) } if videoDatas.Code != "A00000" { return nil, errors.Errorf("can't play this video: %s", videoDatas.Msg) } streams := make(map[string]*extractors.Stream) urlPrefix := videoDatas.Data.VP.Du for _, video := range videoDatas.Data.VP.Tkl[0].Vs { urls := make([]*extractors.Part, len(video.Fs)) for index, v := range video.Fs { realURLData, err := request.Get(urlPrefix+v.L, refer, nil) if err != nil { return nil, errors.WithStack(err) } var realURL iqiyiURL if err = json.Unmarshal([]byte(realURLData), &realURL); err != nil { return nil, errors.WithStack(err) } _, ext, err := utils.GetNameAndExt(realURL.L) if err != nil { return nil, errors.WithStack(err) } urls[index] = &extractors.Part{ URL: realURL.L, Size: v.B, Ext: ext, } } streams[strconv.Itoa(video.Bid)] = &extractors.Stream{ Parts: urls, Size: video.Vsize, Quality: video.Scrsz, } } siteName := "爱奇艺 iqiyi.com" if e.siteType == SiteTypeIQ { siteName = "爱奇艺 iq.com" } return []*extractors.Data{ { Site: siteName, Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/iqiyi/iqiyi_test.go ================================================ package iqiyi import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "http://www.iqiyi.com/v_19rrbdmaj0.html", Title: "新一轮降水将至 冷空气影响中东部地区", Size: 2952228, Quality: "896x504", }, }, { name: "title test 1", args: test.Args{ URL: "http://www.iqiyi.com/v_19rqy2z83w.html", Title: "收了创意视频2018 :58天环球飞行记", Size: 76186786, Quality: "1920x1080", }, }, { name: "curid test 1", args: test.Args{ URL: "https://www.iqiyi.com/v_19rro0jdls.html#curid=350289100_6e6601aae889d0b1004586a52027c321", Title: "Shawn Mendes - Never Be Alone", Size: 79921894, Quality: "1920x800", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New(SiteTypeIqiyi).Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/ixigua/ixigua.go ================================================ package ixigua import ( "encoding/base64" "encoding/json" "fmt" "net/http" "regexp" "strings" browser "github.com/EDDYCJY/fake-useragent" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("ixigua", New()) extractors.Register("toutiao", New()) } type extractor struct{} type Video struct { Title string `json:"title"` Qualities []struct { Quality string `json:"quality"` Size int64 `json:"size"` URL string `json:"url"` Ext string `json:"ext"` } `json:"qualities"` } // New returns a ixigua extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { headers := map[string]string{ "User-Agent": browser.Chrome(), "Cookie": option.Cookie, } // ixigua 有三种格式的 URL // 格式一 https://www.ixigua.com/7053389963487871502 // 格式二 https://v.ixigua.com/RedcbWM/ // 格式三 https://m.toutiao.com/is/dtj1pND/ // 格式二会跳转到格式一 // 格式三会跳转到 https://www.toutiao.com/a7053389963487871502 var finalURL string if strings.HasPrefix(url, "https://www.ixigua.com/") { finalURL = url } if strings.HasPrefix(url, "https://v.ixigua.com/") || strings.HasPrefix(url, "https://m.toutiao.com/") { resp, err := http.Get(url) if err != nil { return nil, errors.WithStack(err) } defer resp.Body.Close() // nolint // follow redirects, https://stackoverflow.com/a/16785343 finalURL = resp.Request.URL.String() } finalURL = strings.ReplaceAll(finalURL, "https://www.toutiao.com/video/", "https://www.ixigua.com/") r := regexp.MustCompile(`(ixigua.com/)(\w+)?`) id := r.FindSubmatch([]byte(finalURL))[2] url2 := fmt.Sprintf("https://www.ixigua.com/%s", string(id)) body, err := request.Get(url2, url, headers) if err != nil { return nil, errors.WithStack(err) } videoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\{.*?\})\<\/script\>`) if videoListJson == nil || len(videoListJson) != 2 { return nil, errors.WithStack(extractors.ErrBodyParseFailed) } videoUrl := videoListJson[1] videoUrl = strings.Replace(videoUrl, ":undefined", ":\"undefined\"", -1) var data xiguanData if err = json.Unmarshal([]byte(videoUrl), &data); err != nil { return nil, errors.WithStack(err) } title := data.AnyVideo.GidInformation.PackerData.Video.Title videoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList streams := make(map[string]*extractors.Stream) for _, v := range videoList { streams[v.Definition] = &extractors.Stream{ Quality: v.Definition, Parts: []*extractors.Part{ { URL: base64Decode(v.MainUrl), Size: v.Size, Ext: v.Vtype, }, }, } } return []*extractors.Data{ { Site: "西瓜视频 ixigua.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } func base64Decode(t string) string { d, _ := base64.StdEncoding.DecodeString(t) return string(d) } ================================================ FILE: extractors/ixigua/ixigua_test.go ================================================ package ixigua import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "test 1", args: test.Args{ URL: "https://www.ixigua.com/7053389963487871502", Title: "漫威斥巨资拍的《永恒族》，刚上架就被多国禁播，究竟拍了什么？", Quality: "1080p", Size: 313091514, }, }, { name: "test 2", args: test.Args{ URL: "https://v.ixigua.com/RedcbWM/", Title: "为长生不老，竟然连小鲛人都杀@中视频伙伴计划官号", Quality: "1080p", Size: 64980732, }, }, { name: "test 3", args: test.Args{ URL: "https://m.toutiao.com/is/dtj1pND/", Title: "卡尔：59杀4200法强小法师，点塔只需一下，W技能瞬秒对方", Quality: "1080p", Size: 468324298, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/ixigua/types.go ================================================ package ixigua type xiguanData struct { AnyVideo struct { GidInformation struct { Gid string `json:"gid"` PackerData struct { Video struct { Title string `json:"title"` PosterUrl string `json:"poster_url"` VideoResource struct { Vid string `json:"vid"` Normal struct { VideoId string `json:"video_id"` VideoList map[string]struct { Definition string `json:"definition"` Quality string `json:"quality"` Vtype string `json:"vtype"` Vwidth int `json:"vwidth"` Vheight int `json:"vheight"` Bitrate int64 `json:"bitrate"` RealBitrate int64 `json:"real_bitrate"` Fps int `json:"fps"` CodecType string `json:"codec_type"` Size int64 `json:"size"` MainUrl string `json:"main_url"` BackupUrl1 string `json:"backup_url_1"` } `json:"video_list"` } `json:"normal"` } `json:"videoResource"` } `json:"video"` Key string `json:"key"` } `json:"packerData"` } `json:"gidInformation"` } `json:"anyVideo"` } ================================================ FILE: extractors/kuaishou/kuaishou.go ================================================ package kuaishou import ( "net/http" "regexp" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("kuaishou", New()) } type extractor struct{} // New returns a kuaishou extractor. func New() extractors.Extractor { return &extractor{} } // fetch url and get the cookie that write by server func fetchCookies(url string, headers map[string]string) (string, error) { res, err := request.Request(http.MethodGet, url, nil, headers) if err != nil { return "", err } defer res.Body.Close() // nolint cookiesArr := make([]string, 0) cookies := res.Cookies() for _, c := range cookies { cookiesArr = append(cookiesArr, c.Name+"="+c.Value) } return strings.Join(cookiesArr, "; "), nil } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { headers := map[string]string{ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0", } cookies, err := fetchCookies(url, headers) if err != nil { return nil, errors.WithStack(err) } headers["Cookie"] = cookies html, err := request.Get(url, url, headers) if err != nil { return nil, errors.WithStack(err) } titles := utils.MatchOneOf(html, `([^<]+)`) if titles == nil || len(titles) < 2 { return nil, errors.New("can not found title") } title := regexp.MustCompile(`\n+`).ReplaceAllString(strings.TrimSpace(titles[1]), " ") qualityRegMap := map[string]*regexp.Regexp{ "sd": regexp.MustCompile(`"photoUrl":\s*"([^"]+)"`), } streams := make(map[string]*extractors.Stream, 1) for quality, qualityReg := range qualityRegMap { matcher := qualityReg.FindStringSubmatch(html) if len(matcher) != 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } u := strings.ReplaceAll(matcher[1], `\u002F`, "/") size, err := request.Size(u, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: u, Size: size, Ext: "mp4", } streams[quality] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, Quality: quality, } } return []*extractors.Data{ { Site: "快手 kuaishou.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/kuaishou/kuaishou_test.go ================================================ package kuaishou import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.kuaishou.com/short-video/3x43cyvcyph57i4?authorId=3xtq3uqyjmhbimq&streamSource=find&area=homexxbrilliant", Title: "现在连戴口罩都开始内卷了吗？！快get口罩心机戴法，直接戴出小V脸啊 ！ #口罩 #显脸小-快手", Size: 1077774, Quality: "sd", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/mgtv/mgtv.go ================================================ package mgtv import ( "encoding/base64" "encoding/json" "fmt" "regexp" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("mgtv", New()) } type mgtvVideoStream struct { Name string `json:"name"` URL string `json:"url"` Def string `json:"def"` } type mgtvVideoInfo struct { Title string `json:"title"` Desc string `json:"desc"` } type mgtvVideoData struct { Stream []mgtvVideoStream `json:"stream"` StreamDomain []string `json:"stream_domain"` Info mgtvVideoInfo `json:"info"` } type mgtv struct { Data mgtvVideoData `json:"data"` } type mgtvVideoAddr struct { Info string `json:"info"` } type mgtvURLInfo struct { URL string Size int64 } type mgtvPm2Data struct { Data struct { Atc struct { Pm2 string `json:"pm2"` } `json:"atc"` Info mgtvVideoInfo `json:"info"` } `json:"data"` } func mgtvM3u8(url string) ([]mgtvURLInfo, int64, error) { var data []mgtvURLInfo var temp mgtvURLInfo var size, totalSize int64 urls, err := utils.M3u8URLs(url) if err != nil { return nil, 0, err } m3u8String, err := request.Get(url, url, nil) if err != nil { return nil, 0, err } sizes := utils.MatchAll(m3u8String, `#EXT-MGTV-File-SIZE:(\d+)`) // sizes: [[#EXT-MGTV-File-SIZE:1893724, 1893724]] for index, u := range urls { size, err = strconv.ParseInt(sizes[index][1], 10, 64) if err != nil { return nil, 0, err } totalSize += size temp = mgtvURLInfo{ URL: u, Size: size, } data = append(data, temp) } return data, totalSize, nil } func encodeTk2(str string) string { encodeString := base64.StdEncoding.EncodeToString([]byte(str)) r1 := regexp.MustCompile(`/\+/g`) r2 := regexp.MustCompile(`///g`) r3 := regexp.MustCompile(`/=/g`) r1.ReplaceAllString(encodeString, "_") r2.ReplaceAllString(encodeString, "~") r3.ReplaceAllString(encodeString, "-") encodeString = utils.Reverse(encodeString) return encodeString } type extractor struct{} // New returns a mgtv extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } vid := utils.MatchOneOf( url, `https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html`, `https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html`, ) if vid == nil { vid = utils.MatchOneOf(html, `vid: (\d+),`) } if vid == nil || len(vid) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } // API extract from https://js.mgtv.com/imgotv-miniv6/global/page/play-tv.js // getSource and getPlayInfo function // Chrome Network JS panel headers := map[string]string{ "Cookie": "PM_CHKID=1", } clit := fmt.Sprintf("clit=%d", time.Now().Unix()/1000) pm2DataString, err := request.Get( fmt.Sprintf( "https://pcweb.api.mgtv.com/player/video?video_id=%s&tk2=%s", vid[1], encodeTk2(fmt.Sprintf( "did=f11dee65-4e0d-4d25-bfce-719ad9dc991d|pno=1030|ver=5.5.1|%s", clit, )), ), url, headers, ) if err != nil { return nil, errors.WithStack(err) } var pm2 mgtvPm2Data if err = json.Unmarshal([]byte(pm2DataString), &pm2); err != nil { return nil, errors.WithStack(err) } dataString, err := request.Get( fmt.Sprintf( "https://pcweb.api.mgtv.com/player/getSource?video_id=%s&tk2=%s&pm2=%s", vid[1], encodeTk2(clit), pm2.Data.Atc.Pm2, ), url, headers, ) if err != nil { return nil, errors.WithStack(err) } var mgtvData mgtv if err = json.Unmarshal([]byte(dataString), &mgtvData); err != nil { return nil, errors.WithStack(err) } title := strings.TrimSpace( pm2.Data.Info.Title + " " + pm2.Data.Info.Desc, ) mgtvStreams := mgtvData.Data.Stream var addr mgtvVideoAddr streams := make(map[string]*extractors.Stream) for _, stream := range mgtvStreams { if stream.URL == "" { continue } // real download address addr = mgtvVideoAddr{} addrInfo, err := request.GetByte(mgtvData.Data.StreamDomain[0]+stream.URL, url, headers) if err != nil { return nil, errors.WithStack(err) } if err = json.Unmarshal(addrInfo, &addr); err != nil { return nil, errors.WithStack(err) } m3u8URLs, totalSize, err := mgtvM3u8(addr.Info) if err != nil { return nil, errors.WithStack(err) } urls := make([]*extractors.Part, len(m3u8URLs)) for index, u := range m3u8URLs { urls[index] = &extractors.Part{ URL: u.URL, Size: u.Size, Ext: "ts", } } streams[stream.Def] = &extractors.Stream{ Parts: urls, Size: totalSize, Quality: stream.Name, } } return []*extractors.Data{ { Site: "芒果TV mgtv.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/mgtv/mgtv_test.go ================================================ package mgtv import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test 1", args: test.Args{ URL: "https://www.mgtv.com/b/322712/4317248.html", Title: "我是大侦探 先导片：何炅吴磊邓伦穿越破案", Size: 86169236, Quality: "超清", }, }, { name: "normal test 2", args: test.Args{ URL: "https://www.mgtv.com/b/308703/4197072.html", Title: "芒果捞星闻 2017 诺一为爷爷和姥爷做翻译超萌", Size: 6486376, Quality: "超清", }, }, { name: "vip test", args: test.Args{ URL: "https://www.mgtv.com/b/322865/4352046.html", Title: "向往的生活 第二季 先导片：何炅黄磊回归质朴生活", Size: 453246944, Quality: "超清", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { New().Extract(tt.args.URL, extractors.Options{}) }) } } ================================================ FILE: extractors/miaopai/miaopai.go ================================================ package miaopai import ( "encoding/json" "fmt" "math/rand" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("miaopai", New()) } type miaopaiData struct { Data struct { Description string `json:"description"` MetaData []struct { URLs struct { M string `json:"m"` } `json:"play_urls"` } `json:"meta_data"` } `json:"data"` } func getRandomString(l int) string { s := make([]string, 0) chars := []string{ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "n", "m", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", } for i := 0; i < l; i++ { s = append(s, chars[rand.Intn(len(chars)-1)]) } return strings.Join(s, "") } type extractor struct{} // New returns a miaopai extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { ids := utils.MatchOneOf(url, `/media/([^\./]+)`, `/show(?:/channel)?/([^\./]+)`) if ids == nil || len(ids) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } id := ids[1] randomString := getRandomString(10) var data miaopaiData jsonString, err := request.Get( fmt.Sprintf("https://n.miaopai.com/api/aj_media/info.json?smid=%s&appid=530&_cb=_jsonp%s", id, randomString), url, nil, ) if err != nil { return nil, errors.WithStack(err) } match := utils.MatchOneOf(jsonString, randomString+`\((.*)\);$`) if match == nil || len(match) < 2 { return nil, errors.New("获取视频信息失败。") } err = json.Unmarshal([]byte(match[1]), &data) if err != nil { return nil, errors.WithStack(err) } realURL := data.Data.MetaData[0].URLs.M size, err := request.Size(realURL, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: realURL, Size: size, Ext: "mp4", } streams := map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{urlData}, Size: size, }, } return []*extractors.Data{ { Site: "秒拍 miaopai.com", Title: data.Data.Description, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/miaopai/miaopai_test.go ================================================ package miaopai import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "http://n.miaopai.com/media/Dqg5Pmb~I6lChdvOb-~r1BpKzzDu~MPr", Title: "小学霸6点半起床学习:想赢在起跑线", Size: 6743958, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/netease/netease.go ================================================ package netease import ( netURL "net/url" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("163", New()) } type extractor struct{} // New returns a netease extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { url = strings.Replace(url, "/#/", "/", 1) vid := utils.MatchOneOf(url, `/(mv|video)\?id=(\w+)`) if vid == nil { return nil, errors.New("invalid url for netease music") } html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } if strings.Contains(html, "u-errlg-404") { return nil, errors.New("404 music not found") } titles := utils.MatchOneOf(html, ``) if titles == nil || len(titles) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } title := titles[1] realURLs := utils.MatchOneOf(html, ``) if realURLs == nil || len(realURLs) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } realURL, _ := netURL.QueryUnescape(realURLs[1]) size, err := request.Size(realURL, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: realURL, Size: size, Ext: "mp4", } streams := map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{urlData}, Size: size, }, } return []*extractors.Data{ { Site: "网易云音乐 music.163.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/netease/netease_test.go ================================================ package netease import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "mv test 1", args: test.Args{ URL: "https://music.163.com/#/mv?id=5547010", Title: "There For You", Size: 24249078, }, }, { name: "video test 1", args: test.Args{ URL: "https://music.163.com/#/video?id=C8C9D11629798595BD28451DE3AC9FF4", Title: "＃金曜日の新垣结衣 总集編〈全9編〉", Size: 37408123, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/odysee/odysee.go ================================================ package odysee import ( "compress/flate" "compress/gzip" "encoding/json" "io" "net/http" "regexp" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/pkg/errors" ) func init() { extractors.Register("odysee", New()) } type extractor struct{} type odyseePayload struct { ContentURL string `json:"contentUrl"` Description string `json:"description"` Name string `json:"name"` ThumbnailURL string `json:"thumbnailUrl"` URL string `json:"url"` } // New returns an odysee extractor. func New() extractors.Extractor { return &extractor{} } func (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) { res, err := request.Request(http.MethodGet, u, nil, nil) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var reader io.ReadCloser switch res.Header.Get("Content-Encoding") { case "gzip": reader, _ = gzip.NewReader(res.Body) case "deflate": reader = flate.NewReader(res.Body) default: reader = res.Body } defer reader.Close() // nolint b, err := io.ReadAll(reader) if err != nil { return nil, errors.WithStack(err) } regScript := regexp.MustCompile(`(?im)\`) if err != nil { return nil, errors.WithStack(extractors.ErrInvalidRegularExpression) } matchers := reg.FindAllStringSubmatch(html, -1) var encryptedScript string for _, scripts := range matchers { script := scripts[1] if !strings.Contains(script, "flashvars_") { continue } else { encryptedScript = script break } } flashId := regexp.MustCompile(`flashvars_\d+`).FindString(encryptedScript) vm := otto.New() _, err = vm.Run(`var playerObjList = {};` + encryptedScript + fmt.Sprintf(`;var __VM__OUTPUT = JSON.stringify(%s.mediaDefinitions)`, flashId)) if err != nil { return nil, errors.WithStack(err) } value, err := vm.Get("__VM__OUTPUT") if err != nil { return nil, errors.WithStack(err) } type MediaDefinition struct { Format string `json:"format"` VideoURL string `json:"videoUrl"` } mediaDefinitions := make([]MediaDefinition, 0) if str, err := value.ToString(); err != nil { return nil, errors.WithStack(err) } else { if err := json.Unmarshal([]byte(str), &mediaDefinitions); err != nil { return nil, errors.WithStack(err) } } var mp4MediaDefinition *MediaDefinition for _, mediaDefinition := range mediaDefinitions { if mediaDefinition.Format == "mp4" { mp4MediaDefinition = &mediaDefinition } } if mp4MediaDefinition == nil { return nil, errors.New("can not found media") } resApi, err := request.Get(mp4MediaDefinition.VideoURL, mp4MediaDefinition.VideoURL, map[string]string{ "Cookie": strings.Join(cookiesArr, "; "), }) if err != nil { return nil, errors.WithStack(err) } pornhubs := make([]pornhubData, 0) if err := json.Unmarshal([]byte(resApi), &pornhubs); err != nil { return nil, errors.WithStack(err) } streams := make(map[string]*extractors.Stream, len(pornhubs)) for _, data := range pornhubs { size, err := request.Size(data.VideoURL, data.VideoURL) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: data.VideoURL, Size: size, Ext: data.Format, } streams[data.Quality] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, Quality: data.Quality, } } return []*extractors.Data{ { Site: "Pornhub pornhub.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/pornhub/pornhub_test.go ================================================ package pornhub import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestPornhub(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://www.pornhub.com/view_video.php?viewkey=ph5cb5fc41c6ebd", Title: "Must watch Milf drilled by the fireplace", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { New().Extract(tt.args.URL, extractors.Options{}) }) } } ================================================ FILE: extractors/qq/qq.go ================================================ package qq import ( "encoding/json" "fmt" "slices" "strconv" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("qq", New()) } type qqVideoInfo struct { Fl struct { Fi []struct { ID int `json:"id"` Name string `json:"name"` Cname string `json:"cname"` Fs int64 `json:"fs"` } `json:"fi"` } `json:"fl"` Vl struct { Vi []struct { Fn string `json:"fn"` Ti string `json:"ti"` Fvkey string `json:"fvkey"` Cl struct { Fc int `json:"fc"` Ci []struct { Idx int `json:"idx"` } `json:"ci"` } `json:"cl"` Ul struct { UI []struct { URL string `json:"url"` } `json:"ui"` } `json:"ul"` } `json:"vi"` } `json:"vl"` Msg string `json:"msg"` } type qqKeyInfo struct { Key string `json:"key"` } const qqPlayerVersion string = "3.2.19.333" func getVinfo(vid, defn, refer string) (qqVideoInfo, error) { html, err := request.Get( fmt.Sprintf( "http://vv.video.qq.com/getinfo?otype=json&platform=11&defnpayver=1&appver=%s&defn=%s&vid=%s", qqPlayerVersion, defn, vid, ), refer, nil, ) if err != nil { return qqVideoInfo{}, err } jsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`) if jsonStrings == nil || len(jsonStrings) < 2 { return qqVideoInfo{}, errors.WithStack(extractors.ErrURLParseFailed) } jsonString := jsonStrings[1] var data qqVideoInfo if err = json.Unmarshal([]byte(jsonString), &data); err != nil { return qqVideoInfo{}, err } return data, nil } func genStreams(vid, cdn string, data qqVideoInfo) (map[string]*extractors.Stream, error) { streams := make(map[string]*extractors.Stream) var vkey string // number of fragments var clips int for _, fi := range data.Fl.Fi { var fmtIDPrefix string var fns []string if slices.Contains([]string{"shd", "fhd"}, fi.Name) { fmtIDPrefix = "p" fmtIDName := fmt.Sprintf("%s%d", fmtIDPrefix, fi.ID%10000) fns = []string{strings.Split(data.Vl.Vi[0].Fn, ".")[0], fmtIDName, "mp4"} if len(fns) > 3 { // delete ID part // e0765r4mwcr.2.mp4 -> e0765r4mwcr.mp4 fns = append(fns[:1], fns[2:]...) } clips = data.Vl.Vi[0].Cl.Fc if clips == 0 { clips = 1 } } else { tmpData, err := getVinfo(vid, fi.Name, cdn) if err != nil { return nil, errors.WithStack(err) } fns = strings.Split(tmpData.Vl.Vi[0].Fn, ".") if len(fns) >= 3 && utils.MatchOneOf(fns[1], `^p(\d{3})$`) != nil { fmtIDPrefix = "p" } clips = tmpData.Vl.Vi[0].Cl.Fc if clips == 0 { clips = 1 } } var urls []*extractors.Part var totalSize int64 var filename string for part := 1; part < clips+1; part++ { // Multiple fragments per streams if fmtIDPrefix == "p" { if len(fns) < 4 { // If the number of fragments > 0, the filename needs to add the number of fragments // n0687peq62x.p709.mp4 -> n0687peq62x.p709.1.mp4 fns = append(fns[:2], append([]string{strconv.Itoa(part)}, fns[2:]...)...) } else { fns[2] = strconv.Itoa(part) } } filename = strings.Join(fns, ".") html, err := request.Get( fmt.Sprintf( "http://vv.video.qq.com/getkey?otype=json&platform=11&appver=%s&filename=%s&format=%d&vid=%s", qqPlayerVersion, filename, fi.ID, vid, ), "", nil, ) if err != nil { return nil, errors.WithStack(err) } jsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`) if jsonStrings == nil || len(jsonStrings) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } jsonString := jsonStrings[1] var keyData qqKeyInfo if err = json.Unmarshal([]byte(jsonString), &keyData); err != nil { return nil, errors.WithStack(err) } vkey = keyData.Key if vkey == "" { vkey = data.Vl.Vi[0].Fvkey } realURL := fmt.Sprintf("%s%s?vkey=%s", cdn, filename, vkey) size, err := request.Size(realURL, cdn) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: realURL, Size: size, Ext: "mp4", } urls = append(urls, urlData) totalSize += size } streams[fi.Name] = &extractors.Stream{ Parts: urls, Size: totalSize, Quality: fi.Cname, } } return streams, nil } type extractor struct{} // New returns a qq extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { vids := utils.MatchOneOf(url, `vid=(\w+)`, `/(\w+)\.html`) if vids == nil || len(vids) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } vid := vids[1] if len(vid) != 11 { u, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } vids = utils.MatchOneOf( u, `vid=(\w+)`, `vid:\s*["'](\w+)`, `vid\s*=\s*["']\s*(\w+)`, ) if vids == nil || len(vids) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } vid = vids[1] } data, err := getVinfo(vid, "shd", url) if err != nil { return nil, errors.WithStack(err) } // API request error if data.Msg != "" { return nil, errors.New(data.Msg) } cdn := data.Vl.Vi[0].Ul.UI[0].URL streams, err := genStreams(vid, cdn, data) if err != nil { return nil, errors.WithStack(err) } return []*extractors.Data{ { Site: "腾讯视频 v.qq.com", Title: data.Vl.Vi[0].Ti, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/qq/qq_test.go ================================================ package qq import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://v.qq.com/x/page/n0687peq62x.html", Title: "世界杯第一期：100秒速成！“伪球迷”世界杯生存指南", Size: 23759683, Quality: "蓝光;(1080P)", }, }, // { // name: "movie and vid test", // args: test.Args{ // URL: "https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html", // Title: "赌侠（粤语版）", // Size: 1046910811, // Quality: "超清;(720P)", // }, // }, { name: "fmt ID test", args: test.Args{ URL: "https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html", Title: "《卡路里》出圈！妖娆男子教学广场舞版，大妈表情亮了！", Size: 14112979, Quality: "超清;(720P)", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/reddit/reddit.go ================================================ package reddit import ( "fmt" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("reddit", New()) } const ( referer = "https://www.reddit.com" siteName = "Reddit reddit.com" redditMP4API = "https://v.redd.it/" redditIMGAPI = "https://i.redd.it/" audioURLPart = "/DASH_audio.mp4" ) var resMap = map[string]string{ "720p": "/DASH_720.mp4", "480p": "/DASH_480.mp4", "360p": "/DASH_360.mp4", "240p": "/DASH_240.mp4", "220p": "/DASH_220.mp4", } type extractor struct{} func New() extractors.Extractor { return &extractor{} } func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, referer, nil) if err != nil { return nil, errors.WithStack(err) } // set thread number to 1 manually to avoid http 412 error option.ThreadNumber = 1 title := utils.MatchOneOf(html, `(.+?)<\/title>`)[1] if utils.MatchOneOf(html, `meta property="og:video" content=.*HLSPlaylist`) != nil { mp4URL := utils.MatchOneOf(html, `https://v.redd.it/(.+?)/HLSPlaylist`)[1] if mp4URL == "" { return nil, errors.New("can't match mp4 content downloadable url") } audioURL := fmt.Sprintf("%s%s%s", redditMP4API, mp4URL, audioURLPart) size, err := request.Size(audioURL, referer) if err != nil { return nil, errors.WithStack(err) } audioPart := &extractors.Part{ URL: audioURL, Size: size, Ext: "mp3", } streams := make(map[string]*extractors.Stream, len(resMap)) for res, urlParts := range resMap { resURL := fmt.Sprintf("%s%s%s", redditMP4API, mp4URL, urlParts) size, err := request.Size(resURL, referer) if err != nil { return nil, errors.WithStack(err) } streams[res] = &extractors.Stream{ Parts: []*extractors.Part{ { URL: resURL, Size: size, Ext: "mp4", }, audioPart, }, Size: size + audioPart.Size, Quality: res, NeedMux: true, } } return []*extractors.Data{ { Site: siteName, Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } else if utils.MatchOneOf(html, `<meta property="og:type" content="image"/>`) != nil { var imgURL string var size int64 if utils.MatchOneOf(html, `content":"https:\/\/i.redd.it\/(.+?)","type":"image"`) != nil { imgURL = redditIMGAPI + utils.MatchOneOf(html, `content":"https:\/\/i.redd.it\/(.+?)","type":"image"`)[1] size, err = request.Size(imgURL, referer) if err != nil { return nil, errors.WithStack(err) } } else { imgURL = utils.MatchOneOf(html, `content":"(.+?)","type":"image"`)[1] imgURL = strings.ReplaceAll(imgURL, "auto=webp\\u0026s", "auto=webp&s") size, err = request.Size(imgURL, referer) if err != nil { return nil, errors.WithStack(err) } } return []*extractors.Data{ { Site: siteName, Title: title, Type: extractors.DataTypeImage, Streams: map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{ { URL: imgURL, Size: size, Ext: "jpg", }, }, Size: size, }, }, URL: url, }, }, nil } else if utils.MatchOneOf(html, `https:\/\/preview\.redd\.it\/.*gif`) != nil { gifURL := utils.MatchOneOf(html, `https:\/\/preview\.redd\.it\/.*?\.gif\?format=mp4.*?"`)[0] if gifURL == "" { return nil, errors.New("can't match gif content downloadable url") } gifURL = strings.ReplaceAll(gifURL, "&", "&") gifURL = strings.ReplaceAll(gifURL, "\"", "") size, err := request.Size(gifURL, "reddit.com") if err != nil { return nil, errors.New("can't get video size") } streams := map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{ { URL: gifURL, Size: size, Ext: "mp4", }, }, Size: size, }, } return []*extractors.Data{ { Site: siteName, Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } return nil, fmt.Errorf("unable to handle url: %s", url) } ================================================ FILE: extractors/reddit/reddit_test.go ================================================ package reddit import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestReddit(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test 0", args: test.Args{ URL: "https://www.reddit.com/r/space/comments/uj8sod/a_couple_of_days_ago_i_visited_this_place_an/", Title: "A couple of days ago I visited this place. An abandoned space shuttle : space", }, }, { name: "normal test 1", args: test.Args{ URL: "https://www.reddit.com/r/DotA2/comments/uq012r/til_how_useful_hurricane_bird_is/", Title: "TIL how useful hurricane bird is : DotA2", }, }, { name: "normal test 2", args: test.Args{ URL: "https://www.reddit.com/r/ProgrammerHumor/comments/uqovco/my_code_works/", Title: "My code works : ProgrammerHumor", }, }, { name: "normal test 3", args: test.Args{ URL: "https://www.reddit.com/r/AnimatedPixelArt/comments/uomu32/animation_for_astral_ascent/", Title: "Animation for Astral Ascent : AnimatedPixelArt", }, }, { name: "normal test 4", args: test.Args{ URL: "https://www.reddit.com/r/linuxmemes/comments/v1a4wh/please_olive_do_something/", Title: "Please Olive, do something... : linuxmemes", }, }, { name: "normal test 5", args: test.Args{ URL: "https://www.reddit.com/r/gaming/comments/v27m79/skyrim_probably/", Title: "Skyrim, probably : gaming", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/rumble/rumble.go ================================================ package rumble import ( "compress/flate" "compress/gzip" "encoding/json" "fmt" "io" "net/http" "net/url" "path" "regexp" "strconv" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("rumble", New()) } type extractor struct{} // New returns a rumble extractor. func New() extractors.Extractor { return &extractor{} } type rumbleData struct { Format string `json:"format"` Name string `json:"name"` EmbedURL string `json:"embedUrl"` ThumbnailURL string `json:"thumbnailUrl"` Type string `json:"@type"` VideoURL string `json:"videoUrl"` Quality string `json:"quality"` } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { res, err := request.Request(http.MethodGet, url, nil, nil) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var reader io.ReadCloser switch res.Header.Get("Content-Encoding") { case "gzip": reader, _ = gzip.NewReader(res.Body) case "deflate": reader = flate.NewReader(res.Body) default: reader = res.Body } defer reader.Close() // nolint b, err := io.ReadAll(reader) if err != nil { return nil, errors.WithStack(err) } html := string(b) var title string matchTitle := utils.MatchOneOf(html, `<title>(.+?)`) if len(matchTitle) > 1 { title = matchTitle[1] } else { title = "rumble video" } payload, err := readPayload(html) if err != nil { return nil, errors.WithStack(err) } videoID, err := getVideoID(payload.EmbedURL) if err != nil { return nil, errors.WithStack(err) } streams, err := fetchVideoQuality(videoID) if err != nil { return nil, errors.WithStack(err) } return []*extractors.Data{ { Site: "Rumble rumble.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } // Read JSON object from the video webpage func readPayload(html string) (*rumbleData, error) { matchPayload := utils.MatchOneOf(html, `\(.+?)\<\/script>`) if len(matchPayload) < 1 { return nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed) } rumbles := make([]rumbleData, 0) if err := json.Unmarshal([]byte(matchPayload[1]), &rumbles); err != nil { return nil, errors.WithStack(err) } for _, it := range rumbles { if it.Type == "VideoObject" { return &it, nil } } return nil, errors.WithStack(extractors.ErrURLParseFailed) } func getVideoID(embedURL string) (string, error) { u, err := url.Parse(embedURL) if err != nil { return "", errors.WithStack(extractors.ErrURLParseFailed) } return path.Base(u.Path), nil } // Rumble response contains the streams in `rumbleStreams` type rumbleResponse struct { Streams *json.RawMessage `json:"ua"` } // Common video meta data type streamInfo struct { URL string `json:"url"` Meta struct { Bitrate uint16 `json:"bitrate"` Size int64 `json:"size"` Width uint16 `json:"w"` Height uint16 `json:"h"` } `json:"meta"` } // common video qualities for `mp4`, `webm` type videoQualities struct { Q240 struct{ streamInfo } `json:"240"` Q360 struct{ streamInfo } `json:"360"` Q480 struct{ streamInfo } `json:"480"` Q720 struct{ streamInfo } `json:"720"` Q1080 struct{ streamInfo } `json:"1080"` Q1440 struct{ streamInfo } `json:"1440"` Q2160 struct{ streamInfo } `json:"2160"` Q2161 struct{ streamInfo } `json:"2161"` } // Video payload for adaptive stream and different qualities type rumbleStreams struct { FMp4 struct { videoQualities } `json:"mp4"` FWebm struct { videoQualities } `json:"webm"` FHLS struct { QAuto struct{ streamInfo } `json:"auto"` } `json:"hls"` FTAR map[string]streamInfo `json:"tar"` } // Unmarshall the video response // Some properties like `mp4`, `webm` are either array or an object func (r *rumbleStreams) UnmarshalJSON(b []byte) error { var resp *rumbleResponse if err := json.Unmarshal(b, &resp); err != nil { return errors.WithStack(extractors.ErrURLParseFailed) } // Get individual stream from the response var obj map[string]*json.RawMessage if err := json.Unmarshal(*resp.Streams, &obj); err != nil { return errors.WithStack(extractors.ErrURLParseFailed) } if v, ok := obj["mp4"]; ok { _ = json.Unmarshal(*v, &r.FMp4) } if v, ok := obj["webm"]; ok { _ = json.Unmarshal(*v, &r.FWebm) } if v, ok := obj["hls"]; ok { _ = json.Unmarshal(*v, &r.FHLS) } if v, ok := obj["tar"]; ok { _ = json.Unmarshal(*v, &r.FTAR) } return nil } // Use this to create all the streams for `mp4`, `webm` func (rs *rumbleStreams) makeAllVODStreams(m map[string]*extractors.Stream) { m["webm"] = makeStreamMeta("480", "webm", &rs.FWebm.Q480.streamInfo) m["240"] = makeStreamMeta("240", "mp4", &rs.FMp4.Q240.streamInfo) m["360"] = makeStreamMeta("360", "mp4", &rs.FMp4.Q360.streamInfo) m["480"] = makeStreamMeta("480", "mp4", &rs.FMp4.Q480.streamInfo) m["720"] = makeStreamMeta("720", "mp4", &rs.FMp4.Q720.streamInfo) m["1080"] = makeStreamMeta("1080", "mp4", &rs.FMp4.Q1080.streamInfo) m["1440"] = makeStreamMeta("1440", "mp4", &rs.FMp4.Q1440.streamInfo) m["2160"] = makeStreamMeta("2160", "mp4", &rs.FMp4.Q2160.streamInfo) m["2161"] = makeStreamMeta("2161", "mp4", &rs.FMp4.Q2161.streamInfo) } var reResolution = regexp.MustCompile(`_(\d{3,4})p\/`) // ex. _720p/ // Use this to create all the streams for live videos func (rs *rumbleStreams) makeAllLiveStreams(m map[string]*extractors.Stream) error { playlists, err := utils.M3u8URLs(rs.FHLS.QAuto.URL) if err != nil { return errors.WithStack(err) } if len(playlists) == 0 { return errors.WithStack(extractors.ErrURLParseFailed) } // Find the highest resolution playlistURL := playlists[0] maxRes := 0 for _, x := range playlists { matched := reResolution.FindStringSubmatch(x) if len(matched) == 0 { continue } res, err := strconv.Atoi(matched[1]) if err != nil { continue } if maxRes < res { maxRes = res playlistURL = x } } tsURLs, err := utils.M3u8URLs(playlistURL) if err != nil { return errors.WithStack(err) } var parts []*extractors.Part for _, x := range tsURLs { part := &extractors.Part{ URL: x, Size: rs.FHLS.QAuto.streamInfo.Meta.Size, Ext: "ts", } parts = append(parts, part) } m["hls"] = &extractors.Stream{ Parts: parts, Size: rs.FHLS.QAuto.streamInfo.Meta.Size, Quality: strconv.Itoa(maxRes), } return nil } func (rs *rumbleStreams) makeAllNewVodStreams(m map[string]*extractors.Stream) error { for size, details := range rs.FTAR { playlists, err := utils.M3u8URLs(details.URL) if err != nil { return errors.WithStack(err) } if len(playlists) == 0 { return errors.WithStack(extractors.ErrURLParseFailed) } var parts []*extractors.Part for _, x := range playlists { part := &extractors.Part{ URL: x, Size: details.Meta.Size, Ext: "ts", } parts = append(parts, part) } m[size] = &extractors.Stream{ Parts: parts, Size: details.Meta.Size, Quality: strconv.Itoa(int(details.Meta.Height)), } } return nil } // Request video formats and qualities func fetchVideoQuality(videoID string) (map[string]*extractors.Stream, error) { reqURL := fmt.Sprintf(`https://rumble.com/embedJS/u3/?request=video&ver=2&v=%s&ext={"ad_count":null}&ad_wt=0`, videoID) res, err := request.Request(http.MethodGet, reqURL, nil, nil) if err != nil { return nil, errors.WithStack(err) } defer res.Body.Close() // nolint var reader io.ReadCloser switch res.Header.Get("Content-Encoding") { case "gzip": reader, _ = gzip.NewReader(res.Body) case "deflate": reader = flate.NewReader(res.Body) default: reader = res.Body } defer reader.Close() // nolint b, err := io.ReadAll(reader) if err != nil { return nil, errors.WithStack(err) } var rs rumbleStreams if err := json.Unmarshal(b, &rs); err != nil { return nil, errors.WithStack(err) } streams := make(map[string]*extractors.Stream, 9) rs.makeAllVODStreams(streams) _ = rs.makeAllLiveStreams(streams) _ = rs.makeAllNewVodStreams(streams) return streams, nil } func makeStreamMeta(q, ext string, info *streamInfo) *extractors.Stream { urlMeta := &extractors.Part{ URL: info.URL, Size: info.Meta.Size, Ext: ext, } return &extractors.Stream{ Parts: []*extractors.Part{urlMeta}, Size: info.Meta.Size, Quality: q, } } ================================================ FILE: extractors/rumble/rumble_test.go ================================================ package rumble import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestRumble(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html", Title: "Just Say YES to Climate Lockdowns!", }, }, { name: "normal test", args: test.Args{ URL: "https://rumble.com/v6rmfm1-monday-full-show-33125-hhs-head-rfk-jr.-pledges-to-stopv.html", Title: "MONDAY FULL SHOW 3/31/25 — HHS Head RFK Jr. Pledges To Stopv", }, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) if err != nil { t.Error(err) } for _, d := range data { found := false for _, s := range d.Streams { if s.Size > 0 { found = true } } if !found { t.Errorf("no streams found in test %d", i) } } }) } } ================================================ FILE: extractors/streamtape/streamtape.go ================================================ package streamtape import ( "fmt" "github.com/pkg/errors" "github.com/robertkrimen/otto" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { e := New() extractors.Register("streamtape", e) extractors.Register("streamta", e) // streamta.pe } type extractor struct{} // New returns a StreamTape extractor func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, nil) if err != nil { return nil, errors.WithStack(err) } scripts := utils.MatchOneOf(html, `document.getElementById\('norobotlink'\).innerHTML = (.+?);`) if len(scripts) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } vm := otto.New() _, err = vm.Run(fmt.Sprintf("var __VM__OUTPUT = %s", scripts[1])) if err != nil { return nil, errors.WithStack(err) } value, err := vm.Get("__VM__OUTPUT") if err != nil { return nil, errors.WithStack(err) } u, err := value.ToString() // //streamtape.com/get_video?id=xx&expires=xx&ip=xx&token=xx if err != nil { return nil, errors.WithStack(err) } u = fmt.Sprintf("https:%s&stream=1", u) // get title var title = "StreamTape Video" titleMatch := utils.MatchOneOf(html, `\`) if len(titleMatch) >= 2 { title = titleMatch[1] } size, err := request.Size(u, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: u, Size: size, Ext: "mp4", } streams := make(map[string]*extractors.Stream) streams["default"] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, } return []*extractors.Data{ { URL: u, Site: "StreamTape streamtape.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, }, }, nil } ================================================ FILE: extractors/streamtape/streamtape_test.go ================================================ package streamtape import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestStreamtape(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test", args: test.Args{ URL: "https://streamtape.com/v/YKLDrr4X9gSvm9q/00819gb0ly1gd4okz3fqbg30b405jnpj.mp4", Title: "00819gb0ly1gd4okz3fqbg30b405jnpj.mp4", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/tangdou/tangdou.go ================================================ package tangdou import ( "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("tangdou", New()) } type extractor struct{} // New returns a tangdou extractor. func New() extractors.Extractor { return &extractor{} } var defaultHeader = map[string]string{ "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "cross-site", "Sec-GPC": "1", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0", } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { return []*extractors.Data{tangdouDownload(url)}, nil } // tangdouDownload download function for single url func tangdouDownload(uri string) *extractors.Data { html, err := request.Get(uri, uri, defaultHeader) if err != nil { return extractors.EmptyData(uri, err) } titles := utils.MatchOneOf( html, `
(.+?)
`, `(.+?)`, ) if titles == nil || len(titles) < 2 { return extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed)) } title := titles[1] videoURLs := utils.MatchOneOf( html, `video:'(.+?)'`, `video:"(.+?)"`, `]*src="(.+?)"`, `play_url:\s*"(.+?)",`, ) if len(videoURLs) < 2 { return extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed)) } realURL := strings.ReplaceAll(videoURLs[1], `\u002F`, "/") size, err := request.Size(realURL, uri) if err != nil { return extractors.EmptyData(uri, err) } streams := map[string]*extractors.Stream{ "default": { Parts: []*extractors.Part{ { URL: realURL, Size: size, Ext: "mp4", }, }, Size: size, }, } return &extractors.Data{ Site: "糖豆广场舞 tangdou.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: uri, } } ================================================ FILE: extractors/tangdou/tangdou_test.go ================================================ package tangdou import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestTangDou(t *testing.T) { tests := []struct { name string args test.Args playlist bool }{ { name: "need call share url first and get the signed video URL test and can get title from head's title tag", args: test.Args{ URL: "https://m.tangdou.com/play/1500676338077", Title: "暴瘦减肚子，不用跑不用跳，8天瘦了16斤 正面演示 背面演示 分解教学__广场舞_糖豆广场舞-糖豆视频", Size: 62258444, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var ( data []*extractors.Data err error ) if tt.playlist { // playlist mode _, err = New().Extract(tt.args.URL, extractors.Options{ Playlist: true, ThreadNumber: 9, }) test.CheckError(t, err) } else { data, err = New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) } }) } } ================================================ FILE: extractors/threads/threads.go ================================================ package threads import ( "fmt" "net" "net/http" netURL "net/url" "strings" "time" "github.com/gocolly/colly/v2" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("threads", New()) } type extractor struct { client *http.Client } // New returns a instagram extractor. func New() extractors.Extractor { return &extractor{ client: &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{ Dial: (&net.Dialer{ Timeout: 5 * time.Second, }).Dial, TLSHandshakeTimeout: 5 * time.Second, }, }, } } type media struct { URL string Type extractors.DataType } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { URL, err := netURL.Parse(url) if err != nil { return nil, errors.WithStack(err) } paths := strings.Split(URL.Path, "/") if len(paths) < 3 { return nil, errors.New("invalid URL format") } poster := paths[1] shortCode := paths[3] medias := make([]media, 0) title := fmt.Sprintf("Threads %s - %s", poster, shortCode) collector := colly.NewCollector() collector.SetClient(e.client) // case single image or video collector.OnHTML("div.SingleInnerMediaContainer", func(e *colly.HTMLElement) { if src := e.ChildAttr("img", "src"); src != "" { medias = append(medias, media{ URL: src, Type: extractors.DataTypeImage, }) } if src := e.ChildAttr("video > source", "src"); src != "" { medias = append(medias, media{ URL: src, Type: extractors.DataTypeVideo, }) } }) // case multiple image or video collector.OnHTML("div.MediaScrollImageContainer", func(e *colly.HTMLElement) { if src := e.ChildAttr("img", "src"); src != "" { medias = append(medias, media{ URL: src, Type: extractors.DataTypeImage, }) } if src := e.ChildAttr("video > source", "src"); src != "" { medias = append(medias, media{ URL: src, Type: extractors.DataTypeVideo, }) } }) // title with caption // collector.OnHTML("span.BodyTextContainer", func(e *colly.HTMLElement) { // title = e.Text // }) if err := collector.Visit(URL.JoinPath("embed").String()); err != nil { return nil, fmt.Errorf("failed to send HTTP request to the Threads: %w", errors.WithStack(err)) } var totalSize int64 var parts []*extractors.Part for _, m := range medias { _, ext, err := utils.GetNameAndExt(m.URL) if err != nil { return nil, errors.WithStack(err) } fileSize, err := request.Size(m.URL, url) if err != nil { return nil, errors.WithStack(err) } part := &extractors.Part{ URL: m.URL, Size: fileSize, Ext: ext, } parts = append(parts, part) } for _, part := range parts { totalSize += part.Size } streams := map[string]*extractors.Stream{ "default": { Parts: parts, Size: totalSize, }, } return []*extractors.Data{ { Site: "Threads www.threads.net", Title: title, Type: extractors.DataTypeImage, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/threads/threads_test.go ================================================ package threads_test import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/extractors/threads" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "video test", args: test.Args{ URL: "https://www.threads.net/@rowancheung/post/C9xPmHcpfiN", Title: `Threads @rowancheung - C9xPmHcpfiN`, Size: 5740684, }, }, { name: "video shared test", args: test.Args{ URL: "https://www.threads.net/@zuck/post/C9xRqbNPbx2", Title: `Threads @zuck - C9xRqbNPbx2`, Size: 5740684, }, }, { name: "image test", args: test.Args{ URL: "https://www.threads.net/@zuck/post/C-BoS7lM8sH", Title: `Threads @zuck - C-BoS7lM8sH`, Size: 159331, }, }, { name: "hybrid album test", args: test.Args{ URL: "https://www.threads.net/@meta/post/C95Z1DrPNhi", Title: `Threads @meta - C95Z1DrPNhi`, Size: 1131229, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := threads.New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/tiktok/tiktok.go ================================================ package tiktok import ( "regexp" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" ) func init() { extractors.Register("tiktok", New()) } type extractor struct{} // New returns a tiktok extractor. func New() extractors.Extractor { return &extractor{} } // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { html, err := request.Get(url, url, map[string]string{ // tiktok require a user agent "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0", }) if err != nil { return nil, errors.WithStack(err) } urlMatcherRegExp := regexp.MustCompile(`"downloadAddr":\s*"([^"]+)"`) downloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html) if len(downloadURLMatcher) == 0 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } videoURL := strings.ReplaceAll(downloadURLMatcher[1], `\u002F`, "/") titleMatcherRegExp := regexp.MustCompile(`]*>([^<]+)`) titleMatcherRegExpOpt := regexp.MustCompile(`"desc":"([^"]*)"`) titleMatcher := titleMatcherRegExp.FindStringSubmatch(html) titleMatcherOpt := titleMatcherRegExpOpt.FindStringSubmatch(html) if len(titleMatcher) == 0 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } title := titleMatcher[1] if title == "TikTok - Make Your Day" { if len(titleMatcherOpt[1]) > 64 { cutoff := titleMatcherOpt[1][:64] lastSpace := strings.LastIndex(cutoff, " ") title = titleMatcherOpt[1][:lastSpace] } else { title = titleMatcherOpt[1] } } titleArr := strings.Split(title, "|") if len(titleArr) == 1 { title = titleArr[0] } else { title = strings.TrimSpace(strings.Join(titleArr[:len(titleArr)-1], "|")) } streams := make(map[string]*extractors.Stream) size, err := request.Size(videoURL, url) if err != nil { return nil, errors.WithStack(err) } urlData := &extractors.Part{ URL: videoURL, Size: size, Ext: "mp4", } streams["default"] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, } return []*extractors.Data{ { Site: "TikTok tiktok.com", Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, }, }, nil } ================================================ FILE: extractors/tiktok/tiktok_test.go ================================================ package tiktok import ( "testing" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/test" ) func TestDownload(t *testing.T) { tests := []struct { name string args test.Args }{ { name: "normal test 1", args: test.Args{ URL: "https://www.tiktok.com/@ginjiro_koyama/video/7164293510617763073?is_copy_url=1&is_from_webapp=v1", Title: "イケすぎたXOXO#xoxo #repezenfoxx #背中男 #kfam #yoshikiさんを泣かせたチーム @K fam @【Repezen Foxx】🦊", Size: 4356253, }, }, { name: "normal test 2", args: test.Args{ URL: "https://www.tiktok.com/@enhypen/video/7165445991238356225?is_copy_url=1&is_from_webapp=v1", Title: "깜짝 퇴장 👋 #ENHYPEN #SUNGHOON #NI_KI #Make_the_change", Size: 3848307, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { data, err := New().Extract(tt.args.URL, extractors.Options{}) test.CheckError(t, err) test.Check(t, tt.args, data[0]) }) } } ================================================ FILE: extractors/tumblr/tumblr.go ================================================ package tumblr import ( "encoding/json" "strings" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) func init() { extractors.Register("tumblr", New()) } type imageList struct { List []string `json:"@list"` } type tumblrImageList struct { Image imageList `json:"image"` } type tumblrImage struct { Image string `json:"image"` } func genURLData(url, referer string) (*extractors.Part, int64, error) { size, err := request.Size(url, referer) if err != nil { return nil, 0, err } _, ext, err := utils.GetNameAndExt(url) if err != nil { return nil, 0, err } return &extractors.Part{ URL: url, Size: size, Ext: ext, }, size, nil } func tumblrImageDownload(url, html, title string) ([]*extractors.Data, error) { jsonStrings := utils.MatchOneOf( html, ``, ) if jsonStrings == nil || len(jsonStrings) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } jsonString := jsonStrings[1] var totalSize int64 urls := make([]*extractors.Part, 0, 1) if strings.Contains(jsonString, `"image":{"@list"`) { // there are two data structures in the same field(image) var imageList tumblrImageList if err := json.Unmarshal([]byte(jsonString), &imageList); err != nil { return nil, errors.WithStack(err) } for _, u := range imageList.Image.List { urlData, size, err := genURLData(u, url) if err != nil { return nil, errors.WithStack(err) } totalSize += size urls = append(urls, urlData) } } else { var image tumblrImage if err := json.Unmarshal([]byte(jsonString), &image); err != nil { return nil, errors.WithStack(err) } urlData, size, err := genURLData(image.Image, url) if err != nil { return nil, errors.WithStack(err) } totalSize = size urls = append(urls, urlData) } streams := map[string]*extractors.Stream{ "default": { Parts: urls, Size: totalSize, }, } return []*extractors.Data{ { Site: "Tumblr tumblr.com", Title: title, Type: extractors.DataTypeImage, Streams: streams, URL: url, }, }, nil } func tumblrVideoDownload(url, html, title string) ([]*extractors.Data, error) { videoURLs := utils.MatchOneOf(html, `